This is an automated email from the ASF dual-hosted git repository. sankarh pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new d0f77cc HIVE-25653: Incorrect results returned by STDDEV, STDDEV_SAMP, STDDEV_POP for floating point data types (Ashish Sharma, reviewed by Adesh Rao, Sankar Hariappan) d0f77cc is described below commit d0f77cca1a6612894837a174440a5fd929cd3bcb Author: Ashish Kumar Sharma <ashishkumarsharm...@gmail.com> AuthorDate: Mon Nov 8 12:23:55 2021 +0530 HIVE-25653: Incorrect results returned by STDDEV, STDDEV_SAMP, STDDEV_POP for floating point data types (Ashish Sharma, reviewed by Adesh Rao, Sankar Hariappan) Signed-off-by: Sankar Hariappan <sank...@apache.org> Closes (#2760) --- .../hadoop/hive/ql/udf/generic/GenericUDAFStd.java | 8 +- .../hive/ql/udf/generic/GenericUDAFVariance.java | 29 ++++-- ql/src/test/queries/clientpositive/stddev.q | 14 +++ .../clientpositive/llap/cbo_rp_windowing_2.q.out | 42 ++++----- .../test/results/clientpositive/llap/stddev.q.out | 102 +++++++++++++++++++++ .../clientpositive/llap/vector_windowing.q.out | 42 ++++----- .../results/clientpositive/llap/windowing.q.out | 42 ++++----- 7 files changed, 205 insertions(+), 74 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java index 79b519c..729455c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java @@ -27,6 +27,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import java.math.BigDecimal; +import java.math.MathContext; + /** * Compute the standard deviation by extending GenericUDAFVariance and * overriding the terminate() method of the evaluator. @@ -90,7 +93,10 @@ public class GenericUDAFStd extends GenericUDAFVariance { * use it, etc. */ public static double calculateStdResult(double variance, long count) { - return Math.sqrt(variance / count); + // TODO: BigDecimal.sqrt() is introduced in java 9. So change the below calculation once hive upgraded to java 9 or above. + BigDecimal bvariance = new BigDecimal(variance); + BigDecimal result = bvariance.divide(new BigDecimal(count), MathContext.DECIMAL128); + return Math.sqrt(result.doubleValue()); } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java index bb55d88..5e60edc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hive.ql.udf.generic; +import java.math.BigDecimal; +import java.math.MathContext; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; @@ -106,9 +108,14 @@ public class GenericUDAFVariance extends AbstractGenericUDAFResolver { */ public static double calculateIntermediate( long count, double sum, double value, double variance) { - double t = count * value - sum; - variance += (t * t) / ((double) count * (count - 1)); - return variance; + BigDecimal bcount,bsum,bvalue,bvariance; + bvariance = new BigDecimal(variance); + bsum = new BigDecimal(sum); + bvalue = new BigDecimal(value); + bcount = new BigDecimal(count); + BigDecimal t = bcount.multiply(bvalue).subtract(bsum); + bvariance = bvariance.add(t.multiply(t).divide(bcount.multiply(bcount.subtract(BigDecimal.ONE)),MathContext.DECIMAL128)); + return bvariance.doubleValue(); } /* @@ -120,14 +127,16 @@ public class GenericUDAFVariance extends AbstractGenericUDAFResolver { long partialCount, long mergeCount, double partialSum, double mergeSum, double partialVariance, double mergeVariance) { - final double doublePartialCount = (double) partialCount; - final double doubleMergeCount = (double) mergeCount; + final BigDecimal bPartialCount = new BigDecimal(partialCount); + final BigDecimal bMergeCount = new BigDecimal(mergeCount); + BigDecimal bmergeVariance = new BigDecimal(mergeVariance); - double t = (doublePartialCount / doubleMergeCount) * mergeSum - partialSum; - mergeVariance += - partialVariance + ((doubleMergeCount / doublePartialCount) / - (doubleMergeCount + doublePartialCount)) * t * t; - return mergeVariance; + BigDecimal t = + bPartialCount.divide(bMergeCount, MathContext.DECIMAL128).multiply(new BigDecimal(mergeSum)).subtract(new BigDecimal(partialSum)); + + bmergeVariance = bmergeVariance.add(new BigDecimal(partialVariance).add( + (bMergeCount.divide(bPartialCount,MathContext.DECIMAL128).divide(bMergeCount.add(bPartialCount),MathContext.DECIMAL128)).multiply(t).multiply(t))); + return bmergeVariance.doubleValue(); } /* diff --git a/ql/src/test/queries/clientpositive/stddev.q b/ql/src/test/queries/clientpositive/stddev.q new file mode 100644 index 0000000..5f3d9ed --- /dev/null +++ b/ql/src/test/queries/clientpositive/stddev.q @@ -0,0 +1,14 @@ +create table test ( col1 decimal(10,3) ); +insert into test values (10230.72),(10230.72),(10230.72),(10230.72),(10230.72),(10230.72),(10230.72); +select STDDEV_SAMP(col1) AS STDDEV_6M , STDDEV(col1) as STDDEV ,STDDEV_POP(col1) as STDDEV_POP , variance(col1) as variance,var_pop(col1) as var_pop,var_samp(col1) as var_samp from test; +drop table test; + +create table testpoint ( col1 decimal(10,3)); +insert into testpoint values (0.12345678),(0.25362123),(0.62437485),(0.65133746),(0.98765432),(0.12435647),(0.7654321445); +select STDDEV_SAMP(col1) AS STDDEV_6M , STDDEV(col1) as STDDEV ,STDDEV_POP(col1) as STDDEV_POP , variance(col1) as variance,var_pop(col1) as var_pop,var_samp(col1) as var_samp from testpoint; +drop table testpoint; + +create table testint(col1 int); +insert into testint values (85),(86),(100),(76),(81),(93),(84),(99),(71),(69),(93),(85),(81),(87),(89); +select STDDEV_SAMP(col1) AS STDDEV_6M , STDDEV(col1) as STDDEV ,STDDEV_POP(col1) as STDDEV_POP, variance(col1) as variance,var_pop(col1) as var_pop,var_samp(col1) as var_samp from testint; +drop table testint; diff --git a/ql/src/test/results/clientpositive/llap/cbo_rp_windowing_2.q.out b/ql/src/test/results/clientpositive/llap/cbo_rp_windowing_2.q.out index 6ec2d60..4146bc0 100644 --- a/ql/src/test/results/clientpositive/llap/cbo_rp_windowing_2.q.out +++ b/ql/src/test/results/clientpositive/llap/cbo_rp_windowing_2.q.out @@ -633,9 +633,9 @@ Manufacturer#1 almond aquamarine burnished black steel 28 5 4 0.8333333333333334 Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 1.0 1.0 3 6 19.0 16.237815945091466 2 42 6 Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 0.2 0.0 1 1 14.0 0.0 4 14 14 Manufacturer#2 almond antique violet turquoise frosted 40 2 2 0.4 0.25 1 2 27.0 13.0 4 40 14 -Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 0.6 0.5 2 3 18.666666666666668 15.86050300449376 4 2 14 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 0.6 0.5 2 3 18.666666666666668 15.860503004493758 4 2 14 Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 0.8 0.75 2 4 20.25 14.00669482783144 4 25 40 -Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1.0 1.0 3 5 19.8 12.560254774486067 4 18 2 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1.0 1.0 3 5 19.8 12.560254774486065 4 18 2 Manufacturer#3 almond antique chartreuse khaki white 17 1 1 0.2 0.0 1 1 17.0 0.0 2 17 17 Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 0.4 0.25 1 2 15.5 1.5 2 14 17 Manufacturer#3 almond antique metallic orange dim 19 3 3 0.6 0.5 2 3 16.666666666666668 2.0548046676563256 2 19 17 @@ -862,31 +862,31 @@ window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 precedi POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Manufacturer#1 almond antique burnished rose metallic 2 258.10677784349235 258.10677784349235 [2,34,6] 66619.10876874991 0.81133 2801.7074999999995 -Manufacturer#1 almond antique burnished rose metallic 2 273.70217881648074 273.70217881648074 [2,34] 74912.8826888888 1.0 4128.782222222221 -Manufacturer#1 almond antique chartreuse lavender yellow 34 230.90151585470358 230.90151585470358 [2,34,6,28] 53315.51002399992 0.69564 2210.7864 -Manufacturer#1 almond antique salmon chartreuse burlywood 6 202.73109328368946 202.73109328368946 [2,34,6,28,42] 41099.896184 0.63079 2009.9536000000007 +Manufacturer#1 almond antique burnished rose metallic 2 258.1067778434924 258.1067778434924 [2,34,6] 66619.10876874994 0.81133 2801.7074999999995 +Manufacturer#1 almond antique burnished rose metallic 2 273.7021788164808 273.7021788164808 [2,34] 74912.88268888883 1.0 4128.782222222221 +Manufacturer#1 almond antique chartreuse lavender yellow 34 230.9015158547036 230.9015158547036 [2,34,6,28] 53315.510023999945 0.69564 2210.7864 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 202.7310932836895 202.7310932836895 [2,34,6,28,42] 41099.89618400001 0.63079 2009.9536000000007 Manufacturer#1 almond aquamarine burnished black steel 28 121.6064517973862 121.6064517973862 [34,6,28,42] 14788.129118750014 0.20367 331.1337500000004 -Manufacturer#1 almond aquamarine pink moccasin thistle 42 96.5751586416853 96.5751586416853 [6,28,42] 9326.761266666683 -1.4E-4 -0.20666666666708502 -Manufacturer#2 almond antique violet chocolate turquoise 14 142.2363169751898 142.2363169751898 [14,40,2] 20231.169866666663 -0.4937 -1113.7466666666658 -Manufacturer#2 almond antique violet turquoise frosted 40 137.76306498840682 137.76306498840682 [14,40,2,25] 18978.662075 -0.52056 -1004.4812499999995 -Manufacturer#2 almond aquamarine midnight light salmon 2 130.03972279269132 130.03972279269132 [14,40,2,25,18] 16910.329504000005 -0.46909 -766.1791999999995 -Manufacturer#2 almond aquamarine rose maroon antique 25 135.55100986344584 135.55100986344584 [40,2,25,18] 18374.07627499999 -0.60914 -1128.1787499999987 -Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 156.44019460768044 156.44019460768044 [2,25,18] 24473.534488888927 -0.95717 -1441.4466666666676 -Manufacturer#3 almond antique chartreuse khaki white 17 196.7742266885805 196.7742266885805 [17,14,19] 38720.09628888887 0.55572 224.6944444444446 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 96.57515864168526 96.57515864168526 [6,28,42] 9326.761266666676 -1.4E-4 -0.20666666666708502 +Manufacturer#2 almond antique violet chocolate turquoise 14 142.23631697518968 142.23631697518968 [14,40,2] 20231.16986666663 -0.4937 -1113.7466666666658 +Manufacturer#2 almond antique violet turquoise frosted 40 137.7630649884067 137.7630649884067 [14,40,2,25] 18978.662074999975 -0.52056 -1004.4812499999995 +Manufacturer#2 almond aquamarine midnight light salmon 2 130.03972279269126 130.03972279269126 [14,40,2,25,18] 16910.32950399999 -0.46909 -766.1791999999995 +Manufacturer#2 almond aquamarine rose maroon antique 25 135.55100986344587 135.55100986344587 [40,2,25,18] 18374.076274999996 -0.60914 -1128.1787499999987 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 156.44019460768038 156.44019460768038 [2,25,18] 24473.53448888891 -0.95717 -1441.4466666666676 +Manufacturer#3 almond antique chartreuse khaki white 17 196.77422668858048 196.77422668858048 [17,14,19] 38720.09628888886 0.55572 224.6944444444446 Manufacturer#3 almond antique forest lavender goldenrod 14 275.14144189852607 275.14144189852607 [17,14,19,1] 75702.81305 -0.67208 -1296.9000000000003 Manufacturer#3 almond antique metallic orange dim 19 260.23473614412046 260.23473614412046 [17,14,19,1,45] 67722.117896 -0.57035 -2129.0664 -Manufacturer#3 almond antique misty red olive 1 275.9139962356932 275.9139962356932 [14,19,1,45] 76128.53331875012 -0.57748 -2547.7868749999993 +Manufacturer#3 almond antique misty red olive 1 275.9139962356931 275.9139962356931 [14,19,1,45] 76128.53331875007 -0.57748 -2547.7868749999993 Manufacturer#3 almond antique olive coral navajo 45 260.5815918713796 260.5815918713796 [19,1,45] 67902.76602222225 -0.87107 -4099.731111111111 -Manufacturer#4 almond antique gainsboro frosted violet 10 170.13011889596618 170.13011889596618 [10,39,27] 28944.25735555559 -0.6657 -1347.4777777777779 -Manufacturer#4 almond antique violet mint lemon 39 242.26834609323197 242.26834609323197 [10,39,27,7] 58693.95151875002 -0.80519 -2537.328125 +Manufacturer#4 almond antique gainsboro frosted violet 10 170.13011889596615 170.13011889596615 [10,39,27] 28944.257355555583 -0.6657 -1347.4777777777779 +Manufacturer#4 almond antique violet mint lemon 39 242.26834609323194 242.26834609323194 [10,39,27,7] 58693.951518750015 -0.80519 -2537.328125 Manufacturer#4 almond aquamarine floral ivory bisque 27 234.10001662537326 234.10001662537326 [10,39,27,7,12] 54802.817784000035 -0.60469 -1719.8079999999995 -Manufacturer#4 almond aquamarine yellow dodger mint 7 247.3342714197732 247.3342714197732 [39,27,7,12] 61174.24181875003 -0.55087 -1719.0368749999975 +Manufacturer#4 almond aquamarine yellow dodger mint 7 247.33427141977316 247.33427141977316 [39,27,7,12] 61174.24181875002 -0.55087 -1719.0368749999975 Manufacturer#4 almond azure aquamarine papaya violet 12 283.3344330566893 283.3344330566893 [27,7,12] 80278.40095555557 -0.77557 -1867.4888888888881 -Manufacturer#5 almond antique blue firebrick mint 31 83.69879024746363 83.69879024746363 [31,6,2] 7005.487488888913 0.39004 418.9233333333353 -Manufacturer#5 almond antique medium spring khaki 6 316.68049612345885 316.68049612345885 [31,6,2,46] 100286.53662500004 -0.71361 -4090.853749999999 -Manufacturer#5 almond antique sky peru orange 2 285.40506298242155 285.40506298242155 [31,6,2,46,23] 81456.04997600002 -0.71286 -3297.2011999999986 -Manufacturer#5 almond aquamarine dodger light gainsboro 46 285.43749038756283 285.43749038756283 [6,2,46,23] 81474.56091875004 -0.98413 -4871.028125000002 +Manufacturer#5 almond antique blue firebrick mint 31 83.69879024746358 83.69879024746358 [31,6,2] 7005.4874888889035 0.39004 418.9233333333353 +Manufacturer#5 almond antique medium spring khaki 6 316.68049612345885 316.68049612345885 [31,6,2,46] 100286.53662500002 -0.71361 -4090.853749999999 +Manufacturer#5 almond antique sky peru orange 2 285.40506298242155 285.40506298242155 [31,6,2,46,23] 81456.04997600001 -0.71286 -3297.2011999999986 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 285.4374903875629 285.4374903875629 [6,2,46,23] 81474.56091875005 -0.98413 -4871.028125000002 Manufacturer#5 almond azure blanched chiffon midnight 23 315.9225931564038 315.9225931564038 [2,46,23] 99807.08486666664 -0.99789 -5664.856666666666 PREHOOK: query: select p_mfgr,p_name, p_size, histogram_numeric(p_retailprice, 5) over w1 as hist, diff --git a/ql/src/test/results/clientpositive/llap/stddev.q.out b/ql/src/test/results/clientpositive/llap/stddev.q.out new file mode 100644 index 0000000..69d462a --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/stddev.q.out @@ -0,0 +1,102 @@ +PREHOOK: query: create table test ( col1 decimal(10,3) ) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test +POSTHOOK: query: create table test ( col1 decimal(10,3) ) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test +PREHOOK: query: insert into test values (10230.72),(10230.72),(10230.72),(10230.72),(10230.72),(10230.72),(10230.72) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test +POSTHOOK: query: insert into test values (10230.72),(10230.72),(10230.72),(10230.72),(10230.72),(10230.72),(10230.72) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test +POSTHOOK: Lineage: test.col1 SCRIPT [] +PREHOOK: query: select STDDEV_SAMP(col1) AS STDDEV_6M , STDDEV(col1) as STDDEV ,STDDEV_POP(col1) as STDDEV_POP , variance(col1) as variance,var_pop(col1) as var_pop,var_samp(col1) as var_samp from test +PREHOOK: type: QUERY +PREHOOK: Input: default@test +#### A masked pattern was here #### +POSTHOOK: query: select STDDEV_SAMP(col1) AS STDDEV_6M , STDDEV(col1) as STDDEV ,STDDEV_POP(col1) as STDDEV_POP , variance(col1) as variance,var_pop(col1) as var_pop,var_samp(col1) as var_samp from test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test +#### A masked pattern was here #### +0.0 0.0 0.0 0.0 0.0 0.0 +PREHOOK: query: drop table test +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@test +PREHOOK: Output: default@test +POSTHOOK: query: drop table test +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@test +POSTHOOK: Output: default@test +PREHOOK: query: create table testpoint ( col1 decimal(10,3)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testpoint +POSTHOOK: query: create table testpoint ( col1 decimal(10,3)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testpoint +PREHOOK: query: insert into testpoint values (0.12345678),(0.25362123),(0.62437485),(0.65133746),(0.98765432),(0.12435647),(0.7654321445) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@testpoint +POSTHOOK: query: insert into testpoint values (0.12345678),(0.25362123),(0.62437485),(0.65133746),(0.98765432),(0.12435647),(0.7654321445) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@testpoint +POSTHOOK: Lineage: testpoint.col1 SCRIPT [] +PREHOOK: query: select STDDEV_SAMP(col1) AS STDDEV_6M , STDDEV(col1) as STDDEV ,STDDEV_POP(col1) as STDDEV_POP , variance(col1) as variance,var_pop(col1) as var_pop,var_samp(col1) as var_samp from testpoint +PREHOOK: type: QUERY +PREHOOK: Input: default@testpoint +#### A masked pattern was here #### +POSTHOOK: query: select STDDEV_SAMP(col1) AS STDDEV_6M , STDDEV(col1) as STDDEV ,STDDEV_POP(col1) as STDDEV_POP , variance(col1) as variance,var_pop(col1) as var_pop,var_samp(col1) as var_samp from testpoint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testpoint +#### A masked pattern was here #### +0.33922629053550113 0.31406251814905023 0.31406251814905023 0.0986352653061225 0.0986352653061225 0.11507447619047624 +PREHOOK: query: drop table testpoint +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testpoint +PREHOOK: Output: default@testpoint +POSTHOOK: query: drop table testpoint +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testpoint +POSTHOOK: Output: default@testpoint +PREHOOK: query: create table testint(col1 int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testint +POSTHOOK: query: create table testint(col1 int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testint +PREHOOK: query: insert into testint values (85),(86),(100),(76),(81),(93),(84),(99),(71),(69),(93),(85),(81),(87),(89) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@testint +POSTHOOK: query: insert into testint values (85),(86),(100),(76),(81),(93),(84),(99),(71),(69),(93),(85),(81),(87),(89) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@testint +POSTHOOK: Lineage: testint.col1 SCRIPT [] +PREHOOK: query: select STDDEV_SAMP(col1) AS STDDEV_6M , STDDEV(col1) as STDDEV ,STDDEV_POP(col1) as STDDEV_POP, variance(col1) as variance,var_pop(col1) as var_pop,var_samp(col1) as var_samp from testint +PREHOOK: type: QUERY +PREHOOK: Input: default@testint +#### A masked pattern was here #### +POSTHOOK: query: select STDDEV_SAMP(col1) AS STDDEV_6M , STDDEV(col1) as STDDEV ,STDDEV_POP(col1) as STDDEV_POP, variance(col1) as variance,var_pop(col1) as var_pop,var_samp(col1) as var_samp from testint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testint +#### A masked pattern was here #### +9.003702941938208 8.698403429493386 8.698403429493386 75.66222222222228 75.66222222222228 81.06666666666673 +PREHOOK: query: drop table testint +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testint +PREHOOK: Output: default@testint +POSTHOOK: query: drop table testint +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testint +POSTHOOK: Output: default@testint diff --git a/ql/src/test/results/clientpositive/llap/vector_windowing.q.out b/ql/src/test/results/clientpositive/llap/vector_windowing.q.out index f33c28f..3d6e9ce 100644 --- a/ql/src/test/results/clientpositive/llap/vector_windowing.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_windowing.q.out @@ -3440,9 +3440,9 @@ Manufacturer#1 almond aquamarine burnished black steel 28 5 4 0.8333333333333334 Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 1.0 1.0 3 6 19.0 16.237815945091466 2 42 6 Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 0.2 0.0 1 1 14.0 0.0 4 14 14 Manufacturer#2 almond antique violet turquoise frosted 40 2 2 0.4 0.25 1 2 27.0 13.0 4 40 14 -Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 0.6 0.5 2 3 18.666666666666668 15.86050300449376 4 2 14 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 0.6 0.5 2 3 18.666666666666668 15.860503004493758 4 2 14 Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 0.8 0.75 2 4 20.25 14.00669482783144 4 25 40 -Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1.0 1.0 3 5 19.8 12.560254774486067 4 18 2 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1.0 1.0 3 5 19.8 12.560254774486065 4 18 2 Manufacturer#3 almond antique chartreuse khaki white 17 1 1 0.2 0.0 1 1 17.0 0.0 2 17 17 Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 0.4 0.25 1 2 15.5 1.5 2 14 17 Manufacturer#3 almond antique metallic orange dim 19 3 3 0.6 0.5 2 3 16.666666666666668 2.0548046676563256 2 19 17 @@ -4671,31 +4671,31 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### p_mfgr p_name p_size sdev sdev_pop uniq_size var cor covarp -Manufacturer#1 almond antique burnished rose metallic 2 258.10677784349235 258.10677784349235 [2,34,6] 66619.10876874991 0.81133 2801.7074999999995 -Manufacturer#1 almond antique burnished rose metallic 2 273.70217881648074 273.70217881648074 [2,34] 74912.8826888888 1.0 4128.782222222221 -Manufacturer#1 almond antique chartreuse lavender yellow 34 230.90151585470358 230.90151585470358 [2,34,6,28] 53315.51002399992 0.69564 2210.7864 -Manufacturer#1 almond antique salmon chartreuse burlywood 6 202.73109328368946 202.73109328368946 [2,34,6,28,42] 41099.896184 0.63079 2009.9536000000007 +Manufacturer#1 almond antique burnished rose metallic 2 258.1067778434924 258.1067778434924 [2,34,6] 66619.10876874994 0.81133 2801.7074999999995 +Manufacturer#1 almond antique burnished rose metallic 2 273.7021788164808 273.7021788164808 [2,34] 74912.88268888883 1.0 4128.782222222221 +Manufacturer#1 almond antique chartreuse lavender yellow 34 230.9015158547036 230.9015158547036 [2,34,6,28] 53315.510023999945 0.69564 2210.7864 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 202.7310932836895 202.7310932836895 [2,34,6,28,42] 41099.89618400001 0.63079 2009.9536000000007 Manufacturer#1 almond aquamarine burnished black steel 28 121.6064517973862 121.6064517973862 [34,6,28,42] 14788.129118750014 0.20367 331.1337500000004 -Manufacturer#1 almond aquamarine pink moccasin thistle 42 96.5751586416853 96.5751586416853 [6,28,42] 9326.761266666683 -1.4E-4 -0.20666666666708502 -Manufacturer#2 almond antique violet chocolate turquoise 14 142.2363169751898 142.2363169751898 [14,40,2] 20231.169866666663 -0.4937 -1113.7466666666658 -Manufacturer#2 almond antique violet turquoise frosted 40 137.76306498840682 137.76306498840682 [14,40,2,25] 18978.662075 -0.52056 -1004.4812499999995 -Manufacturer#2 almond aquamarine midnight light salmon 2 130.03972279269132 130.03972279269132 [14,40,2,25,18] 16910.329504000005 -0.46909 -766.1791999999995 -Manufacturer#2 almond aquamarine rose maroon antique 25 135.55100986344584 135.55100986344584 [40,2,25,18] 18374.07627499999 -0.60914 -1128.1787499999987 -Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 156.44019460768044 156.44019460768044 [2,25,18] 24473.534488888927 -0.95717 -1441.4466666666676 -Manufacturer#3 almond antique chartreuse khaki white 17 196.7742266885805 196.7742266885805 [17,14,19] 38720.09628888887 0.55572 224.6944444444446 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 96.57515864168526 96.57515864168526 [6,28,42] 9326.761266666676 -1.4E-4 -0.20666666666708502 +Manufacturer#2 almond antique violet chocolate turquoise 14 142.23631697518968 142.23631697518968 [14,40,2] 20231.16986666663 -0.4937 -1113.7466666666658 +Manufacturer#2 almond antique violet turquoise frosted 40 137.7630649884067 137.7630649884067 [14,40,2,25] 18978.662074999975 -0.52056 -1004.4812499999995 +Manufacturer#2 almond aquamarine midnight light salmon 2 130.03972279269126 130.03972279269126 [14,40,2,25,18] 16910.32950399999 -0.46909 -766.1791999999995 +Manufacturer#2 almond aquamarine rose maroon antique 25 135.55100986344587 135.55100986344587 [40,2,25,18] 18374.076274999996 -0.60914 -1128.1787499999987 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 156.44019460768038 156.44019460768038 [2,25,18] 24473.53448888891 -0.95717 -1441.4466666666676 +Manufacturer#3 almond antique chartreuse khaki white 17 196.77422668858048 196.77422668858048 [17,14,19] 38720.09628888886 0.55572 224.6944444444446 Manufacturer#3 almond antique forest lavender goldenrod 14 275.14144189852607 275.14144189852607 [17,14,19,1] 75702.81305 -0.67208 -1296.9000000000003 Manufacturer#3 almond antique metallic orange dim 19 260.23473614412046 260.23473614412046 [17,14,19,1,45] 67722.117896 -0.57035 -2129.0664 -Manufacturer#3 almond antique misty red olive 1 275.9139962356932 275.9139962356932 [14,19,1,45] 76128.53331875012 -0.57748 -2547.7868749999993 +Manufacturer#3 almond antique misty red olive 1 275.9139962356931 275.9139962356931 [14,19,1,45] 76128.53331875007 -0.57748 -2547.7868749999993 Manufacturer#3 almond antique olive coral navajo 45 260.5815918713796 260.5815918713796 [19,1,45] 67902.76602222225 -0.87107 -4099.731111111111 -Manufacturer#4 almond antique gainsboro frosted violet 10 170.13011889596618 170.13011889596618 [10,39,27] 28944.25735555559 -0.6657 -1347.4777777777779 -Manufacturer#4 almond antique violet mint lemon 39 242.26834609323197 242.26834609323197 [10,39,27,7] 58693.95151875002 -0.80519 -2537.328125 +Manufacturer#4 almond antique gainsboro frosted violet 10 170.13011889596615 170.13011889596615 [10,39,27] 28944.257355555583 -0.6657 -1347.4777777777779 +Manufacturer#4 almond antique violet mint lemon 39 242.26834609323194 242.26834609323194 [10,39,27,7] 58693.951518750015 -0.80519 -2537.328125 Manufacturer#4 almond aquamarine floral ivory bisque 27 234.10001662537326 234.10001662537326 [10,39,27,7,12] 54802.817784000035 -0.60469 -1719.8079999999995 -Manufacturer#4 almond aquamarine yellow dodger mint 7 247.3342714197732 247.3342714197732 [39,27,7,12] 61174.24181875003 -0.55087 -1719.0368749999975 +Manufacturer#4 almond aquamarine yellow dodger mint 7 247.33427141977316 247.33427141977316 [39,27,7,12] 61174.24181875002 -0.55087 -1719.0368749999975 Manufacturer#4 almond azure aquamarine papaya violet 12 283.3344330566893 283.3344330566893 [27,7,12] 80278.40095555557 -0.77557 -1867.4888888888881 -Manufacturer#5 almond antique blue firebrick mint 31 83.69879024746363 83.69879024746363 [31,6,2] 7005.487488888913 0.39004 418.9233333333353 -Manufacturer#5 almond antique medium spring khaki 6 316.68049612345885 316.68049612345885 [31,6,2,46] 100286.53662500004 -0.71361 -4090.853749999999 -Manufacturer#5 almond antique sky peru orange 2 285.40506298242155 285.40506298242155 [31,6,2,46,23] 81456.04997600002 -0.71286 -3297.2011999999986 -Manufacturer#5 almond aquamarine dodger light gainsboro 46 285.43749038756283 285.43749038756283 [6,2,46,23] 81474.56091875004 -0.98413 -4871.028125000002 +Manufacturer#5 almond antique blue firebrick mint 31 83.69879024746358 83.69879024746358 [31,6,2] 7005.4874888889035 0.39004 418.9233333333353 +Manufacturer#5 almond antique medium spring khaki 6 316.68049612345885 316.68049612345885 [31,6,2,46] 100286.53662500002 -0.71361 -4090.853749999999 +Manufacturer#5 almond antique sky peru orange 2 285.40506298242155 285.40506298242155 [31,6,2,46,23] 81456.04997600001 -0.71286 -3297.2011999999986 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 285.4374903875629 285.4374903875629 [6,2,46,23] 81474.56091875005 -0.98413 -4871.028125000002 Manufacturer#5 almond azure blanched chiffon midnight 23 315.9225931564038 315.9225931564038 [2,46,23] 99807.08486666664 -0.99789 -5664.856666666666 PREHOOK: query: explain vectorization detail select p_mfgr,p_name, p_size, diff --git a/ql/src/test/results/clientpositive/llap/windowing.q.out b/ql/src/test/results/clientpositive/llap/windowing.q.out index 8c1f93a..c7415e7 100644 --- a/ql/src/test/results/clientpositive/llap/windowing.q.out +++ b/ql/src/test/results/clientpositive/llap/windowing.q.out @@ -633,9 +633,9 @@ Manufacturer#1 almond aquamarine burnished black steel 28 5 4 0.8333333333333334 Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 1.0 1.0 3 6 19.0 16.237815945091466 2 42 6 Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 0.2 0.0 1 1 14.0 0.0 4 14 14 Manufacturer#2 almond antique violet turquoise frosted 40 2 2 0.4 0.25 1 2 27.0 13.0 4 40 14 -Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 0.6 0.5 2 3 18.666666666666668 15.86050300449376 4 2 14 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 0.6 0.5 2 3 18.666666666666668 15.860503004493758 4 2 14 Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 0.8 0.75 2 4 20.25 14.00669482783144 4 25 40 -Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1.0 1.0 3 5 19.8 12.560254774486067 4 18 2 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1.0 1.0 3 5 19.8 12.560254774486065 4 18 2 Manufacturer#3 almond antique chartreuse khaki white 17 1 1 0.2 0.0 1 1 17.0 0.0 2 17 17 Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 0.4 0.25 1 2 15.5 1.5 2 14 17 Manufacturer#3 almond antique metallic orange dim 19 3 3 0.6 0.5 2 3 16.666666666666668 2.0548046676563256 2 19 17 @@ -862,31 +862,31 @@ window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 precedi POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Manufacturer#1 almond antique burnished rose metallic 2 258.10677784349235 258.10677784349235 [2,34,6] 66619.10876874991 0.81133 2801.7074999999995 -Manufacturer#1 almond antique burnished rose metallic 2 273.70217881648074 273.70217881648074 [2,34] 74912.8826888888 1.0 4128.782222222221 -Manufacturer#1 almond antique chartreuse lavender yellow 34 230.90151585470358 230.90151585470358 [2,34,6,28] 53315.51002399992 0.69564 2210.7864 -Manufacturer#1 almond antique salmon chartreuse burlywood 6 202.73109328368946 202.73109328368946 [2,34,6,28,42] 41099.896184 0.63079 2009.9536000000007 +Manufacturer#1 almond antique burnished rose metallic 2 258.1067778434924 258.1067778434924 [2,34,6] 66619.10876874994 0.81133 2801.7074999999995 +Manufacturer#1 almond antique burnished rose metallic 2 273.7021788164808 273.7021788164808 [2,34] 74912.88268888883 1.0 4128.782222222221 +Manufacturer#1 almond antique chartreuse lavender yellow 34 230.9015158547036 230.9015158547036 [2,34,6,28] 53315.510023999945 0.69564 2210.7864 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 202.7310932836895 202.7310932836895 [2,34,6,28,42] 41099.89618400001 0.63079 2009.9536000000007 Manufacturer#1 almond aquamarine burnished black steel 28 121.6064517973862 121.6064517973862 [34,6,28,42] 14788.129118750014 0.20367 331.1337500000004 -Manufacturer#1 almond aquamarine pink moccasin thistle 42 96.5751586416853 96.5751586416853 [6,28,42] 9326.761266666683 -1.4E-4 -0.20666666666708502 -Manufacturer#2 almond antique violet chocolate turquoise 14 142.2363169751898 142.2363169751898 [14,40,2] 20231.169866666663 -0.4937 -1113.7466666666658 -Manufacturer#2 almond antique violet turquoise frosted 40 137.76306498840682 137.76306498840682 [14,40,2,25] 18978.662075 -0.52056 -1004.4812499999995 -Manufacturer#2 almond aquamarine midnight light salmon 2 130.03972279269132 130.03972279269132 [14,40,2,25,18] 16910.329504000005 -0.46909 -766.1791999999995 -Manufacturer#2 almond aquamarine rose maroon antique 25 135.55100986344584 135.55100986344584 [40,2,25,18] 18374.07627499999 -0.60914 -1128.1787499999987 -Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 156.44019460768044 156.44019460768044 [2,25,18] 24473.534488888927 -0.95717 -1441.4466666666676 -Manufacturer#3 almond antique chartreuse khaki white 17 196.7742266885805 196.7742266885805 [17,14,19] 38720.09628888887 0.55572 224.6944444444446 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 96.57515864168526 96.57515864168526 [6,28,42] 9326.761266666676 -1.4E-4 -0.20666666666708502 +Manufacturer#2 almond antique violet chocolate turquoise 14 142.23631697518968 142.23631697518968 [14,40,2] 20231.16986666663 -0.4937 -1113.7466666666658 +Manufacturer#2 almond antique violet turquoise frosted 40 137.7630649884067 137.7630649884067 [14,40,2,25] 18978.662074999975 -0.52056 -1004.4812499999995 +Manufacturer#2 almond aquamarine midnight light salmon 2 130.03972279269126 130.03972279269126 [14,40,2,25,18] 16910.32950399999 -0.46909 -766.1791999999995 +Manufacturer#2 almond aquamarine rose maroon antique 25 135.55100986344587 135.55100986344587 [40,2,25,18] 18374.076274999996 -0.60914 -1128.1787499999987 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 156.44019460768038 156.44019460768038 [2,25,18] 24473.53448888891 -0.95717 -1441.4466666666676 +Manufacturer#3 almond antique chartreuse khaki white 17 196.77422668858048 196.77422668858048 [17,14,19] 38720.09628888886 0.55572 224.6944444444446 Manufacturer#3 almond antique forest lavender goldenrod 14 275.14144189852607 275.14144189852607 [17,14,19,1] 75702.81305 -0.67208 -1296.9000000000003 Manufacturer#3 almond antique metallic orange dim 19 260.23473614412046 260.23473614412046 [17,14,19,1,45] 67722.117896 -0.57035 -2129.0664 -Manufacturer#3 almond antique misty red olive 1 275.9139962356932 275.9139962356932 [14,19,1,45] 76128.53331875012 -0.57748 -2547.7868749999993 +Manufacturer#3 almond antique misty red olive 1 275.9139962356931 275.9139962356931 [14,19,1,45] 76128.53331875007 -0.57748 -2547.7868749999993 Manufacturer#3 almond antique olive coral navajo 45 260.5815918713796 260.5815918713796 [19,1,45] 67902.76602222225 -0.87107 -4099.731111111111 -Manufacturer#4 almond antique gainsboro frosted violet 10 170.13011889596618 170.13011889596618 [10,39,27] 28944.25735555559 -0.6657 -1347.4777777777779 -Manufacturer#4 almond antique violet mint lemon 39 242.26834609323197 242.26834609323197 [10,39,27,7] 58693.95151875002 -0.80519 -2537.328125 +Manufacturer#4 almond antique gainsboro frosted violet 10 170.13011889596615 170.13011889596615 [10,39,27] 28944.257355555583 -0.6657 -1347.4777777777779 +Manufacturer#4 almond antique violet mint lemon 39 242.26834609323194 242.26834609323194 [10,39,27,7] 58693.951518750015 -0.80519 -2537.328125 Manufacturer#4 almond aquamarine floral ivory bisque 27 234.10001662537326 234.10001662537326 [10,39,27,7,12] 54802.817784000035 -0.60469 -1719.8079999999995 -Manufacturer#4 almond aquamarine yellow dodger mint 7 247.3342714197732 247.3342714197732 [39,27,7,12] 61174.24181875003 -0.55087 -1719.0368749999975 +Manufacturer#4 almond aquamarine yellow dodger mint 7 247.33427141977316 247.33427141977316 [39,27,7,12] 61174.24181875002 -0.55087 -1719.0368749999975 Manufacturer#4 almond azure aquamarine papaya violet 12 283.3344330566893 283.3344330566893 [27,7,12] 80278.40095555557 -0.77557 -1867.4888888888881 -Manufacturer#5 almond antique blue firebrick mint 31 83.69879024746363 83.69879024746363 [31,6,2] 7005.487488888913 0.39004 418.9233333333353 -Manufacturer#5 almond antique medium spring khaki 6 316.68049612345885 316.68049612345885 [31,6,2,46] 100286.53662500004 -0.71361 -4090.853749999999 -Manufacturer#5 almond antique sky peru orange 2 285.40506298242155 285.40506298242155 [31,6,2,46,23] 81456.04997600002 -0.71286 -3297.2011999999986 -Manufacturer#5 almond aquamarine dodger light gainsboro 46 285.43749038756283 285.43749038756283 [6,2,46,23] 81474.56091875004 -0.98413 -4871.028125000002 +Manufacturer#5 almond antique blue firebrick mint 31 83.69879024746358 83.69879024746358 [31,6,2] 7005.4874888889035 0.39004 418.9233333333353 +Manufacturer#5 almond antique medium spring khaki 6 316.68049612345885 316.68049612345885 [31,6,2,46] 100286.53662500002 -0.71361 -4090.853749999999 +Manufacturer#5 almond antique sky peru orange 2 285.40506298242155 285.40506298242155 [31,6,2,46,23] 81456.04997600001 -0.71286 -3297.2011999999986 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 285.4374903875629 285.4374903875629 [6,2,46,23] 81474.56091875005 -0.98413 -4871.028125000002 Manufacturer#5 almond azure blanched chiffon midnight 23 315.9225931564038 315.9225931564038 [2,46,23] 99807.08486666664 -0.99789 -5664.856666666666 PREHOOK: query: select p_mfgr,p_name, p_size, histogram_numeric(p_retailprice, 5) over w1 as hist,