Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out Fri Jan 9 15:30:41 2015 @@ -199,8 +199,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) - Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 3), Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (GROUP, 2) + Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out Fri Jan 9 15:30:41 2015 @@ -153,7 +153,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP, 1) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -274,7 +274,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP, 1) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/temp_table_join1.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/temp_table_join1.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/temp_table_join1.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/temp_table_join1.q.out Fri Jan 9 15:30:41 2015 @@ -40,7 +40,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -136,7 +136,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -234,7 +234,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union25.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union25.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/union25.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union25.q.out Fri Jan 9 15:30:41 2015 @@ -66,8 +66,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (GROUP, 3), Map 5 (GROUP, 3) - Reducer 2 <- Map 1 (GROUP, 3), Reducer 4 (GROUP, 3) + Reducer 4 <- Map 3 (GROUP, 2), Map 5 (GROUP, 2) + Reducer 2 <- Map 1 (GROUP, 2), Reducer 4 (GROUP, 2) #### A masked pattern was here #### Vertices: Map 1 Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union33.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union33.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/union33.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union33.q.out Fri Jan 9 15:30:41 2015 @@ -187,8 +187,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 2 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (GROUP, 2) #### A masked pattern was here #### Vertices: Map 1 Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_19.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_19.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_19.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_19.q.out Fri Jan 9 15:30:41 2015 @@ -261,8 +261,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) - Reducer 4 <- Map 3 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP, 2) + Reducer 4 <- Map 3 (GROUP, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -430,8 +430,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) - Reducer 4 <- Map 3 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP, 2) + Reducer 4 <- Map 3 (GROUP, 2) #### A masked pattern was here #### Vertices: Map 1 Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out Fri Jan 9 15:30:41 2015 @@ -119,7 +119,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP, 2) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out Fri Jan 9 15:30:41 2015 @@ -44,7 +44,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -149,7 +149,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP, 2) #### A masked pattern was here #### Vertices: Map 1 Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out Fri Jan 9 15:30:41 2015 @@ -119,7 +119,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP, 2) #### A masked pattern was here #### Vertices: Map 1 Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out Fri Jan 9 15:30:41 2015 @@ -119,7 +119,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP, 2) #### A masked pattern was here #### Vertices: Map 1 Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out Fri Jan 9 15:30:41 2015 @@ -231,7 +231,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP, 2) #### A masked pattern was here #### Vertices: Map 1 Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out Fri Jan 9 15:30:41 2015 @@ -115,7 +115,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP, 2) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_string_concat.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_string_concat.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_string_concat.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_string_concat.q.out Fri Jan 9 15:30:41 2015 @@ -272,12 +272,14 @@ PREHOOK: query: EXPLAIN SELECT CONCAT(CONCAT(CONCAT('Quarter ',CAST(CAST((MONTH(dt) - 1) / 3 + 1 AS INT) AS STRING)),'-'),CAST(YEAR(dt) AS STRING)) AS `field` FROM vectortab2korc GROUP BY CONCAT(CONCAT(CONCAT('Quarter ',CAST(CAST((MONTH(dt) - 1) / 3 + 1 AS INT) AS STRING)),'-'),CAST(YEAR(dt) AS STRING)) + ORDER BY `field` LIMIT 50 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN SELECT CONCAT(CONCAT(CONCAT('Quarter ',CAST(CAST((MONTH(dt) - 1) / 3 + 1 AS INT) AS STRING)),'-'),CAST(YEAR(dt) AS STRING)) AS `field` FROM vectortab2korc GROUP BY CONCAT(CONCAT(CONCAT('Quarter ',CAST(CAST((MONTH(dt) - 1) / 3 + 1 AS INT) AS STRING)),'-'),CAST(YEAR(dt) AS STRING)) + ORDER BY `field` LIMIT 50 POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -288,7 +290,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP, 2) + Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -318,20 +321,27 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 459356 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + Statistics: Num rows: 1000 Data size: 459356 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 50 + Execution mode: vectorized + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 459356 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 50 + Statistics: Num rows: 50 Data size: 22950 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 50 Data size: 22950 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 50 Data size: 22950 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized Stage: Stage-0 @@ -343,6 +353,7 @@ STAGE PLANS: PREHOOK: query: SELECT CONCAT(CONCAT(CONCAT('Quarter ',CAST(CAST((MONTH(dt) - 1) / 3 + 1 AS INT) AS STRING)),'-'),CAST(YEAR(dt) AS STRING)) AS `field` FROM vectortab2korc GROUP BY CONCAT(CONCAT(CONCAT('Quarter ',CAST(CAST((MONTH(dt) - 1) / 3 + 1 AS INT) AS STRING)),'-'),CAST(YEAR(dt) AS STRING)) + ORDER BY `field` LIMIT 50 PREHOOK: type: QUERY PREHOOK: Input: default@vectortab2korc @@ -350,57 +361,58 @@ PREHOOK: Input: default@vectortab2korc POSTHOOK: query: SELECT CONCAT(CONCAT(CONCAT('Quarter ',CAST(CAST((MONTH(dt) - 1) / 3 + 1 AS INT) AS STRING)),'-'),CAST(YEAR(dt) AS STRING)) AS `field` FROM vectortab2korc GROUP BY CONCAT(CONCAT(CONCAT('Quarter ',CAST(CAST((MONTH(dt) - 1) / 3 + 1 AS INT) AS STRING)),'-'),CAST(YEAR(dt) AS STRING)) + ORDER BY `field` LIMIT 50 POSTHOOK: type: QUERY POSTHOOK: Input: default@vectortab2korc #### A masked pattern was here #### -Quarter 3-2051 -Quarter 3-2045 -Quarter 4-2095 -Quarter 1-2020 -Quarter 3-2042 -Quarter 2-1971 -Quarter 1-2044 -Quarter 4-1987 -Quarter 1-2047 -Quarter 1-2101 -Quarter 4-2080 -Quarter 3-2087 -Quarter 4-2047 -Quarter 2-2061 -Quarter 2-2103 +NULL +Quarter 1-1970 +Quarter 1-1971 +Quarter 1-1972 +Quarter 1-1973 +Quarter 1-1974 +Quarter 1-1975 +Quarter 1-1976 +Quarter 1-1977 Quarter 1-1978 -Quarter 4-1984 -Quarter 3-1982 -Quarter 2-2001 -Quarter 3-2024 -Quarter 2-2076 -Quarter 1-2074 -Quarter 3-2105 -Quarter 4-1999 -Quarter 3-2072 -Quarter 3-2000 -Quarter 2-2040 -Quarter 4-1972 -Quarter 4-2041 -Quarter 1-2026 -Quarter 3-2015 -Quarter 1-2041 -Quarter 3-1979 -Quarter 3-1970 -Quarter 3-2066 -Quarter 1-2077 -Quarter 4-2059 -Quarter 1-2086 -Quarter 3-2030 -Quarter 2-2067 -Quarter 4-2065 -Quarter 3-2021 -Quarter 3-2012 -Quarter 2-2037 +Quarter 1-1979 +Quarter 1-1980 +Quarter 1-1981 +Quarter 1-1982 +Quarter 1-1983 +Quarter 1-1984 +Quarter 1-1985 +Quarter 1-1986 Quarter 1-1987 -Quarter 4-2014 -Quarter 4-2038 -Quarter 4-1975 -Quarter 1-2053 -Quarter 4-2068 +Quarter 1-1988 +Quarter 1-1989 +Quarter 1-1990 +Quarter 1-1991 +Quarter 1-1992 +Quarter 1-1993 +Quarter 1-1994 +Quarter 1-1995 +Quarter 1-1996 +Quarter 1-1997 +Quarter 1-1998 +Quarter 1-1999 +Quarter 1-2000 +Quarter 1-2001 +Quarter 1-2002 +Quarter 1-2003 +Quarter 1-2004 +Quarter 1-2005 +Quarter 1-2006 +Quarter 1-2007 +Quarter 1-2008 +Quarter 1-2009 +Quarter 1-2010 +Quarter 1-2011 +Quarter 1-2012 +Quarter 1-2013 +Quarter 1-2014 +Quarter 1-2015 +Quarter 1-2016 +Quarter 1-2017 +Quarter 1-2018 Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_13.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_13.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_13.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_13.q.out Fri Jan 9 15:30:41 2015 @@ -326,7 +326,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP, 2) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out Fri Jan 9 15:30:41 2015 @@ -1866,7 +1866,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP, 2) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -2074,7 +2074,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP, 2) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -2329,7 +2329,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP, 2) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -2662,7 +2662,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) + Reducer 2 <- Map 1 (GROUP, 2) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out?rev=1650565&r1=1650564&r2=1650565&view=diff ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out (original) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out Fri Jan 9 15:30:41 2015 @@ -228,8 +228,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -513,9 +513,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -831,7 +831,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -1087,8 +1087,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -1375,8 +1375,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -1672,9 +1672,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 2 (GROUP, 3) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (GROUP, 2) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -1937,8 +1937,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 3), Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -2250,8 +2250,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 3) - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Reducer 4 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 2) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 4 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -2572,8 +2572,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -2849,8 +2849,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -3130,8 +3130,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -3418,9 +3418,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -3745,8 +3745,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -4130,9 +4130,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 3), Reducer 2 (PARTITION-LEVEL SORT, 3) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -4457,8 +4457,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 2 (GROUP, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (GROUP, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -4728,9 +4728,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 3) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (GROUP, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -5197,11 +5197,11 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 7 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 6 (PARTITION-LEVEL SORT, 3) - Reducer 5 <- Reducer 7 (PARTITION-LEVEL SORT, 3) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) + Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 7 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 6 (PARTITION-LEVEL SORT, 2) + Reducer 5 <- Reducer 7 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -5711,9 +5711,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -6067,10 +6067,10 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) - Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2) + Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -6421,9 +6421,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -6770,10 +6770,10 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) - Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2) + Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -7169,9 +7169,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -7518,9 +7518,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java?rev=1650565&r1=1650564&r2=1650565&view=diff ============================================================================== --- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java (original) +++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java Fri Jan 9 15:30:41 2015 @@ -72,4 +72,9 @@ public interface SparkClient extends Ser * Get the count of executors. */ Future<Integer> getExecutorCount(); + + /** + * Get default parallelism. For standalone mode, this can be used to get total number of cores. + */ + Future<Integer> getDefaultParallelism(); } Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java?rev=1650565&r1=1650564&r2=1650565&view=diff ============================================================================== --- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java (original) +++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java Fri Jan 9 15:30:41 2015 @@ -155,6 +155,11 @@ class SparkClientImpl implements SparkCl return submit(new GetExecutorCountJob()); } + @Override + public Future<Integer> getDefaultParallelism() { + return submit(new GetDefaultParallelismJob()); + } + void cancel(String jobId) { protocol.cancel(jobId); } @@ -489,10 +494,20 @@ class SparkClientImpl implements SparkCl @Override public Integer call(JobContext jc) throws Exception { - int count = jc.sc().sc().getExecutorMemoryStatus().size(); + // minus 1 here otherwise driver is also counted as an executor + int count = jc.sc().sc().getExecutorMemoryStatus().size() - 1; return Integer.valueOf(count); } } + private static class GetDefaultParallelismJob implements Job<Integer> { + private static final long serialVersionUID = 1L; + + @Override + public Integer call(JobContext jc) throws Exception { + return jc.sc().sc().defaultParallelism(); + } + } + }
