Author: jdere
Date: Thu Mar 26 17:55:27 2015
New Revision: 1669378
URL: http://svn.apache.org/r1669378
Log:
HIVE-10072: Add vectorization support for Hybrid Grace Hash Join (Wei Zheng,
reviewed by Matt McCline)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
hive/trunk/ql/src/test/queries/clientpositive/hybridhashjoin.q
hive/trunk/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1669378&r1=1669377&r2=1669378&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
Thu Mar 26 17:55:27 2015
@@ -404,7 +404,7 @@ public class MapJoinOperator extends Abs
* @param hybridHtContainer Hybrid hashtable container
* @param row big table row
*/
- private void spillBigTableRow(MapJoinTableContainer hybridHtContainer,
Object row) {
+ protected void spillBigTableRow(MapJoinTableContainer hybridHtContainer,
Object row) throws HiveException {
HybridHashTableContainer ht = (HybridHashTableContainer) hybridHtContainer;
int partitionId = ht.getToSpillPartitionId();
HashPartition hp = ht.getHashPartitions()[partitionId];
@@ -469,8 +469,7 @@ public class MapJoinOperator extends Abs
}
/**
- * Continue processing each pair of spilled hashtable and big table row
container,
- * by bringing them back to memory and calling process() again.
+ * Continue processing each pair of spilled hashtable and big table row
container
* @param partition hash partition to process
* @param hybridHtContainer Hybrid hashtable container
* @throws HiveException
@@ -481,13 +480,7 @@ public class MapJoinOperator extends Abs
private void continueProcess(HashPartition partition,
HybridHashTableContainer hybridHtContainer)
throws HiveException, IOException, ClassNotFoundException,
SerDeException {
reloadHashTable(partition, hybridHtContainer);
- // Iterate thru the on-disk matchfile, and feed processOp with leftover
rows
- ObjectContainer bigTable = partition.getMatchfileObjContainer();
- while (bigTable.hasNext()) {
- Object row = bigTable.next();
- process(row, tag);
- }
- bigTable.clear();
+ reProcessBigTable(partition);
}
/**
@@ -547,6 +540,20 @@ public class MapJoinOperator extends Abs
}
/**
+ * Iterate over the big table row container and feed process() with leftover
rows
+ * @param partition the hash partition being brought back to memory at the
moment
+ * @throws HiveException
+ */
+ protected void reProcessBigTable(HashPartition partition) throws
HiveException {
+ ObjectContainer bigTable = partition.getMatchfileObjContainer();
+ while (bigTable.hasNext()) {
+ Object row = bigTable.next();
+ process(row, tag);
+ }
+ bigTable.clear();
+ }
+
+ /**
* Implements the getName function for the Node Interface.
*
* @return the name of the operator
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java?rev=1669378&r1=1669377&r2=1669378&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ObjectContainer.java
Thu Mar 26 17:55:27 2015
@@ -49,7 +49,7 @@ public class ObjectContainer<ROW> {
private boolean readBufferUsed = false; // indicates if read buffer has data
private int rowsInReadBuffer = 0; // number of rows in the temporary
read buffer
private int readCursor = 0; // cursor during reading
- private int rowsOnDisk = 0; // total number of pairs in output
+ private int rowsOnDisk = 0; // total number of objects in output
private File parentFile;
private File tmpFile;
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java?rev=1669378&r1=1669377&r2=1669378&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
Thu Mar 26 17:55:27 2015
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.exec.vector;
+import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -31,8 +32,10 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.JoinUtil;
import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
+import org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
import
org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer.ReusableGetAdaptor;
+import org.apache.hadoop.hive.ql.exec.persistence.ObjectContainer;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import
org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
import
org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory;
@@ -42,6 +45,7 @@ import org.apache.hadoop.hive.ql.plan.Ma
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.io.DataOutputBuffer;
/**
* The vectorized version of the MapJoinOperator.
@@ -68,6 +72,7 @@ public class VectorMapJoinOperator exten
//---------------------------------------------------------------------------
private transient VectorizedRowBatch outputBatch;
+ private transient VectorizedRowBatch scratchBatch; // holds restored (from
disk) big table rows
private transient VectorExpressionWriter[] valueWriters;
private transient Map<ObjectInspector, VectorColumnAssign[]>
outputVectorAssigners;
@@ -81,6 +86,10 @@ public class VectorMapJoinOperator exten
private transient VectorizedRowBatchCtx vrbCtx = null;
+ private transient int tag; // big table alias
+ private VectorExpressionWriter[] rowWriters; // Writer for producing row
from input batch
+ protected transient Object[] singleRow;
+
public VectorMapJoinOperator() {
super();
}
@@ -117,6 +126,19 @@ public class VectorMapJoinOperator exten
@Override
public Collection<Future<?>> initializeOp(Configuration hconf) throws
HiveException {
+ // Code borrowed from VectorReduceSinkOperator.initializeOp
+ VectorExpressionWriterFactory.processVectorInspector(
+ (StructObjectInspector) inputObjInspectors[0],
+ new VectorExpressionWriterFactory.SingleOIDClosure() {
+ @Override
+ public void assign(VectorExpressionWriter[] writers,
+ ObjectInspector objectInspector) {
+ rowWriters = writers;
+ inputObjInspectors[0] = objectInspector;
+ }
+ });
+ singleRow = new Object[rowWriters.length];
+
Collection<Future<?>> result = super.initializeOp(hconf);
List<ExprNodeDesc> keyDesc = conf.getKeys().get(posBigTable);
@@ -213,6 +235,7 @@ public class VectorMapJoinOperator exten
@Override
public void closeOp(boolean aborted) throws HiveException {
+ super.closeOp(aborted);
for (MapJoinTableContainer tableContainer : mapJoinTables) {
if (tableContainer != null) {
tableContainer.dumpMetrics();
@@ -221,7 +244,6 @@ public class VectorMapJoinOperator exten
if (!aborted && 0 < outputBatch.size) {
flushOutput();
}
- super.closeOp(aborted);
}
@Override
@@ -235,6 +257,12 @@ public class VectorMapJoinOperator exten
byte alias = (byte) tag;
VectorizedRowBatch inBatch = (VectorizedRowBatch) row;
+ // Preparation for hybrid grace hash join
+ this.tag = tag;
+ if (scratchBatch == null) {
+ scratchBatch = makeLike(inBatch);
+ }
+
if (null != bigTableFilterExpressions) {
for(VectorExpression ve:bigTableFilterExpressions) {
ve.evaluate(inBatch);
@@ -270,4 +298,94 @@ public class VectorMapJoinOperator exten
public VectorizationContext getOuputVectorizationContext() {
return vOutContext;
}
+
+ @Override
+ protected void spillBigTableRow(MapJoinTableContainer hybridHtContainer,
Object row)
+ throws HiveException {
+ // Extract the actual row from row batch
+ VectorizedRowBatch inBatch = (VectorizedRowBatch) row;
+ Object[] actualRow = getRowObject(inBatch, batchIndex);
+ super.spillBigTableRow(hybridHtContainer, actualRow);
+ }
+
+ @Override
+ protected void reProcessBigTable(HybridHashTableContainer.HashPartition
partition)
+ throws HiveException {
+ ObjectContainer bigTable = partition.getMatchfileObjContainer();
+
+ DataOutputBuffer dataOutputBuffer = new DataOutputBuffer();
+ while (bigTable.hasNext()) {
+ Object row = bigTable.next();
+ VectorizedBatchUtil.addProjectedRowToBatchFrom(row,
+ (StructObjectInspector) inputObjInspectors[posBigTable],
+ scratchBatch.size, scratchBatch, dataOutputBuffer);
+ scratchBatch.size++;
+
+ if (scratchBatch.size == VectorizedRowBatch.DEFAULT_SIZE) {
+ process(scratchBatch, tag); // call process once we have a full batch
+ scratchBatch.reset();
+ dataOutputBuffer.reset();
+ }
+ }
+ // Process the row batch that has less than DEFAULT_SIZE rows
+ if (scratchBatch.size > 0) {
+ process(scratchBatch, tag);
+ scratchBatch.reset();
+ dataOutputBuffer.reset();
+ }
+ bigTable.clear();
+ }
+
+ // Code borrowed from VectorReduceSinkOperator
+ private Object[] getRowObject(VectorizedRowBatch vrb, int rowIndex) throws
HiveException {
+ int batchIndex = rowIndex;
+ if (vrb.selectedInUse) {
+ batchIndex = vrb.selected[rowIndex];
+ }
+ for (int i = 0; i < vrb.projectionSize; i++) {
+ ColumnVector vectorColumn = vrb.cols[vrb.projectedColumns[i]];
+ if (vectorColumn != null) {
+ singleRow[i] = rowWriters[i].writeValue(vectorColumn, batchIndex);
+ } else {
+ // Some columns from tables are not used.
+ singleRow[i] = null;
+ }
+ }
+ return singleRow;
+ }
+
+ /**
+ * Make a new (scratch) batch, which is exactly "like" the batch provided,
except that it's empty
+ * @param batch the batch to imitate
+ * @return the new batch
+ * @throws HiveException
+ */
+ VectorizedRowBatch makeLike(VectorizedRowBatch batch) throws HiveException {
+ VectorizedRowBatch newBatch = new VectorizedRowBatch(batch.numCols);
+ for (int i = 0; i < batch.numCols; i++) {
+ ColumnVector colVector = batch.cols[i];
+ if (colVector != null) {
+ ColumnVector newColVector;
+ if (colVector instanceof LongColumnVector) {
+ newColVector = new LongColumnVector();
+ } else if (colVector instanceof DoubleColumnVector) {
+ newColVector = new DoubleColumnVector();
+ } else if (colVector instanceof BytesColumnVector) {
+ newColVector = new BytesColumnVector();
+ } else if (colVector instanceof DecimalColumnVector) {
+ DecimalColumnVector decColVector = (DecimalColumnVector) colVector;
+ newColVector = new DecimalColumnVector(decColVector.precision,
decColVector.scale);
+ } else {
+ throw new HiveException("Column vector class " +
colVector.getClass().getName() +
+ " is not supported!");
+ }
+ newBatch.cols[i] = newColVector;
+ newBatch.cols[i].init();
+ }
+ }
+ newBatch.projectedColumns = Arrays.copyOf(batch.projectedColumns,
batch.projectedColumns.length);
+ newBatch.projectionSize = batch.projectionSize;
+ newBatch.reset();
+ return newBatch;
+ }
}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java?rev=1669378&r1=1669377&r2=1669378&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
Thu Mar 26 17:55:27 2015
@@ -242,11 +242,31 @@ public class VectorizedBatchUtil {
final int off = colOffset;
// Iterate thru the cols and load the batch
for (int i = 0; i < fieldRefs.size(); i++) {
- setVector(row, oi, fieldRefs, batch, buffer, rowIndex, i, off);
+ setVector(row, oi, fieldRefs.get(i), batch, buffer, rowIndex, i, off);
}
}
/**
+ * Add only the projected column of a regular row to the specified
vectorized row batch
+ * @param row the regular row
+ * @param oi object inspector for the row
+ * @param rowIndex the offset to add in the batch
+ * @param batch vectorized row batch
+ * @param buffer data output buffer
+ * @throws HiveException
+ */
+ public static void addProjectedRowToBatchFrom(Object row,
StructObjectInspector oi,
+ int rowIndex, VectorizedRowBatch batch, DataOutputBuffer buffer) throws
HiveException {
+ List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs();
+ for (int i = 0; i < fieldRefs.size(); i++) {
+ int projectedOutputCol = batch.projectedColumns[i];
+ if (batch.cols[projectedOutputCol] == null) {
+ continue;
+ }
+ setVector(row, oi, fieldRefs.get(i), batch, buffer, rowIndex,
projectedOutputCol, 0);
+ }
+ }
+ /**
* Iterates thru all the columns in a given row and populates the batch
* from a given offset
*
@@ -275,21 +295,21 @@ public class VectorizedBatchUtil {
// The value will have already been set before we're called, so don't
overwrite it
continue;
}
- setVector(row, oi, fieldRefs, batch, buffer, rowIndex, i, 0);
+ setVector(row, oi, fieldRefs.get(i), batch, buffer, rowIndex, i, 0);
}
}
private static void setVector(Object row,
StructObjectInspector oi,
- List<? extends StructField> fieldRefs,
+ StructField field,
VectorizedRowBatch batch,
DataOutputBuffer buffer,
int rowIndex,
int colIndex,
int offset) throws HiveException {
- Object fieldData = oi.getStructFieldData(row, fieldRefs.get(colIndex));
- ObjectInspector foi = fieldRefs.get(colIndex).getFieldObjectInspector();
+ Object fieldData = oi.getStructFieldData(row, field);
+ ObjectInspector foi = field.getFieldObjectInspector();
// Vectorization only supports PRIMITIVE data types. Assert the same
assert (foi.getCategory() == Category.PRIMITIVE);
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1669378&r1=1669377&r2=1669378&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
Thu Mar 26 17:55:27 2015
@@ -1301,9 +1301,6 @@ public class Vectorizer implements Physi
switch (op.getType()) {
case MAPJOIN:
- // Disable Hybrid Grace Hash Join when vectorization is in effect, for
now
- HiveConf.setBoolVar(physicalContext.getConf(),
- HiveConf.ConfVars.HIVEUSEHYBRIDGRACEHASHJOIN, false);
case GROUPBY:
case FILTER:
case SELECT:
Modified: hive/trunk/ql/src/test/queries/clientpositive/hybridhashjoin.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/hybridhashjoin.q?rev=1669378&r1=1669377&r2=1669378&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/hybridhashjoin.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/hybridhashjoin.q Thu Mar 26
17:55:27 2015
@@ -210,3 +210,41 @@ select count(*) from
;
drop table parttbl;
+
+
+-- Test vectorization
+-- Test case borrowed from vector_decimal_mapjoin.q
+CREATE TABLE decimal_mapjoin STORED AS ORC AS
+ SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1,
+ CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2,
+ cint
+ FROM alltypesorc;
+
+SET hive.auto.convert.join=true;
+SET hive.auto.convert.join.noconditionaltask=true;
+SET hive.auto.convert.join.noconditionaltask.size=50000000;
+set hive.mapjoin.optimized.hashtable.wbsize=10000;
+SET hive.vectorized.execution.enabled=true;
+set hive.mapjoin.hybridgrace.hashtable=false;
+
+EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981;
+SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981;
+
+set hive.mapjoin.hybridgrace.hashtable=true;
+
+EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981;
+SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981;
+
+DROP TABLE decimal_mapjoin;
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out?rev=1669378&r1=1669377&r2=1669378&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out
(original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/hybridhashjoin.q.out Thu
Mar 26 17:55:27 2015
@@ -1154,3 +1154,407 @@ POSTHOOK: query: drop table parttbl
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@parttbl
POSTHOOK: Output: default@parttbl
+PREHOOK: query: -- Test vectorization
+-- Test case borrowed from vector_decimal_mapjoin.q
+CREATE TABLE decimal_mapjoin STORED AS ORC AS
+ SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1,
+ CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2,
+ cint
+ FROM alltypesorc
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: database:default
+PREHOOK: Output: default@decimal_mapjoin
+POSTHOOK: query: -- Test vectorization
+-- Test case borrowed from vector_decimal_mapjoin.q
+CREATE TABLE decimal_mapjoin STORED AS ORC AS
+ SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1,
+ CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2,
+ cint
+ FROM alltypesorc
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@decimal_mapjoin
+PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: l
+ Statistics: Num rows: 12288 Data size: 2165060 Basic stats:
COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (cint = 6981) (type: boolean)
+ Statistics: Num rows: 6144 Data size: 1082530 Basic stats:
COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 6981 (type: int)
+ 1 6981 (type: int)
+ outputColumnNames: _col1, _col9
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 6758 Data size: 1190783 Basic
stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: 6981 (type: int), 6981 (type: int), _col1
(type: decimal(20,10)), _col9 (type: decimal(23,14))
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 6758 Data size: 1190783 Basic
stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6758 Data size: 1190783 Basic
stats: COMPLETE Column stats: NONE
+ table:
+ input format:
org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: r
+ Statistics: Num rows: 12288 Data size: 2165060 Basic stats:
COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (cint = 6981) (type: boolean)
+ Statistics: Num rows: 6144 Data size: 1082530 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 6981 (type: int)
+ sort order: +
+ Map-reduce partition columns: 6981 (type: int)
+ Statistics: Num rows: 6144 Data size: 1082530 Basic
stats: COMPLETE Column stats: NONE
+ value expressions: cdecimal2 (type: decimal(23,14))
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_mapjoin
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_mapjoin
+#### A masked pattern was here ####
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 -617.5607769230769
+6981 6981 5831542.269248378 -617.5607769230769
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 6984454.211097692
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 -617.5607769230769
+6981 6981 -515.621072973 -617.5607769230769
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 6984454.211097692
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 -617.5607769230769
+6981 6981 -515.621072973 -617.5607769230769
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 6984454.211097692
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: l
+ Statistics: Num rows: 12288 Data size: 2165060 Basic stats:
COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (cint = 6981) (type: boolean)
+ Statistics: Num rows: 6144 Data size: 1082530 Basic stats:
COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 6981 (type: int)
+ 1 6981 (type: int)
+ outputColumnNames: _col1, _col9
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 6758 Data size: 1190783 Basic
stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: 6981 (type: int), 6981 (type: int), _col1
(type: decimal(20,10)), _col9 (type: decimal(23,14))
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 6758 Data size: 1190783 Basic
stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6758 Data size: 1190783 Basic
stats: COMPLETE Column stats: NONE
+ table:
+ input format:
org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: r
+ Statistics: Num rows: 12288 Data size: 2165060 Basic stats:
COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (cint = 6981) (type: boolean)
+ Statistics: Num rows: 6144 Data size: 1082530 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 6981 (type: int)
+ sort order: +
+ Map-reduce partition columns: 6981 (type: int)
+ Statistics: Num rows: 6144 Data size: 1082530 Basic
stats: COMPLETE Column stats: NONE
+ value expressions: cdecimal2 (type: decimal(23,14))
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_mapjoin
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2
+ FROM decimal_mapjoin l
+ JOIN decimal_mapjoin r ON l.cint = r.cint
+ WHERE l.cint = 6981
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_mapjoin
+#### A masked pattern was here ####
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 -617.5607769230769
+6981 6981 5831542.269248378 -617.5607769230769
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 6984454.211097692
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 5831542.269248378 NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL -617.5607769230769
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL 6984454.211097692
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 NULL NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 -617.5607769230769
+6981 6981 -515.621072973 -617.5607769230769
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 6984454.211097692
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 -617.5607769230769
+6981 6981 -515.621072973 -617.5607769230769
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 6984454.211097692
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+6981 6981 -515.621072973 NULL
+PREHOOK: query: DROP TABLE decimal_mapjoin
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@decimal_mapjoin
+PREHOOK: Output: default@decimal_mapjoin
+POSTHOOK: query: DROP TABLE decimal_mapjoin
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@decimal_mapjoin
+POSTHOOK: Output: default@decimal_mapjoin
