Author: gunther
Date: Fri Oct 18 21:54:58 2013
New Revision: 1533645
URL: http://svn.apache.org/r1533645
Log:
Merge latest trunk into branch. (Gunther Hagleitner)
Added:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssign.java
- copied unchanged from r1533637,
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssign.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java
- copied unchanged from r1533637,
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
- copied unchanged from r1533637,
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
hive/branches/tez/ql/src/test/queries/clientpositive/vectorized_mapjoin.q
- copied unchanged from r1533637,
hive/trunk/ql/src/test/queries/clientpositive/vectorized_mapjoin.q
hive/branches/tez/ql/src/test/results/clientpositive/vectorized_mapjoin.q.out
- copied unchanged from r1533637,
hive/trunk/ql/src/test/results/clientpositive/vectorized_mapjoin.q.out
Modified:
hive/branches/tez/ (props changed)
hive/branches/tez/build.properties
hive/branches/tez/common/build.xml
hive/branches/tez/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
hive/branches/tez/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
Propchange: hive/branches/tez/
------------------------------------------------------------------------------
Merged /hive/trunk:r1533304-1533637
Modified: hive/branches/tez/build.properties
URL:
http://svn.apache.org/viewvc/hive/branches/tez/build.properties?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
--- hive/branches/tez/build.properties (original)
+++ hive/branches/tez/build.properties Fri Oct 18 21:54:58 2013
@@ -77,7 +77,7 @@ common.jar=${hadoop.root}/lib/commons-ht
# full profile
iterate.hive.full.all=ant,shims,common,serde,metastore,ql,contrib,service,cli,jdbc,beeline,hwi,hbase-handler,testutils,hcatalog
iterate.hive.full.modules=shims,common,serde,metastore,ql,contrib,service,cli,jdbc,beeline,hwi,hbase-handler,testutils,hcatalog
-iterate.hive.full.tests=ql,contrib,hbase-handler,hwi,jdbc,beeline,metastore,odbc,serde,service,hcatalog
+iterate.hive.full.tests=common,ql,contrib,hbase-handler,hwi,jdbc,beeline,metastore,odbc,serde,service,hcatalog
iterate.hive.full.thrift=ql,service,metastore,serde
iterate.hive.full.protobuf=ql
iterate.hive.full.cpp=odbc
@@ -86,7 +86,7 @@ iterate.hive.full.cpp=odbc
iterate.hive.nohcat.all=ant,shims,common,serde,metastore,ql,contrib,service,cli,jdbc,beeline,hwi,hbase-handler,testutils
iterate.hive.nohcat.modules=shims,common,serde,metastore,ql,contrib,service,cli,jdbc,beeline,hwi,hbase-handler,testutils
iterate.hive.nohcat.tests=ql,contrib,hbase-handler,hwi,jdbc,beeline,metastore,odbc,serde,service
-iterate.hive.nohcat.thrift=ql,service,metastore,serde
+iterate.hive.nohcat.thrift=common,ql,service,metastore,serde
iterate.hive.nohcat.protobuf=ql
iterate.hive.nohcat.cpp=odbc
Modified: hive/branches/tez/common/build.xml
URL:
http://svn.apache.org/viewvc/hive/branches/tez/common/build.xml?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
--- hive/branches/tez/common/build.xml (original)
+++ hive/branches/tez/common/build.xml Fri Oct 18 21:54:58 2013
@@ -52,37 +52,4 @@ to call at top-level: ant deploy-contrib
</copy>
</target>
- <!-- target to run the tests -->
- <target name="test"
- depends="test-conditions,gen-test,compile-test,test-jar,test-init">
- <antcall target="testonly" />
- </target>
-
-
-
- <!-- target to run the tests -->
- <target name="testonly"
- depends="test-conditions,test-init">
- <echo message="Project: ${ant.project.name}"/>
- <junit showoutput="${test.output}" printsummary="yes" haltonfailure="no"
- fork="yes" maxmemory="512m" dir="${basedir}"
timeout="${test.junit.timeout}"
- errorProperty="tests.failed" failureProperty="tests.failed"
filtertrace="off">
- <sysproperty key="test.build.resources"
value="${test.build.resources}"/>
- <classpath refid="${test.classpath.id}"/>
- <formatter type="${test.junit.output.format}"
usefile="${test.junit.output.usefile}" />
- <batchtest todir="${test.build.dir}" unless="testcase">
- <fileset dir="${test.build.classes}"
- includes="**/${test.include}.class"
- excludes="**/*$*.class,${test.junit.exclude}" />
- </batchtest>
- <batchtest todir="${test.build.dir}" if="testcase">
- <fileset dir="${test.build.classes}" includes="**/${testcase}.class"/>
- </batchtest>
- <assertions>
- <enable />
- </assertions>
- </junit>
- <fail if="tests.failed">Tests failed!</fail>
- </target>
-
</project>
Modified:
hive/branches/tez/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
---
hive/branches/tez/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
(original)
+++
hive/branches/tez/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
Fri Oct 18 21:54:58 2013
@@ -17,11 +17,11 @@
*/
package org.apache.hadoop.hive.conf;
-import junit.framework.TestCase;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.junit.Assert;
+import org.junit.Test;
/**
@@ -30,31 +30,35 @@ import org.apache.hadoop.hive.conf.HiveC
* Test cases for HiveConf. Loads configuration files located
* in common/src/test/resources.
*/
-public class TestHiveConf extends TestCase {
-
+public class TestHiveConf {
+ @Test
public void testHiveSitePath() throws Exception {
String expectedPath =
new Path(System.getProperty("test.build.resources") +
"/hive-site.xml").toUri().getPath();
- assertEquals(expectedPath, new HiveConf().getHiveSiteLocation().getPath());
+ Assert.assertEquals(expectedPath, new
HiveConf().getHiveSiteLocation().getPath());
}
private void checkHadoopConf(String name, String expectedHadoopVal) throws
Exception {
- assertEquals(expectedHadoopVal, new Configuration().get(name));
+ Assert.assertEquals(expectedHadoopVal, new Configuration().get(name));
}
private void checkConfVar(ConfVars var, String expectedConfVarVal) throws
Exception {
- assertEquals(expectedConfVarVal, var.defaultVal);
+ Assert.assertEquals(expectedConfVarVal, var.defaultVal);
}
private void checkHiveConf(String name, String expectedHiveVal) throws
Exception {
- assertEquals(expectedHiveVal, new HiveConf().get(name));
+ Assert.assertEquals(expectedHiveVal, new HiveConf().get(name));
}
+ @Test
public void testConfProperties() throws Exception {
// Make sure null-valued ConfVar properties do not override the Hadoop
Configuration
- checkHadoopConf(ConfVars.HADOOPFS.varname, "core-site.xml");
- checkConfVar(ConfVars.HADOOPFS, null);
- checkHiveConf(ConfVars.HADOOPFS.varname, "core-site.xml");
+ // NOTE: Comment out the following test case for now until a better way to
test is found,
+ // as this test case cannot be reliably tested. The reason for this is
that Hive does
+ // overwrite fs.default.name in HiveConf if the property is set in system
properties.
+ // checkHadoopConf(ConfVars.HADOOPFS.varname, "core-site.xml");
+ // checkConfVar(ConfVars.HADOOPFS, null);
+ // checkHiveConf(ConfVars.HADOOPFS.varname, "core-site.xml");
// Make sure non-null-valued ConfVar properties *do* override the Hadoop
Configuration
checkHadoopConf(ConfVars.HADOOPNUMREDUCERS.varname, "1");
@@ -79,9 +83,10 @@ public class TestHiveConf extends TestCa
checkHiveConf("test.var.hiveconf.property",
ConfVars.DEFAULTPARTITIONNAME.defaultVal);
}
+ @Test
public void testColumnNameMapping() throws Exception {
for (int i = 0 ; i < 20 ; i++ ){
- assertTrue(i ==
HiveConf.getPositionFromInternalName(HiveConf.getColumnInternalName(i)));
+ Assert.assertTrue(i ==
HiveConf.getPositionFromInternalName(HiveConf.getColumnInternalName(i)));
}
}
}
Modified:
hive/branches/tez/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
---
hive/branches/tez/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java
(original)
+++
hive/branches/tez/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java
Fri Oct 18 21:54:58 2013
@@ -17,17 +17,15 @@
*/
package org.apache.hadoop.hive.conf;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
import junit.framework.TestCase;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.common.LogUtils;
import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
/**
* TestHiveLogging
@@ -36,7 +34,7 @@ import java.io.InputStreamReader;
* Loads configuration files located in common/src/test/resources.
*/
public class TestHiveLogging extends TestCase {
- private Runtime runTime;
+ private final Runtime runTime;
private Process process;
public TestHiveLogging() {
@@ -86,8 +84,9 @@ public class TestHiveLogging extends Tes
String line = "";
try {
while((line = buf.readLine()) != null) {
- if (line.equals(logFile))
+ if (line.equals(logFile)) {
logCreated = true;
+ }
}
} catch (IOException e) {
e.printStackTrace();
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
Fri Oct 18 21:54:58 2013
@@ -185,7 +185,7 @@ public abstract class CommonJoinOperator
return joinOutputObjectInspector;
}
- Configuration hconf;
+ protected Configuration hconf;
@Override
@SuppressWarnings("unchecked")
@@ -409,9 +409,9 @@ public abstract class CommonJoinOperator
//
// for MapJoin, filter tag is pre-calculated in MapredLocalTask and stored
with value.
// when reading the hashtable, MapJoinObjectValue calculates alias filter
and provide it to join
- protected ArrayList<Object> getFilteredValue(byte alias, Object row) throws
HiveException {
+ protected List<Object> getFilteredValue(byte alias, Object row) throws
HiveException {
boolean hasFilter = hasFilter(alias);
- ArrayList<Object> nr = JoinUtil.computeValues(row, joinValues[alias],
+ List<Object> nr = JoinUtil.computeValues(row, joinValues[alias],
joinValuesObjectInspectors[alias], hasFilter);
if (hasFilter) {
short filterTag = JoinUtil.isFiltered(row, joinFilters[alias],
@@ -436,7 +436,7 @@ public abstract class CommonJoinOperator
}
}
if (forward) {
- forward(forwardCache, null);
+ internalForward(forwardCache, null);
countAfterReport = 0;
}
}
@@ -641,6 +641,10 @@ public abstract class CommonJoinOperator
checkAndGenObject();
}
+ protected void internalForward(Object row, ObjectInspector outputOI) throws
HiveException {
+ forward(row, outputOI);
+ }
+
private void genUniqueJoinObject(int aliasNum, int forwardCachePos)
throws HiveException {
AbstractRowContainer<List<Object>> alias = storage[order[aliasNum]];
@@ -651,7 +655,7 @@ public abstract class CommonJoinOperator
forwardCache[p++] = row.get(j);
}
if (aliasNum == numAliases - 1) {
- forward(forwardCache, outputObjInspector);
+ internalForward(forwardCache, outputObjInspector);
countAfterReport = 0;
} else {
genUniqueJoinObject(aliasNum + 1, p);
@@ -670,7 +674,7 @@ public abstract class CommonJoinOperator
}
}
- forward(forwardCache, outputObjInspector);
+ internalForward(forwardCache, outputObjInspector);
countAfterReport = 0;
}
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
Fri Oct 18 21:54:58 2013
@@ -75,7 +75,7 @@ public class JoinOperator extends Common
// get alias
alias = (byte) tag;
- ArrayList<Object> nr = getFilteredValue(alias, row);
+ List<Object> nr = getFilteredValue(alias, row);
if (handleSkewJoin) {
skewJoinKeyContext.handleSkew(tag);
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
(original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
Fri Oct 18 21:54:58 2013
@@ -199,14 +199,15 @@ public class JoinUtil {
* Return the value as a standard object. StandardObject can be inspected by
a
* standard ObjectInspector.
* If it would be tagged by filter, reserve one more slot for that.
+ * outValues can be passed in to avoid allocation
*/
- public static ArrayList<Object> computeValues(Object row,
+ public static List<Object> computeValues(Object row,
List<ExprNodeEvaluator> valueFields, List<ObjectInspector>
valueFieldsOI, boolean hasFilter)
throws HiveException {
// Compute the values
int reserve = hasFilter ? valueFields.size() + 1 : valueFields.size();
- ArrayList<Object> nr = new ArrayList<Object>(reserve);
+ List<Object> nr = new ArrayList<Object>(reserve);
for (int i = 0; i < valueFields.size(); i++) {
nr.add(ObjectInspectorUtils.copyToStandardObject(valueFields.get(i)
.evaluate(row), valueFieldsOI.get(i),
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
Fri Oct 18 21:54:58 2013
@@ -19,7 +19,7 @@
package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
-import java.util.ArrayList;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -59,7 +59,7 @@ public class MapJoinOperator extends Abs
"Mapside join exceeds available memory. "
+ "Please try removing the mapjoin hint."};
- private transient MapJoinTableContainer[] mapJoinTables;
+ protected transient MapJoinTableContainer[] mapJoinTables;
private transient MapJoinTableContainerSerDe[] mapJoinTableSerdes;
private transient boolean hashTblInitedOnce;
private transient MapJoinKey key;
@@ -174,6 +174,11 @@ public class MapJoinOperator extends Abs
}
}
+ protected MapJoinKey computeMapJoinKey(Object row, byte alias) throws
HiveException {
+ return JoinUtil.computeMapJoinKeys(key, row, joinKeys[alias],
+ joinKeysObjectInspectors[alias]);
+ }
+
@Override
public void processOp(Object row, int tag) throws HiveException {
try {
@@ -185,8 +190,7 @@ public class MapJoinOperator extends Abs
alias = (byte)tag;
// compute keys and values as StandardObjects
- key = JoinUtil.computeMapJoinKeys(key, row, joinKeys[alias],
- joinKeysObjectInspectors[alias]);
+ key = computeMapJoinKey(row, alias);
boolean joinNeeded = false;
for (byte pos = 0; pos < order.length; pos++) {
if (pos != alias) {
@@ -207,7 +211,7 @@ public class MapJoinOperator extends Abs
}
}
if (joinNeeded) {
- ArrayList<Object> value = getFilteredValue(alias, row);
+ List<Object> value = getFilteredValue(alias, row);
// Add the value to the ArrayList
storage[alias].add(value);
// generate the output records
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
Fri Oct 18 21:54:58 2013
@@ -26,6 +26,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.VectorFilterOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorLimitOperator;
+import org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext;
@@ -118,6 +119,7 @@ public final class OperatorFactory {
vectorOpvec = new ArrayList<OpTuple>();
vectorOpvec.add(new OpTuple<SelectDesc>(SelectDesc.class,
VectorSelectOperator.class));
vectorOpvec.add(new OpTuple<GroupByDesc>(GroupByDesc.class,
VectorGroupByOperator.class));
+ vectorOpvec.add(new OpTuple<MapJoinDesc>(MapJoinDesc.class,
VectorMapJoinOperator.class));
vectorOpvec.add(new OpTuple<ReduceSinkDesc>(ReduceSinkDesc.class,
VectorReduceSinkOperator.class));
vectorOpvec.add(new OpTuple<FileSinkDesc>(FileSinkDesc.class,
VectorFileSinkOperator.class));
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
Fri Oct 18 21:54:58 2013
@@ -262,7 +262,7 @@ public class SMBMapJoinOperator extends
// compute keys and values as StandardObjects
ArrayList<Object> key = JoinUtil.computeKeys(row, joinKeys[alias],
joinKeysObjectInspectors[alias]);
- ArrayList<Object> value = getFilteredValue(alias, row);
+ List<Object> value = getFilteredValue(alias, row);
//have we reached a new key group?
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
Fri Oct 18 21:54:58 2013
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.exec.vector;
+import java.util.Arrays;
+
import org.apache.hadoop.io.Writable;
/**
@@ -26,33 +28,33 @@ import org.apache.hadoop.io.Writable;
* repeats, i.e. has all values the same, so only the first
* one is set. This is used to accelerate query performance
* by handling a whole vector in O(1) time when applicable.
- *
+ *
* The fields are public by design since this is a performance-critical
* structure that is used in the inner loop of query execution.
*/
public abstract class ColumnVector {
-
+
/*
- * If hasNulls is true, then this array contains true if the value
- * is null, otherwise false. The array is always allocated, so a batch can
be re-used
+ * If hasNulls is true, then this array contains true if the value
+ * is null, otherwise false. The array is always allocated, so a batch can
be re-used
* later and nulls added.
*/
- public boolean[] isNull;
-
+ public boolean[] isNull;
+
// If the whole column vector has no nulls, this is true, otherwise false.
public boolean noNulls;
-
- /*
- * True if same value repeats for whole column vector.
+
+ /*
+ * True if same value repeats for whole column vector.
* If so, vector[0] holds the repeating value.
*/
- public boolean isRepeating;
+ public boolean isRepeating;
public abstract Writable getWritableObject(int index);
/**
* Constructor for super-class ColumnVector. This is not called directly,
* but used to initialize inherited fields.
- *
+ *
* @param len Vector length
*/
public ColumnVector(int len) {
@@ -60,5 +62,19 @@ public abstract class ColumnVector {
noNulls = true;
isRepeating = false;
}
-}
+
+ /**
+ * Resets the column to default state
+ * - fills the isNull array with false
+ * - sets noNulls to true
+ * - sets isRepeating to false
+ */
+ public void reset() {
+ if (false == noNulls) {
+ Arrays.fill(isNull, false);
+ }
+ noNulls = true;
+ isRepeating = false;
+ }
+ }
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
Fri Oct 18 21:54:58 2013
@@ -452,25 +452,20 @@ public class VectorHashKeyWrapperBatch {
for(int i=0; i < keyExpressions.length; ++i) {
indexLookup[i] = new KeyLookupHelper();
String outputType = keyExpressions[i].getOutputType();
- if (outputType.equalsIgnoreCase("tinyint") ||
- outputType.equalsIgnoreCase("smallint") ||
- outputType.equalsIgnoreCase("int") ||
- outputType.equalsIgnoreCase("bigint") ||
- outputType.equalsIgnoreCase("timestamp") ||
- outputType.equalsIgnoreCase("boolean")) {
+ if (VectorizationContext.isIntFamily(outputType) ||
+ VectorizationContext.isDatetimeFamily(outputType)) {
longIndices[longIndicesIndex] = i;
indexLookup[i].longIndex = longIndicesIndex;
indexLookup[i].doubleIndex = -1;
indexLookup[i].stringIndex = -1;
++longIndicesIndex;
- } else if (outputType.equalsIgnoreCase("double") ||
- outputType.equalsIgnoreCase("float")) {
+ } else if (VectorizationContext.isFloatFamily(outputType)) {
doubleIndices[doubleIndicesIndex] = i;
indexLookup[i].longIndex = -1;
indexLookup[i].doubleIndex = doubleIndicesIndex;
indexLookup[i].stringIndex = -1;
++doubleIndicesIndex;
- } else if (outputType.equalsIgnoreCase("string")) {
+ } else if (VectorizationContext.isStringFamily(outputType)) {
stringIndices[stringIndicesIndex]= i;
indexLookup[i].longIndex = -1;
indexLookup[i].doubleIndex = -1;
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java
Fri Oct 18 21:54:58 2013
@@ -123,7 +123,7 @@ public class VectorReduceSinkOperator ex
VectorExpressionWriterFactory.processVectorExpressions(
conf.getKeyCols(),
conf.getOutputKeyColumnNames(),
- new VectorExpressionWriterFactory.Closure() {
+ new VectorExpressionWriterFactory.SingleOIDClosure() {
@Override
public void assign(VectorExpressionWriter[] writers,
ObjectInspector objectInspector) {
@@ -155,7 +155,7 @@ public class VectorReduceSinkOperator ex
VectorExpressionWriterFactory.processVectorExpressions(
conf.getValueCols(),
conf.getOutputValueColumnNames(),
- new VectorExpressionWriterFactory.Closure() {
+ new VectorExpressionWriterFactory.SingleOIDClosure() {
@Override
public void assign(VectorExpressionWriter[] writers,
ObjectInspector objectInspector) {
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
Fri Oct 18 21:54:58 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec.v
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.exec.SelectOperator;
@@ -54,10 +55,19 @@ public class VectorSelectOperator extend
vContext.setOperatorType(OperatorType.SELECT);
vExpressions = new VectorExpression[colList.size()];
for (int i = 0; i < colList.size(); i++) {
- vExpressions[i] = vContext.getVectorExpression(colList.get(i));
+ ExprNodeDesc expr = colList.get(i);
+ VectorExpression ve = vContext.getVectorExpression(expr);
+ vExpressions[i] = ve;
+ }
+
+ Map<String, Integer> cMap = vContext.getColumnMap();
+ for (int i=0; i < colList.size(); ++i) {
String columnName = this.conf.getOutputColumnNames().get(i);
- // Update column map with output column names
- vContext.addToColumnMap(columnName, vExpressions[i].getOutputColumn());
+ if (!cMap.containsKey(columnName)) {
+ VectorExpression ve = vExpressions[i];
+ // Update column map with output column names
+ vContext.addToColumnMap(columnName, ve.getOutputColumn());
+ }
}
}
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
Fri Oct 18 21:54:58 2013
@@ -190,6 +190,8 @@ public class VectorizationContext {
private static final String GENERATED_EXPR_PACKAGE =
"org.apache.hadoop.hive.ql.exec.vector.expressions.gen";
+ private String fileKey = null;
+
public VectorizationContext(Map<String, Integer> columnMap,
int initialOutputCol) {
this.columnMap = columnMap;
@@ -197,7 +199,18 @@ public class VectorizationContext {
this.firstOutputColumnIndex = initialOutputCol;
}
+ public String getFileKey() {
+ return fileKey;
+ }
+
+ public void setFileKey(String fileKey) {
+ this.fileKey = fileKey;
+ }
+
private int getInputColumnIndex(String name) {
+ if (!columnMap.containsKey(name)) {
+ LOG.error(String.format("The column %s is not in the vectorization
context column map.", name));
+ }
return columnMap.get(name);
}
@@ -269,6 +282,7 @@ public class VectorizationContext {
//Important: It will come here only if the column is being used as a
boolean
expr = new SelectColumnIsTrue(columnNum);
break;
+ case MAPJOIN:
case SELECT:
case GROUPBY:
case REDUCESINK:
@@ -1171,6 +1185,14 @@ public class VectorizationContext {
return ve;
}
+ public static boolean isStringFamily(String resultType) {
+ return resultType.equalsIgnoreCase("string");
+ }
+
+ public static boolean isDatetimeFamily(String resultType) {
+ return resultType.equalsIgnoreCase("timestamp");
+ }
+
// return true if this is any kind of float
public static boolean isFloatFamily(String resultType) {
return resultType.equalsIgnoreCase("double")
@@ -1183,7 +1205,23 @@ public class VectorizationContext {
|| resultType.equalsIgnoreCase("smallint")
|| resultType.equalsIgnoreCase("int")
|| resultType.equalsIgnoreCase("bigint")
- || resultType.equalsIgnoreCase("boolean");
+ || resultType.equalsIgnoreCase("boolean")
+ || resultType.equalsIgnoreCase("long");
+ }
+
+ public static String mapJavaTypeToVectorType(String javaType)
+ throws HiveException {
+ if (isStringFamily(javaType)) {
+ return "string";
+ }
+ if (isFloatFamily(javaType)) {
+ return "double";
+ }
+ if (isIntFamily(javaType) ||
+ isDatetimeFamily(javaType)) {
+ return "bigint";
+ }
+ throw new HiveException("Unsuported type for vectorization: " + javaType);
}
/* Return a unary string vector expression. This is used for functions like
@@ -2073,21 +2111,43 @@ public class VectorizationContext {
return map;
}
- public ColumnVector allocateColumnVector(String type, int defaultSize) {
- if (type.equalsIgnoreCase("double")) {
+ public Map<String, Integer> getColumnMap() {
+ return columnMap;
+ }
+
+ public static ColumnVector allocateColumnVector(String type, int
defaultSize) {
+ if (isFloatFamily(type)) {
return new DoubleColumnVector(defaultSize);
- } else if (type.equalsIgnoreCase("string")) {
+ } else if (isStringFamily(type)) {
return new BytesColumnVector(defaultSize);
} else {
return new LongColumnVector(defaultSize);
}
}
+ public void addToColumnMap(String columnName, int outputColumn) throws
HiveException {
+ if (columnMap.containsKey(columnName) && (columnMap.get(columnName) !=
outputColumn)) {
+ throw new HiveException(String.format("Column %s is already mapped to
%d. Cannot remap to %d.",
+ columnName, columnMap.get(columnName), outputColumn));
+ }
+ columnMap.put(columnName, outputColumn);
+ }
- public void addToColumnMap(String columnName, int outputColumn) {
- if (columnMap != null) {
- columnMap.put(columnName, outputColumn);
+ public <T> Map<T, VectorExpression[]> getMapVectorExpressions(
+ Map<T, List<ExprNodeDesc>> expressions) throws HiveException {
+ Map<T, VectorExpression[]> result = new HashMap<T, VectorExpression[]>();
+ if (null != expressions) {
+ for(T key: expressions.keySet()) {
+ result.put(key, getVectorExpressions(expressions.get(key)));
+ }
}
+ return result;
+ }
+
+ public void addOutputColumn(String columnName, String columnType) throws
HiveException {
+ String vectorType = mapJavaTypeToVectorType(columnType);
+ int columnIndex = ocm.allocateOutputColumn(vectorType);
+ this.addToColumnMap(columnName, columnIndex);
}
-}
+ }
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
Fri Oct 18 21:54:58 2013
@@ -20,6 +20,10 @@ package org.apache.hadoop.hive.ql.exec.v
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
import
org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -172,5 +176,67 @@ public class VectorizedRowBatch implemen
public void setValueWriters(VectorExpressionWriter[] valueWriters) {
this.valueWriters = valueWriters;
}
-}
+ public static VectorizedRowBatch buildBatch(Map<Integer, String> typeMap,
+ Map<String, Integer> columnMap) throws HiveException {
+
+ Map<Integer, ColumnVector> mapVectorColumn = new HashMap<Integer,
ColumnVector>(typeMap.size());
+ int maxIndex = 0;
+
+ Iterator<Entry<Integer, String>> typeMapIt = typeMap.entrySet().iterator();
+ while(typeMapIt.hasNext()) {
+ Entry<Integer, String> type = typeMapIt.next();
+ ColumnVector cv =
VectorizationContext.allocateColumnVector(type.getValue(),
+ VectorizedRowBatch.DEFAULT_SIZE);
+ mapVectorColumn.put(type.getKey(), cv);
+ if (maxIndex < type.getKey()) {
+ maxIndex = type.getKey();
+ }
+ }
+
+ VectorizedRowBatch batch = new VectorizedRowBatch(maxIndex+1);
+ for(int i=0; i <= maxIndex; ++i) {
+ ColumnVector cv = mapVectorColumn.get(i);
+ if (cv == null) {
+ // allocate a default type for the unused column.
+ // there are APIs that expect all cols[i] to be non NULL
+ cv = VectorizationContext.allocateColumnVector("long",
+ VectorizedRowBatch.DEFAULT_SIZE);
+ }
+ batch.cols[i] = cv;
+ }
+
+ // Validate that every column in the column map exists
+ Iterator<Entry<String, Integer>> columnMapIt =
columnMap.entrySet().iterator();
+ while(columnMapIt.hasNext()) {
+ Entry<String, Integer> cm = columnMapIt.next();
+ if (batch.cols.length <= cm.getValue() || batch.cols[cm.getValue()] ==
null) {
+ throw new HiveException(String.format(
+ "Internal error: The type map has no entry for column %d %s",
+ cm.getValue(), cm.getKey()));
+ }
+ }
+
+ batch.reset();
+
+ return batch;
+ }
+
+ /**
+ * Resets the row batch to default state
+ * - sets selectedInUse to false
+ * - sets size to 0
+ * - sets endOfFile to false
+ * - resets each column
+ */
+ public void reset() {
+ selectedInUse = false;
+ size = 0;
+ endOfFile = false;
+ for (ColumnVector vc : cols) {
+ if (vc != null) {
+ vc.reset();
+ }
+ }
+ }
+}
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
Fri Oct 18 21:54:58 2013
@@ -75,7 +75,7 @@ public final class VectorExpressionWrite
}
if (null == objectInspector) {
throw new HiveException(String.format(
- "Failed to initialize VectorExpressionWriter for expr: %s",
+ "Failed to initialize VectorExpressionWriter for expr: %s",
nodeDesc.getExprString()));
}
return this;
@@ -378,18 +378,22 @@ public final class VectorExpressionWrite
* A poor man Java closure. Works around the problem of having to return
multiple objects
* from one function call.
*/
- public static interface Closure {
+ public static interface SingleOIDClosure {
void assign(VectorExpressionWriter[] writers, ObjectInspector
objectInspector);
}
+ public static interface ListOIDClosure {
+ void assign(VectorExpressionWriter[] writers, List<ObjectInspector> oids);
+ }
+
/**
* Creates the value writers for a column vector expression list.
* Creates an appropriate output object inspector.
*/
public static void processVectorExpressions(
List<ExprNodeDesc> nodesDesc,
- List<String> outputColumnNames,
- Closure closure)
+ List<String> columnNames,
+ SingleOIDClosure closure)
throws HiveException {
VectorExpressionWriter[] writers = getExpressionWriters(nodesDesc);
List<ObjectInspector> oids = new
ArrayList<ObjectInspector>(writers.length);
@@ -397,10 +401,26 @@ public final class VectorExpressionWrite
oids.add(writers[i].getObjectInspector());
}
ObjectInspector objectInspector = ObjectInspectorFactory.
- getStandardStructObjectInspector(outputColumnNames,oids);
+ getStandardStructObjectInspector(columnNames,oids);
closure.assign(writers, objectInspector);
}
+ /**
+ * Creates the value writers for a column vector expression list.
+ * Creates an appropriate output object inspector.
+ */
+ public static void processVectorExpressions(
+ List<ExprNodeDesc> nodesDesc,
+ ListOIDClosure closure)
+ throws HiveException {
+ VectorExpressionWriter[] writers = getExpressionWriters(nodesDesc);
+ List<ObjectInspector> oids = new
ArrayList<ObjectInspector>(writers.length);
+ for(int i=0; i<writers.length; ++i) {
+ oids.add(writers[i].getObjectInspector());
+ }
+ closure.assign(writers, oids);
+ }
+
/**
* Returns {@link VectorExpressionWriter} objects for the fields in the given
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
Fri Oct 18 21:54:58 2013
@@ -307,10 +307,6 @@ public class ZooKeeperHiveLockManager im
try {
if (tryNum > 1) {
Thread.sleep(sleepTime);
- if (zooKeeper.getState() == ZooKeeper.States.CLOSED) {
- // Reconnect if the connection is closed.
- zooKeeper = null;
- }
prepareRetry();
}
ret = lockPrimitive(key, mode, keepAlive, parentCreated);
@@ -752,6 +748,10 @@ public class ZooKeeperHiveLockManager im
@Override
public void prepareRetry() throws LockException {
try {
+ if (zooKeeper != null && zooKeeper.getState() ==
ZooKeeper.States.CLOSED) {
+ // Reconnect if the connection is closed.
+ zooKeeper = null;
+ }
renewZookeeperInstance(sessionTimeout, quorumServers);
} catch (Exception e) {
throw new LockException(e);
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
Fri Oct 18 21:54:58 2013
@@ -37,6 +37,7 @@ import org.apache.hadoop.hive.ql.exec.Co
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.FilterOperator;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
+import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.OperatorFactory;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
@@ -68,6 +69,7 @@ import org.apache.hadoop.hive.ql.plan.Ag
import org.apache.hadoop.hive.ql.plan.BaseWork;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
@@ -344,6 +346,7 @@ public class Vectorizer implements Physi
HashMap<Node, Object> nodeOutput = new HashMap<Node, Object>();
ogw.startWalking(topNodes, nodeOutput);
mapWork.setScratchColumnVectorTypes(vnp.getScratchColumnVectorTypes());
+ mapWork.setScratchColumnMap(vnp.getScratchColumnMap());
return;
}
}
@@ -396,6 +399,17 @@ public class Vectorizer implements Physi
return scratchColumnVectorTypes;
}
+ public Map<String, Map<String, Integer>> getScratchColumnMap() {
+ Map<String, Map<String, Integer>> scratchColumnMap =
+ new HashMap<String, Map<String, Integer>>();
+ for(String oneFile: vectorizationContexts.keySet()) {
+ VectorizationContext vc = vectorizationContexts.get(oneFile);
+ Map<String, Integer> cmap = vc.getColumnMap();
+ scratchColumnMap.put(oneFile, cmap);
+ }
+ return scratchColumnMap;
+ }
+
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
@@ -419,6 +433,7 @@ public class Vectorizer implements Physi
}
}
vContext = getVectorizationContext(tsOp, physicalContext);
+ vContext.setFileKey(fileKey);
vectorizationContexts.put(fileKey, vContext);
vContextsByTSOp.put(tsOp, vContext);
}
@@ -473,6 +488,9 @@ public class Vectorizer implements Physi
boolean validateOperator(Operator<? extends OperatorDesc> op) {
boolean ret = false;
switch (op.getType()) {
+ case MAPJOIN:
+ ret = validateMapJoinOperator((MapJoinOperator) op);
+ break;
case GROUPBY:
ret = validateGroupByOperator((GroupByOperator) op);
break;
@@ -497,6 +515,17 @@ public class Vectorizer implements Physi
return ret;
}
+ private boolean validateMapJoinOperator(MapJoinOperator op) {
+ MapJoinDesc desc = op.getConf();
+ byte posBigTable = (byte) desc.getPosBigTable();
+ List<ExprNodeDesc> filterExprs = desc.getFilters().get(posBigTable);
+ List<ExprNodeDesc> keyExprs = desc.getKeys().get(posBigTable);
+ List<ExprNodeDesc> valueExprs = desc.getExprs().get(posBigTable);
+ return validateExprNodeDesc(filterExprs) &&
+ validateExprNodeDesc(keyExprs) &&
+ validateExprNodeDesc(valueExprs);
+ }
+
private boolean validateReduceSinkOperator(ReduceSinkOperator op) {
List<ExprNodeDesc> keyDescs = op.getConf().getKeyCols();
List<ExprNodeDesc> partitionDescs = op.getConf().getPartitionCols();
@@ -616,11 +645,22 @@ public class Vectorizer implements Physi
return new VectorizationContext(cmap, columnCount);
}
+
+ private void extendVectorizationContextOutput(Operator<? extends
OperatorDesc> op,
+ VectorizationContext vContext) throws HiveException {
+
+ RowResolver rr =
physicalContext.getParseContext().getOpParseCtx().get(op).getRowResolver();
+ for(ColumnInfo c : rr.getColumnInfos()) {
+ vContext.addOutputColumn(c.getInternalName(), c.getTypeName());
+ }
+ }
+
Operator<? extends OperatorDesc> vectorizeOperator(Operator<? extends
OperatorDesc> op,
VectorizationContext vContext) throws HiveException {
Operator<? extends OperatorDesc> vectorOp = null;
switch (op.getType()) {
+ case MAPJOIN:
case GROUPBY:
case FILTER:
case SELECT:
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java?rev=1533645&r1=1533644&r2=1533645&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
(original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
Fri Oct 18 21:54:58 2013
@@ -113,6 +113,7 @@ public class MapWork extends BaseWork {
private boolean useBucketizedHiveInputFormat;
private Map<String, Map<Integer, String>> scratchColumnVectorTypes = null;
+ private Map<String, Map<String, Integer>> scratchColumnMap = null;
private boolean vectorMode = false;
public MapWork() {}
@@ -495,11 +496,21 @@ public class MapWork extends BaseWork {
this.scratchColumnVectorTypes = scratchColumnVectorTypes;
}
+ public Map<String, Map<String, Integer>> getScratchColumnMap() {
+ return scratchColumnMap;
+ }
+
+ public void setScratchColumnMap(Map<String, Map<String, Integer>>
scratchColumnMap) {
+ this.scratchColumnMap = scratchColumnMap;
+ }
+
public boolean getVectorMode() {
return vectorMode;
}
+ @Override
public void setVectorMode(boolean vectorMode) {
this.vectorMode = vectorMode;
}
+
}
