Author: hashutosh
Date: Fri Apr 17 01:51:36 2015
New Revision: 1674187
URL: http://svn.apache.org/r1674187
Log:
Merged latest trunk into cbo branch (Ashutosh Chauhan)
Modified:
hive/branches/cbo/ (props changed)
hive/branches/cbo/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
hive/branches/cbo/pom.xml
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
Propchange: hive/branches/cbo/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Apr 17 01:51:36 2015
@@ -3,4 +3,4 @@
/hive/branches/spark:1608589-1660298
/hive/branches/tez:1494760-1622766
/hive/branches/vectorization:1466908-1527856
-/hive/trunk:1605012-1674131
+/hive/trunk:1605012-1674185
Modified:
hive/branches/cbo/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java?rev=1674187&r1=1674186&r2=1674187&view=diff
==============================================================================
---
hive/branches/cbo/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
(original)
+++
hive/branches/cbo/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
Fri Apr 17 01:51:36 2015
@@ -380,6 +380,7 @@ public class HBaseStorageHandler extends
// do this for reconciling HBaseStorageHandler for use in HCatalog
// check to see if this an input job or an outputjob
if (this.configureInputJobProps) {
+ LOG.info("Configuring input job properties");
String snapshotName = HiveConf.getVar(jobConf,
HiveConf.ConfVars.HIVE_HBASE_SNAPSHOT_NAME);
if (snapshotName != null) {
HBaseTableSnapshotInputFormatUtil.assertSupportsTableSnapshots();
@@ -428,6 +429,7 @@ public class HBaseStorageHandler extends
} //input job properties
}
else {
+ LOG.info("Configuring output job properties");
if (isHBaseGenerateHFiles(jobConf)) {
// only support bulkload when a hfile.family.path has been specified.
// TODO: support detecting cf's from column mapping
Modified: hive/branches/cbo/pom.xml
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/pom.xml?rev=1674187&r1=1674186&r2=1674187&view=diff
==============================================================================
--- hive/branches/cbo/pom.xml (original)
+++ hive/branches/cbo/pom.xml Fri Apr 17 01:51:36 2015
@@ -100,7 +100,7 @@
<antlr.version>3.4</antlr.version>
<avro.version>1.7.5</avro.version>
<bonecp.version>0.8.0.RELEASE</bonecp.version>
- <calcite.version>1.2.0-incubating-SNAPSHOT</calcite.version>
+ <calcite.version>1.2.0-incubating</calcite.version>
<datanucleus-api-jdo.version>3.2.6</datanucleus-api-jdo.version>
<datanucleus-core.version>3.2.10</datanucleus-core.version>
<datanucleus-rdbms.version>3.2.9</datanucleus-rdbms.version>
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java?rev=1674187&r1=1674186&r2=1674187&view=diff
==============================================================================
---
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
(original)
+++
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
Fri Apr 17 01:51:36 2015
@@ -23,6 +23,7 @@ import static org.apache.hadoop.hive.con
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.Serializable;
+import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -30,6 +31,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import java.util.concurrent.Future;
@@ -59,6 +61,7 @@ import org.apache.hadoop.hive.ql.plan.Fi
import org.apache.hadoop.hive.ql.plan.ListBucketingCtx;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.apache.hadoop.hive.ql.plan.SkewedColumnPositionPair;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.ql.stats.StatsCollectionTaskIndependent;
import org.apache.hadoop.hive.ql.stats.StatsPublisher;
@@ -339,7 +342,12 @@ public class FileSinkOperator extends Te
taskId = Utilities.getTaskId(hconf);
initializeSpecPath();
fs = specPath.getFileSystem(hconf);
- hiveOutputFormat = HiveFileFormatUtils.getHiveOutputFormat(hconf,
conf.getTableInfo());
+ try {
+ createHiveOutputFormat(hconf);
+ } catch (HiveException ex) {
+ logOutputFormatError(hconf, ex);
+ throw ex;
+ }
isCompressed = conf.getCompressed();
parent = Utilities.toTempPath(conf.getDirName());
statsCollectRawDataSize = conf.isStatsCollectRawDataSize();
@@ -440,6 +448,26 @@ public class FileSinkOperator extends Te
return result;
}
+ private void logOutputFormatError(Configuration hconf, HiveException ex) {
+ StringWriter errorWriter = new StringWriter();
+ errorWriter.append("Failed to create output format; configuration: ");
+ try {
+ Configuration.dumpConfiguration(hconf, errorWriter);
+ } catch (IOException ex2) {
+ errorWriter.append("{ failed to dump configuration: " + ex2.getMessage()
+ " }");
+ }
+ Properties tdp = null;
+ if (this.conf.getTableInfo() != null
+ && (tdp = this.conf.getTableInfo().getProperties()) != null) {
+ errorWriter.append(";\n table properties: { ");
+ for (Map.Entry<Object, Object> e : tdp.entrySet()) {
+ errorWriter.append(e.getKey() + ": " + e.getValue() + ", ");
+ }
+ errorWriter.append('}');
+ }
+ LOG.error(errorWriter.toString(), ex);
+ }
+
/**
* Initialize list bucketing information
*/
@@ -1082,10 +1110,10 @@ public class FileSinkOperator extends Te
public void checkOutputSpecs(FileSystem ignored, JobConf job) throws
IOException {
if (hiveOutputFormat == null) {
- Utilities.copyTableJobPropertiesToConf(conf.getTableInfo(), job);
try {
- hiveOutputFormat = HiveFileFormatUtils.getHiveOutputFormat(job,
getConf().getTableInfo());
- } catch (Exception ex) {
+ createHiveOutputFormat(job);
+ } catch (HiveException ex) {
+ logOutputFormatError(job, ex);
throw new IOException(ex);
}
}
@@ -1101,6 +1129,17 @@ public class FileSinkOperator extends Te
}
}
+ private void createHiveOutputFormat(Configuration hconf) throws
HiveException {
+ if (hiveOutputFormat == null) {
+ Utilities.copyTableJobPropertiesToConf(conf.getTableInfo(), hconf);
+ }
+ try {
+ hiveOutputFormat = HiveFileFormatUtils.getHiveOutputFormat(hconf,
getConf().getTableInfo());
+ } catch (Throwable t) {
+ throw (t instanceof HiveException) ? (HiveException)t : new
HiveException(t);
+ }
+ }
+
private void publishStats() throws HiveException {
boolean isStatsReliable = conf.isStatsReliable();
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1674187&r1=1674186&r2=1674187&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
(original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
Fri Apr 17 01:51:36 2015
@@ -2398,7 +2398,7 @@ public final class Utilities {
* @param job
* configuration which receives configured properties
*/
- public static void copyTableJobPropertiesToConf(TableDesc tbl, JobConf job) {
+ public static void copyTableJobPropertiesToConf(TableDesc tbl, Configuration
job) {
Properties tblProperties = tbl.getProperties();
for(String name: tblProperties.stringPropertyNames()) {
if (job.get(name) == null) {
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java?rev=1674187&r1=1674186&r2=1674187&view=diff
==============================================================================
---
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
(original)
+++
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
Fri Apr 17 01:51:36 2015
@@ -74,7 +74,6 @@ public class MetadataReader {
// filter and combine the io to read row index and bloom filters for
that column together
if (stream.hasKind() && (stream.getKind() ==
OrcProto.Stream.Kind.ROW_INDEX)) {
boolean readBloomFilter = false;
- // TODO#: HERE
if (sargColumns != null && sargColumns[col] &&
nextStream.getKind() == OrcProto.Stream.Kind.BLOOM_FILTER) {
len += nextStream.getLength();
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=1674187&r1=1674186&r2=1674187&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
(original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
Fri Apr 17 01:51:36 2015
@@ -812,7 +812,7 @@ public final class PlanUtils {
tableDesc,
jobProperties);
} catch(AbstractMethodError e) {
- LOG.debug("configureInputJobProperties not found "+
+ LOG.info("configureInputJobProperties not found "+
"using configureTableJobProperties",e);
storageHandler.configureTableJobProperties(tableDesc,
jobProperties);
}
@@ -823,7 +823,7 @@ public final class PlanUtils {
tableDesc,
jobProperties);
} catch(AbstractMethodError e) {
- LOG.debug("configureOutputJobProperties not found"+
+ LOG.info("configureOutputJobProperties not found"+
"using configureTableJobProperties",e);
storageHandler.configureTableJobProperties(tableDesc,
jobProperties);
}
