svn commit: r1640916 - in /hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark: HiveSparkClientFactory.java session/SparkSession.java session/SparkSessionImpl.java
Author: xuefu
Date: Fri Nov 21 14:06:08 2014
New Revision: 1640916
URL: http://svn.apache.org/r1640916
Log:
HIVE-8868: SparkSession and SparkClient mapping[Spark Branch] (Rui via Xuefu)
Modified:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSession.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
Modified:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java?rev=1640916r1=1640915r2=1640916view=diff
==
---
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
(original)
+++
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
Fri Nov 21 14:06:08 2014
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.hadoop.hive.ql.exec.spark;
import org.apache.commons.compress.utils.CharsetNames;
Modified:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSession.java
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSession.java?rev=1640916r1=1640915r2=1640916view=diff
==
---
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSession.java
(original)
+++
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSession.java
Fri Nov 21 14:06:08 2014
@@ -20,13 +20,14 @@ package org.apache.hadoop.hive.ql.exec.s
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.exec.spark.status.SparkJobRef;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.SparkWork;
public interface SparkSession {
/**
* Initializes a Spark session for DAG execution.
*/
- public void open(HiveConf conf);
+ public void open(HiveConf conf) throws HiveException;
/**
* Submit given isparkWork/i to SparkClient
Modified:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java?rev=1640916r1=1640915r2=1640916view=diff
==
---
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
(original)
+++
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
Fri Nov 21 14:06:08 2014
@@ -26,15 +26,13 @@ import org.apache.hadoop.hive.ql.DriverC
import org.apache.hadoop.hive.ql.exec.spark.HiveSparkClientFactory;
import org.apache.hadoop.hive.ql.exec.spark.HiveSparkClient;
import org.apache.hadoop.hive.ql.exec.spark.status.SparkJobRef;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.SparkWork;
+import org.apache.spark.SparkException;
import java.io.IOException;
import java.util.UUID;
-/**
- * Simple implementation of iSparkSession/i which currently just submits
jobs to
- * SparkClient which is shared by all SparkSession instances.
- */
public class SparkSessionImpl implements SparkSession {
private static final Log LOG = LogFactory.getLog(SparkSession.class);
@@ -48,16 +46,19 @@ public class SparkSessionImpl implements
}
@Override
- public void open(HiveConf conf) {
+ public void open(HiveConf conf) throws HiveException {
this.conf = conf;
isOpen = true;
+try {
+ hiveSparkClient = HiveSparkClientFactory.createHiveSparkClient(conf);
+} catch (Exception e) {
+ throw new HiveException(Failed to create spark client., e);
+}
}
@Override
public SparkJobRef submit(DriverContext driverContext, SparkWork sparkWork)
throws Exception {
Preconditions.checkState(isOpen, Session is not open. Can't submit
jobs.);
-Configuration hiveConf = driverContext.getCtx().getConf();
-hiveSparkClient = HiveSparkClientFactory.createHiveSparkClient(hiveConf);
return hiveSparkClient.execute(driverContext, sparkWork);
}
svn commit: r1640934 - in /hive/branches/spark: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/parse/spark/ ql/src/test/results/clientpositive/spark/
Author: xuefu
Date: Fri Nov 21 15:17:16 2014
New Revision: 1640934
URL: http://svn.apache.org/r1640934
Log:
HIVE-8756: numRows and rawDataSize are not collected by the Spark stats [Spark
Branch] (Na via Xuefu)
Added:
hive/branches/spark/ql/src/test/results/clientpositive/spark/stats1.q.out
Modified:
hive/branches/spark/itests/src/test/resources/testconfiguration.properties
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/semijoin.q.out
Modified:
hive/branches/spark/itests/src/test/resources/testconfiguration.properties
URL:
http://svn.apache.org/viewvc/hive/branches/spark/itests/src/test/resources/testconfiguration.properties?rev=1640934r1=1640933r2=1640934view=diff
==
--- hive/branches/spark/itests/src/test/resources/testconfiguration.properties
(original)
+++ hive/branches/spark/itests/src/test/resources/testconfiguration.properties
Fri Nov 21 15:17:16 2014
@@ -834,6 +834,7 @@ spark.query.files=add_part_multiple.q, \
stats_only_null.q, \
stats_partscan_1_23.q, \
stats0.q, \
+ stats1.q, \
stats10.q, \
stats12.q, \
stats13.q, \
Modified:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java?rev=1640934r1=1640933r2=1640934view=diff
==
---
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java
(original)
+++
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java
Fri Nov 21 15:17:16 2014
@@ -130,6 +130,7 @@ public class GenSparkProcContext impleme
public final SetReduceSinkOperator clonedReduceSinks;
public final SetFileSinkOperator fileSinkSet;
+ public final MapFileSinkOperator, ListFileSinkOperator fileSinkMap;
// remember which reducesinks we've already connected
public final SetReduceSinkOperator connectedReduceSinks;
@@ -169,6 +170,7 @@ public class GenSparkProcContext impleme
this.workWithUnionOperators = new LinkedHashSetBaseWork();
this.clonedReduceSinks = new LinkedHashSetReduceSinkOperator();
this.fileSinkSet = new LinkedHashSetFileSinkOperator();
+this.fileSinkMap = new LinkedHashMapFileSinkOperator,
ListFileSinkOperator();
this.connectedReduceSinks = new LinkedHashSetReduceSinkOperator();
}
}
Modified:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java?rev=1640934r1=1640933r2=1640934view=diff
==
---
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
(original)
+++
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
Fri Nov 21 15:17:16 2014
@@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.exec.FetchTask;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
@@ -48,6 +49,7 @@ import org.apache.hadoop.hive.ql.plan.Re
import org.apache.hadoop.hive.ql.plan.SparkEdgeProperty;
import org.apache.hadoop.hive.ql.plan.SparkWork;
import org.apache.hadoop.hive.ql.plan.UnionWork;
+import org.apache.hadoop.hive.ql.stats.StatsFactory;
import java.util.ArrayList;
import java.util.Deque;
@@ -182,6 +184,15 @@ public class GenSparkUtils {
context.inputs, partitions, root, alias, context.conf, false);
}
+ private void collectOperators (Operator? op, ListOperator? opList) {
+opList.add(op);
+for (Object child : op.getChildOperators()) {
+ if (child != null) {
+collectOperators((Operator?)child, opList);
+ }
+}
+ }
+
// removes any union operator and clones the plan
public void removeUnionOperators(Configuration conf, GenSparkProcContext
context,
BaseWork work)
@@ -196,6 +207,29 @@ public class GenSparkUtils {
// need to clone the plan.
ListOperator? newRoots = Utilities.cloneOperatorTree(conf, roots);
+// Build a map to map the original FileSinkOperator and the
svn commit: r1640943 - in /hive/trunk: common/src/java/org/apache/hive/common/util/ metastore/src/java/org/apache/hadoop/hive/metastore/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientp
Author: hashutosh
Date: Fri Nov 21 15:42:35 2014
New Revision: 1640943
URL: http://svn.apache.org/r1640943
Log:
HIVE-8863 : Cannot drop table with uppercase name after compute statistics for
columns (Chaoyu Tang via Ashutosh Chauhan)
Added:
hive/trunk/ql/src/test/queries/clientpositive/drop_partition_with_stats.q
hive/trunk/ql/src/test/queries/clientpositive/drop_table_with_stats.q
hive/trunk/ql/src/test/results/clientpositive/drop_partition_with_stats.q.out
hive/trunk/ql/src/test/results/clientpositive/drop_table_with_stats.q.out
Modified:
hive/trunk/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
Modified:
hive/trunk/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
URL:
http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hive/common/util/HiveStringUtils.java?rev=1640943r1=1640942r2=1640943view=diff
==
--- hive/trunk/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
(original)
+++ hive/trunk/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
Fri Nov 21 15:42:35 2014
@@ -882,4 +882,8 @@ public class HiveStringUtils {
}
return len;
}
+
+ public static String normalizeIdentifier(String identifier) {
+ return identifier.trim().toLowerCase();
+ }
}
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1640943r1=1640942r2=1640943view=diff
==
---
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
(original)
+++
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
Fri Nov 21 15:42:35 2014
@@ -137,6 +137,7 @@ import org.apache.hadoop.hive.serde2.typ
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hive.common.util.HiveStringUtils;
import org.apache.thrift.TException;
import org.datanucleus.store.rdbms.exceptions.MissingTableException;
@@ -506,7 +507,7 @@ public class ObjectStore implements RawS
boolean commited = false;
try {
openTransaction();
- name = name.toLowerCase().trim();
+ name = HiveStringUtils.normalizeIdentifier(name);
Query query = pm.newQuery(MDatabase.class, name == dbname);
query.declareParameters(java.lang.String dbname);
query.setUnique(true);
@@ -618,7 +619,7 @@ public class ObjectStore implements RawS
public boolean dropDatabase(String dbname) throws NoSuchObjectException,
MetaException {
boolean success = false;
LOG.info(Dropping database + dbname + along with all tables);
-dbname = dbname.toLowerCase();
+dbname = HiveStringUtils.normalizeIdentifier(dbname);
try {
openTransaction();
@@ -915,7 +916,7 @@ public class ObjectStore implements RawS
ListString tbls = null;
try {
openTransaction();
- dbName = dbName.toLowerCase().trim();
+ dbName = HiveStringUtils.normalizeIdentifier(dbName);
// Take the pattern and split it on the | to get all the composing
// patterns
String[] subpatterns = pattern.trim().split(\\|);
@@ -961,8 +962,8 @@ public class ObjectStore implements RawS
boolean commited = false;
try {
openTransaction();
- db = db.toLowerCase().trim();
- table = table.toLowerCase().trim();
+ db = HiveStringUtils.normalizeIdentifier(db);
+ table = HiveStringUtils.normalizeIdentifier(table);
Query query = pm.newQuery(MTable.class, tableName == table
database.name == db);
query.declareParameters(java.lang.String table, java.lang.String db);
query.setUnique(true);
@@ -985,7 +986,7 @@ public class ObjectStore implements RawS
try {
openTransaction();
- db = db.toLowerCase().trim();
+ db = HiveStringUtils.normalizeIdentifier(db);
Query dbExistsQuery = pm.newQuery(MDatabase.class, name == db);
dbExistsQuery.declareParameters(java.lang.String db);
dbExistsQuery.setUnique(true);
@@ -997,7 +998,7 @@ public class ObjectStore implements RawS
ListString lowered_tbl_names = new ArrayListString();
for (String t : tbl_names) {
-lowered_tbl_names.add(t.toLowerCase().trim());
+lowered_tbl_names.add(HiveStringUtils.normalizeIdentifier(t));
}
Query query = pm.newQuery(MTable.class);
query.setFilter(database.name == db tbl_names.contains(tableName));
@@ -1077,7 +1078,7 @@ public class ObjectStore implements RawS
}
// A new table is always created with a new column descriptor
-return new
svn commit: r1640963 - /hive/trunk/data/files/
Author: brock Date: Fri Nov 21 17:29:52 2014 New Revision: 1640963 URL: http://svn.apache.org/r1640963 Log: HIVE-8909 - Hive doesn't correctly read Parquet nested types (Data Files) Added: hive/trunk/data/files/AvroPrimitiveInList.parquet (with props) hive/trunk/data/files/AvroSingleFieldGroupInList.parquet (with props) hive/trunk/data/files/HiveRequiredGroupInList.parquet (with props) hive/trunk/data/files/MultiFieldGroupInList.parquet (with props) hive/trunk/data/files/NewOptionalGroupInList.parquet (with props) hive/trunk/data/files/NewRequiredGroupInList.parquet (with props) hive/trunk/data/files/SingleFieldGroupInList.parquet (with props) hive/trunk/data/files/ThriftPrimitiveInList.parquet (with props) hive/trunk/data/files/ThriftSingleFieldGroupInList.parquet (with props) hive/trunk/data/files/UnannotatedListOfGroups.parquet (with props) hive/trunk/data/files/UnannotatedListOfPrimitives.parquet (with props) Added: hive/trunk/data/files/AvroPrimitiveInList.parquet URL: http://svn.apache.org/viewvc/hive/trunk/data/files/AvroPrimitiveInList.parquet?rev=1640963view=auto == Binary file - no diff available. Propchange: hive/trunk/data/files/AvroPrimitiveInList.parquet -- svn:mime-type = application/octet-stream Added: hive/trunk/data/files/AvroSingleFieldGroupInList.parquet URL: http://svn.apache.org/viewvc/hive/trunk/data/files/AvroSingleFieldGroupInList.parquet?rev=1640963view=auto == Binary file - no diff available. Propchange: hive/trunk/data/files/AvroSingleFieldGroupInList.parquet -- svn:mime-type = application/octet-stream Added: hive/trunk/data/files/HiveRequiredGroupInList.parquet URL: http://svn.apache.org/viewvc/hive/trunk/data/files/HiveRequiredGroupInList.parquet?rev=1640963view=auto == Binary file - no diff available. Propchange: hive/trunk/data/files/HiveRequiredGroupInList.parquet -- svn:mime-type = application/octet-stream Added: hive/trunk/data/files/MultiFieldGroupInList.parquet URL: http://svn.apache.org/viewvc/hive/trunk/data/files/MultiFieldGroupInList.parquet?rev=1640963view=auto == Binary file - no diff available. Propchange: hive/trunk/data/files/MultiFieldGroupInList.parquet -- svn:mime-type = application/octet-stream Added: hive/trunk/data/files/NewOptionalGroupInList.parquet URL: http://svn.apache.org/viewvc/hive/trunk/data/files/NewOptionalGroupInList.parquet?rev=1640963view=auto == Binary file - no diff available. Propchange: hive/trunk/data/files/NewOptionalGroupInList.parquet -- svn:mime-type = application/octet-stream Added: hive/trunk/data/files/NewRequiredGroupInList.parquet URL: http://svn.apache.org/viewvc/hive/trunk/data/files/NewRequiredGroupInList.parquet?rev=1640963view=auto == Binary file - no diff available. Propchange: hive/trunk/data/files/NewRequiredGroupInList.parquet -- svn:mime-type = application/octet-stream Added: hive/trunk/data/files/SingleFieldGroupInList.parquet URL: http://svn.apache.org/viewvc/hive/trunk/data/files/SingleFieldGroupInList.parquet?rev=1640963view=auto == Binary file - no diff available. Propchange: hive/trunk/data/files/SingleFieldGroupInList.parquet -- svn:mime-type = application/octet-stream Added: hive/trunk/data/files/ThriftPrimitiveInList.parquet URL: http://svn.apache.org/viewvc/hive/trunk/data/files/ThriftPrimitiveInList.parquet?rev=1640963view=auto == Binary file - no diff available. Propchange: hive/trunk/data/files/ThriftPrimitiveInList.parquet -- svn:mime-type = application/octet-stream Added: hive/trunk/data/files/ThriftSingleFieldGroupInList.parquet URL: http://svn.apache.org/viewvc/hive/trunk/data/files/ThriftSingleFieldGroupInList.parquet?rev=1640963view=auto
svn commit: r1640981 - in /hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton: AppConfig.java tool/TempletonUtils.java tool/TrivialExecService.java
Author: ekoifman
Date: Fri Nov 21 19:24:54 2014
New Revision: 1640981
URL: http://svn.apache.org/r1640981
Log:
HIVE-8877 improve context logging during job submission via WebHCat (reviewed
by Thejas Nair)
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java?rev=1640981r1=1640980r2=1640981view=diff
==
---
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
(original)
+++
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
Fri Nov 21 19:24:54 2014
@@ -28,12 +28,14 @@ import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.StringTokenizer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.SystemVariables;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hive.hcatalog.templeton.tool.JobState;
@@ -224,14 +226,8 @@ public class AppConfig extends Configura
* support/debugging. Later it may be worth adding a REST call which will
return this data.
*/
private String dumpEnvironent() {
-StringBuilder sb = new StringBuilder(WebHCat environment:\n);
-MapString, String env = System.getenv();
-ListString propKeys = new ArrayListString(env.keySet());
-Collections.sort(propKeys);
-for(String propKey : propKeys) {
- sb.append(propKey).append('=').append(env.get(propKey)).append('\n');
-}
-sb.append(Configration properties: \n);
+StringBuilder sb = TempletonUtils.dumpPropMap(WebHCat
System.getenv(), System.getenv());
+sb.append(STARTWebHCat AppConfig.iterator(): \n);
IteratorMap.EntryString, String configIter = this.iterator();
ListMap.EntryString, String configVals = new
ArrayListMap.EntryString, String();
while(configIter.hasNext()) {
@@ -245,8 +241,19 @@ public class AppConfig extends Configura
});
for(Map.EntryString, String entry : configVals) {
//use get() to make sure variable substitution works
-
sb.append(entry.getKey()).append('=').append(get(entry.getKey())).append('\n');
+ if(entry.getKey().toLowerCase().contains(path)) {
+StringTokenizer st = new StringTokenizer(get(entry.getKey()),
File.pathSeparator);
+sb.append(entry.getKey()).append(=\n);
+while(st.hasMoreTokens()) {
+ sb.append(
).append(st.nextToken()).append(File.pathSeparator).append('\n');
+}
+ }
+ else {
+
sb.append(entry.getKey()).append('=').append(get(entry.getKey())).append('\n');
+ }
}
+sb.append(ENDWebHCat AppConfig.iterator(): \n);
+sb.append(TempletonUtils.dumpPropMap(WebHCat
System.getProperties(), System.getProperties()));
return sb.toString();
}
public void startCleanup() {
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java?rev=1640981r1=1640980r2=1640981view=diff
==
---
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
(original)
+++
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
Fri Nov 21 19:24:54 2014
@@ -18,6 +18,7 @@
*/
package org.apache.hive.hcatalog.templeton.tool;
+import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
@@ -34,6 +35,8 @@ import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
+import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -408,4 +411,34 @@ public class TempletonUtils {
}
return null;
}
+ public static StringBuilder dumpPropMap(String header, Properties props) {
+MapString, String map = new HashMapString, String();
+for(Map.EntryObject, Object
svn commit: r1641007 - /hive/branches/HIVE-8065/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Author: brock
Date: Fri Nov 21 21:57:23 2014
New Revision: 1641007
URL: http://svn.apache.org/r1641007
Log:
HIVE-8945 - Allow user to read encrypted read-only tables only if the scratch
directory is encrypted (Sergio Pena via Brock)
Modified:
hive/branches/HIVE-8065/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Modified:
hive/branches/HIVE-8065/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL:
http://svn.apache.org/viewvc/hive/branches/HIVE-8065/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1641007r1=1641006r2=1641007view=diff
==
---
hive/branches/HIVE-8065/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
(original)
+++
hive/branches/HIVE-8065/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Fri Nov 21 21:57:23 2014
@@ -1815,13 +1815,9 @@ public class SemanticAnalyzer extends Ba
} else {
// This is the only place where isQuery is set to true; it
defaults to false.
qb.setIsQuery(true);
- Path table_path = getStrongestEncryptedTablePath(qb);
- if (table_path == null) {
-fname = ctx.getMRTmpPath().toString();
- } else {
-fname = ctx.getMRTmpPath(table_path.toUri()).toString();
- }
- ctx.setResDir(new Path(fname));
+ Path stagingPath = getStagingDirectoryPathname(qb);
+ fname = stagingPath.toString();
+ ctx.setResDir(stagingPath);
}
}
qb.getMetaData().setDestForAlias(name, fname,
@@ -1878,6 +1874,81 @@ public class SemanticAnalyzer extends Ba
}
/**
+ * Checks if a given path is encrypted (valid only for HDFS files)
+ * @param path The path to check for encryption
+ * @return True if the path is encrypted; False if it is not encrypted
+ * @throws HiveException If an error occurs while checking for encryption
+ */
+ private boolean isPathEncrypted(Path path) throws HiveException {
+HadoopShims.HdfsEncryptionShim hdfsEncryptionShim;
+
+hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim();
+if (hdfsEncryptionShim != null) {
+ try {
+if (hdfsEncryptionShim.isPathEncrypted(path)) {
+ return true;
+}
+ } catch (Exception e) {
+throw new HiveException(Unable to determine if + path + is
encrypted: + e, e);
+ }
+}
+
+return false;
+ }
+
+ /**
+ * Compares to path key encryption strenghts.
+ *
+ * @param p1 Path to an HDFS file system
+ * @param p2 Path to an HDFS file system
+ * @return -1 if strength is weak; 0 if is equals; 1 if it is stronger
+ * @throws HiveException If an error occurs while comparing key strengths.
+ */
+ private int comparePathKeyStrength(Path p1, Path p2) throws HiveException {
+HadoopShims.HdfsEncryptionShim hdfsEncryptionShim;
+
+hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim();
+if (hdfsEncryptionShim != null) {
+ try {
+return hdfsEncryptionShim.comparePathKeyStrength(p1, p2);
+ } catch (Exception e) {
+throw new HiveException(Unable to compare key strength for + p1 +
and + p2 + : + e, e);
+ }
+}
+
+return 0; // Non-encrypted path (or equals strength)
+ }
+
+ /**
+ * Checks if a given path has read-only access permissions.
+ *
+ * @param path The path to check for read-only permissions.
+ * @return True if the path is read-only; False otherwise.
+ * @throws HiveException If an error occurs while checking file permissions.
+ */
+ private boolean isPathReadOnly(Path path) throws HiveException {
+HiveConf conf = SessionState.get().getConf();
+try {
+ FileSystem fs = path.getFileSystem(conf);
+ UserGroupInformation ugi =
ShimLoader.getHadoopShims().getUGIForConf(conf);
+ FileStatus status = fs.getFileStatus(path);
+
+ // We just check for writing permissions. If it fails with
AccessControException, then it
+ // means the location may be read-only.
+ FileUtils.checkFileAccessWithImpersonation(fs, status, FsAction.WRITE,
ugi.getUserName());
+
+ // Path has writing permissions
+ return false;
+} catch (AccessControlException e) {
+ // An AccessControlException may be caused for other different errors,
+ // but we take it as if our path is read-only
+ return true;
+} catch (Exception e) {
+ throw new HiveException(Unable to determine if + path + is read
only: + e, e);
+}
+ }
+
+ /**
* Gets the strongest encrypted table path.
*
* @param qb The QB object that contains a list of all table locations.
@@ -1887,7 +1958,6 @@ public class SemanticAnalyzer extends Ba
private Path getStrongestEncryptedTablePath(QB qb) throws HiveException {
ListString tabAliases = new ArrayListString(qb.getTabAliases());
svn commit: r1641009 - /hive/trunk/pom.xml
Author: cws Date: Fri Nov 21 22:13:55 2014 New Revision: 1641009 URL: http://svn.apache.org/r1641009 Log: HIVE-8933. Check release builds for SNAPSHOT dependencies Modified: hive/trunk/pom.xml Modified: hive/trunk/pom.xml URL: http://svn.apache.org/viewvc/hive/trunk/pom.xml?rev=1641009r1=1641008r2=1641009view=diff == --- hive/trunk/pom.xml (original) +++ hive/trunk/pom.xml Fri Nov 21 22:13:55 2014 @@ -776,6 +776,28 @@ /plugin plugin groupIdorg.apache.maven.plugins/groupId +artifactIdmaven-enforcer-plugin/artifactId +executions + execution +idenforce-no-snapshots/id +goals + goalenforce/goal +/goals +configuration + rules +requireReleaseDeps + messageRelease builds are not allowed to have SNAPSHOT depenendencies/message + searchTransitivetrue/searchTransitive + onlyWhenReleasetrue/onlyWhenRelease +/requireReleaseDeps + /rules + failtrue/fail +/configuration + /execution +/executions + /plugin + plugin +groupIdorg.apache.maven.plugins/groupId artifactIdmaven-surefire-plugin/artifactId configuration excludes
svn commit: r1641011 - /hive/branches/branch-0.14/pom.xml
Author: cws Date: Fri Nov 21 22:30:42 2014 New Revision: 1641011 URL: http://svn.apache.org/r1641011 Log: HIVE-8933. Check release builds for SNAPSHOT dependencies Modified: hive/branches/branch-0.14/pom.xml Modified: hive/branches/branch-0.14/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/pom.xml?rev=1641011r1=1641010r2=1641011view=diff == --- hive/branches/branch-0.14/pom.xml (original) +++ hive/branches/branch-0.14/pom.xml Fri Nov 21 22:30:42 2014 @@ -772,6 +772,28 @@ /plugin plugin groupIdorg.apache.maven.plugins/groupId +artifactIdmaven-enforcer-plugin/artifactId +executions + execution +idenforce-no-snapshots/id +goals + goalenforce/goal +/goals +configuration + rules +requireReleaseDeps + messageRelease builds are not allowed to have SNAPSHOT depenendencies/message + searchTransitivetrue/searchTransitive + onlyWhenReleasetrue/onlyWhenRelease +/requireReleaseDeps + /rules + failtrue/fail +/configuration + /execution +/executions + /plugin + plugin +groupIdorg.apache.maven.plugins/groupId artifactIdmaven-surefire-plugin/artifactId configuration excludes
svn commit: r1641023 - /hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java
Author: szehon
Date: Sat Nov 22 01:40:25 2014
New Revision: 1641023
URL: http://svn.apache.org/r1641023
Log:
HIVE-8942 : Investigate test failure on skewjoin.q [Spark Branch] (Chao Sun via
Szehon)
Modified:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java
Modified:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java?rev=1641023r1=1641022r2=1641023view=diff
==
---
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java
(original)
+++
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java
Sat Nov 22 01:40:25 2014
@@ -276,6 +276,10 @@ public class SparkReduceSinkMapJoinProc
MapJoinDesc mjDesc = mapJoinOp.getConf();
HiveConf conf = context.conf;
+// Unlike in MR, we may call this method multiple times, for each
+// small table HTS. But, since it's idempotent, it should be OK.
+mjDesc.resetOrder();
+
float hashtableMemoryUsage;
if (hasGroupBy(mapJoinOp, context)) {
hashtableMemoryUsage = conf.getFloatVar(
svn commit: r1641024 - in /hive/trunk/metastore/scripts/upgrade/postgres: 019-HIVE-7784.postgres.sql pre-0-upgrade-0.13.0-to-0.14.0.postgres.sql
Author: sershe Date: Sat Nov 22 01:40:34 2014 New Revision: 1641024 URL: http://svn.apache.org/r1641024 Log: HIVE-8947 : HIVE-8876 also affects Postgres 9.2 (Sergey Shelukhin, reviewed by Ashutosh Chauhan) Added: hive/trunk/metastore/scripts/upgrade/postgres/pre-0-upgrade-0.13.0-to-0.14.0.postgres.sql Modified: hive/trunk/metastore/scripts/upgrade/postgres/019-HIVE-7784.postgres.sql Modified: hive/trunk/metastore/scripts/upgrade/postgres/019-HIVE-7784.postgres.sql URL: http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/postgres/019-HIVE-7784.postgres.sql?rev=1641024r1=1641023r2=1641024view=diff == --- hive/trunk/metastore/scripts/upgrade/postgres/019-HIVE-7784.postgres.sql (original) +++ hive/trunk/metastore/scripts/upgrade/postgres/019-HIVE-7784.postgres.sql Sat Nov 22 01:40:34 2014 @@ -1,29 +1 @@ --- --- Create the table if it doesn't exist. --- - -CREATE TABLE IF NOT EXISTS PART_COL_STATS ( - CS_ID bigint NOT NULL, - DB_NAME character varying(128) DEFAULT NULL::character varying, - TABLE_NAME character varying(128) DEFAULT NULL::character varying, - PARTITION_NAME character varying(767) DEFAULT NULL::character varying, - COLUMN_NAME character varying(128) DEFAULT NULL::character varying, - COLUMN_TYPE character varying(128) DEFAULT NULL::character varying, - PART_ID bigint NOT NULL, - LONG_LOW_VALUE bigint, - LONG_HIGH_VALUE bigint, - DOUBLE_LOW_VALUE double precision, - DOUBLE_HIGH_VALUE double precision, - BIG_DECIMAL_LOW_VALUE character varying(4000) DEFAULT NULL::character varying, - BIG_DECIMAL_HIGH_VALUE character varying(4000) DEFAULT NULL::character varying, - NUM_NULLS bigint NOT NULL, - NUM_DISTINCTS bigint, - AVG_COL_LEN double precision, - MAX_COL_LEN bigint, - NUM_TRUES bigint, - NUM_FALSES bigint, - LAST_ANALYZED bigint NOT NULL -); - - CREATE INDEX PCS_STATS_IDX ON PART_COL_STATS USING btree (DB_NAME,TABLE_NAME,COLUMN_NAME,PARTITION_NAME); Added: hive/trunk/metastore/scripts/upgrade/postgres/pre-0-upgrade-0.13.0-to-0.14.0.postgres.sql URL: http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/postgres/pre-0-upgrade-0.13.0-to-0.14.0.postgres.sql?rev=1641024view=auto == --- hive/trunk/metastore/scripts/upgrade/postgres/pre-0-upgrade-0.13.0-to-0.14.0.postgres.sql (added) +++ hive/trunk/metastore/scripts/upgrade/postgres/pre-0-upgrade-0.13.0-to-0.14.0.postgres.sql Sat Nov 22 01:40:34 2014 @@ -0,0 +1,23 @@ + +CREATE TABLE PART_COL_STATS ( + CS_ID bigint NOT NULL, + DB_NAME character varying(128) DEFAULT NULL::character varying, + TABLE_NAME character varying(128) DEFAULT NULL::character varying, + PARTITION_NAME character varying(767) DEFAULT NULL::character varying, + COLUMN_NAME character varying(128) DEFAULT NULL::character varying, + COLUMN_TYPE character varying(128) DEFAULT NULL::character varying, + PART_ID bigint NOT NULL, + LONG_LOW_VALUE bigint, + LONG_HIGH_VALUE bigint, + DOUBLE_LOW_VALUE double precision, + DOUBLE_HIGH_VALUE double precision, + BIG_DECIMAL_LOW_VALUE character varying(4000) DEFAULT NULL::character varying, + BIG_DECIMAL_HIGH_VALUE character varying(4000) DEFAULT NULL::character varying, + NUM_NULLS bigint NOT NULL, + NUM_DISTINCTS bigint, + AVG_COL_LEN double precision, + MAX_COL_LEN bigint, + NUM_TRUES bigint, + NUM_FALSES bigint, + LAST_ANALYZED bigint NOT NULL +);
