svn commit: r1671011 - /hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveJoin.java
Author: jcamacho
Date: Fri Apr 3 08:03:28 2015
New Revision: 1671011
URL: http://svn.apache.org/r1671011
Log:
HIVE-10195 : CBO (Calcite Return Path): More details for Join operator in CBO
cost based algorithm selection [CBO branch] (Jesus Camacho Rodriguez, reviewed
by Ashutosh Chauhan)
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveJoin.java
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveJoin.java
URL:
http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveJoin.java?rev=1671011r1=1671010r2=1671011view=diff
==
---
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveJoin.java
(original)
+++
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveJoin.java
Fri Apr 3 08:03:28 2015
@@ -26,6 +26,7 @@ import java.util.Set;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptCost;
import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.InvalidRelException;
import org.apache.calcite.rel.RelCollations;
@@ -156,7 +157,7 @@ public class HiveJoin extends Join imple
// 4. For each possible algorithm, calculate cost, and select best
RelOptCost selfCost = null;
if (LOG.isDebugEnabled()) {
- LOG.debug(Join algorithm selection for: + this);
+ LOG.debug(Join algorithm selection for:\n + RelOptUtil.toString(this));
}
for (JoinAlgorithm possibleAlgorithm : possibleAlgorithms) {
switch (possibleAlgorithm) {
svn commit: r1671087 - in /hive/trunk: metastore/src/java/org/apache/hadoop/hive/metastore/ ql/src/java/org/apache/hadoop/hive/ql/metadata/ ql/src/test/org/apache/hadoop/hive/ql/metadata/ ql/src/test/
Author: jxiang
Date: Fri Apr 3 15:20:37 2015
New Revision: 1671087
URL: http://svn.apache.org/r1671087
Log:
HIVE-10108: Index#getIndexTableName() returns db.index_table_name
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
hive/trunk/ql/src/test/results/clientnegative/alter_concatenate_indexed_table.q.out
hive/trunk/ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out
hive/trunk/ql/src/test/results/clientpositive/index_auth.q.out
hive/trunk/ql/src/test/results/clientpositive/index_auto.q.out
hive/trunk/ql/src/test/results/clientpositive/index_auto_file_format.q.out
hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out
hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out
hive/trunk/ql/src/test/results/clientpositive/index_auto_multiple.q.out
hive/trunk/ql/src/test/results/clientpositive/index_auto_partitioned.q.out
hive/trunk/ql/src/test/results/clientpositive/index_auto_self_join.q.out
hive/trunk/ql/src/test/results/clientpositive/index_auto_update.q.out
hive/trunk/ql/src/test/results/clientpositive/index_bitmap_auto_partitioned.q.out
hive/trunk/ql/src/test/results/clientpositive/index_bitmap_compression.q.out
hive/trunk/ql/src/test/results/clientpositive/index_compression.q.out
hive/trunk/ql/src/test/results/clientpositive/index_serde.q.out
hive/trunk/ql/src/test/results/clientpositive/index_skewtable.q.out
hive/trunk/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx.q.out
hive/trunk/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx_cbo_1.q.out
hive/trunk/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx_cbo_2.q.out
hive/trunk/ql/src/test/results/clientpositive/show_indexes_edge_cases.q.out
hive/trunk/ql/src/test/results/clientpositive/show_indexes_syntax.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/ql_rewrite_gbtoidx.q.out
hive/trunk/ql/src/test/results/clientpositive/spark/ql_rewrite_gbtoidx_cbo_1.q.out
hive/trunk/ql/src/test/results/clientpositive/union_view.q.out
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1671087r1=1671086r2=1671087view=diff
==
---
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
(original)
+++
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
Fri Apr 3 15:20:37 2015
@@ -3121,10 +3121,6 @@ public class ObjectStore implements RawS
MTable origTable = mIndex.getOrigTable();
MTable indexTable = mIndex.getIndexTable();
-String[] qualified = MetaStoreUtils.getQualifiedName(
-origTable.getDatabase().getName(), indexTable.getTableName());
-String indexTableName = qualified[0] + . + qualified[1];
-
return new Index(
mIndex.getIndexName(),
mIndex.getIndexHandlerClass(),
@@ -3132,7 +3128,7 @@ public class ObjectStore implements RawS
origTable.getTableName(),
mIndex.getCreateTime(),
mIndex.getLastAccessTime(),
-indexTableName,
+indexTable.getTableName(),
convertToStorageDescriptor(mIndex.getSd()),
mIndex.getParameters(),
mIndex.getDeferredRebuild());
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1671087r1=1671086r2=1671087view=diff
==
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Fri Apr
3 15:20:37 2015
@@ -132,6 +132,7 @@ import com.google.common.collect.Sets;
* get methods in this class.
*/
+@SuppressWarnings({deprecation, rawtypes})
public class Hive {
static final private Log LOG = LogFactory.getLog(hive.ql.metadata.Hive);
@@ -759,6 +760,13 @@ public class Hive {
throws HiveException {
try {
+ String tdname = Utilities.getDatabaseName(tableName);
+ String idname = Utilities.getDatabaseName(indexTblName);
+ if (!idname.equals(tdname)) {
+throw new HiveException(Index on different database ( + idname
+ + ) from base table ( + tdname + ) is not supported.);
+ }
+
Index old_index = null;
try {
old_index = getIndex(tableName, indexName);
@@ -849,9 +857,8 @@ public class Hive {
org.apache.hadoop.hive.metastore.api.Table tt = null;
HiveIndexHandler indexHandler =
HiveUtils.getIndexHandler(this.getConf(),
svn commit: r1671110 - /hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
Author: ekoifman Date: Fri Apr 3 17:46:03 2015 New Revision: 1671110 URL: http://svn.apache.org/r1671110 Log: HIVE-10208 templeton.hive.extra.files should be commented out in webhcat-default.xml Modified: hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml Modified: hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml?rev=1671110r1=1671109r2=1671110view=diff == --- hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml (original) +++ hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml Fri Apr 3 17:46:03 2015 @@ -111,11 +111,12 @@ property nametempleton.hive.extra.files/name -value/tez-client/conf/tez-site.xml,/tez-client/,/tez-client/lib/value +value/value descriptionThe resources in this list will be localized to the node running LaunchMapper and added to HADOOP_CLASSPTH before launching 'hive' command. If the path /foo/bar is a directory, the contents of the the entire dir will be localized and ./bar/* will be added to HADOOP_CLASSPATH. Note that since classpath path processing does not recurse into subdirectories, - the paths in this property may be overlapping. In the example above, ./tez-site.xml:./tez-client/*:./lib/* will be added to + the paths in this property may be overlapping. For example, to run Hive on Tez jobs, 3 items need to be localized: + /tez-client/conf/tez-site.xml,/tez-client/,/tez-client/lib. In this example, ./tez-site.xml:./tez-client/*:./lib/* will be added to HADOOP_CLASSPATH. This can be used to specify config files, Tez artifacts, etc. This will be sent -files option of hadoop jar command thus each path is interpreted by Generic Option Parser. It can be local or hdfs path.
svn commit: r1671121 - /hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
Author: szehon
Date: Fri Apr 3 18:30:47 2015
New Revision: 1671121
URL: http://svn.apache.org/r1671121
Log:
HIVE-8818 : Create unit test where we insert into an encrypted table and then
read from it with hcatalog mapreduce (Dong Chen via Szehon)
Modified:
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
Modified:
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java?rev=1671121r1=1671120r2=1671121view=diff
==
---
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
(original)
+++
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
Fri Apr 3 18:30:47 2015
@@ -21,20 +21,25 @@ package org.apache.hive.hcatalog.pig;
import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.io.IOConstants;
@@ -47,10 +52,18 @@ import org.apache.hadoop.hive.ql.session
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hive.hcatalog.HcatTestUtils;
import org.apache.hive.hcatalog.common.HCatUtil;
+import org.apache.hive.hcatalog.data.HCatRecord;
import org.apache.hive.hcatalog.data.Pair;
+import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.data.Tuple;
@@ -87,6 +100,7 @@ public class TestHCatLoaderEncryption {
private boolean isEncryptionTestEnabled = true;
private Driver driver;
private MapInteger, PairInteger, String basicInputData;
+ private static ListHCatRecord readRecords = new ArrayListHCatRecord();
private static final MapString, SetString DISABLED_STORAGE_FORMATS =
new HashMapString, SetString() {{
@@ -300,7 +314,7 @@ public class TestHCatLoaderEncryption {
}
@Test
- public void testReadDataFromEncryptedHiveTable() throws IOException {
+ public void testReadDataFromEncryptedHiveTableByPig() throws IOException {
assumeTrue(isEncryptionTestEnabled);
assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS));
PigServer server = new PigServer(ExecType.LOCAL);
@@ -322,6 +336,67 @@ public class TestHCatLoaderEncryption {
assertEquals(failed with storage format: + this.storageFormat,
basicInputData.size(), numTuplesRead);
}
+ @Test
+ public void testReadDataFromEncryptedHiveTableByHCatMR() throws Exception {
+assumeTrue(isEncryptionTestEnabled);
+assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS));
+
+readRecords.clear();
+Configuration conf = new Configuration();
+Job job = new Job(conf, hcat mapreduce read encryption test);
+job.setJarByClass(this.getClass());
+job.setMapperClass(TestHCatLoaderEncryption.MapRead.class);
+
+// input/output settings
+job.setInputFormatClass(HCatInputFormat.class);
+job.setOutputFormatClass(TextOutputFormat.class);
+
+HCatInputFormat.setInput(job, MetaStoreUtils.DEFAULT_DATABASE_NAME,
ENCRYPTED_TABLE, null);
+
+job.setMapOutputKeyClass(BytesWritable.class);
+job.setMapOutputValueClass(Text.class);
+
+job.setNumReduceTasks(0);
+
+FileSystem fs = new LocalFileSystem();
+Path path = new Path(TEST_DATA_DIR + /testHCatMREncryptionOutput);
+if (fs.exists(path)) {
+ fs.delete(path, true);
+}
+
+TextOutputFormat.setOutputPath(job, path);
+
+job.waitForCompletion(true);
+
+int numTuplesRead = 0;
+for (HCatRecord hCatRecord : readRecords) {
+ assertEquals(2, hCatRecord.size());
+ assertNotNull(hCatRecord.get(0));
+
