Modified: hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java (original) +++ hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java Fri Nov 7 20:41:34 2014 @@ -480,7 +480,9 @@ class PigHCatUtil { Map<String, Object> result = new HashMap<String, Object>(); for (Entry<?, ?> entry : map.entrySet()) { // since map key for Pig has to be Strings - result.put(entry.getKey().toString(), extractPigObject(entry.getValue(), hfs.getMapValueSchema().get(0))); + if (entry.getKey()!=null) { + result.put(entry.getKey().toString(), extractPigObject(entry.getValue(), hfs.getMapValueSchema().get(0))); + } } return result; }
Modified: hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java (original) +++ hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java Fri Nov 7 20:41:34 2014 @@ -101,12 +101,6 @@ public class TestHCatLoader { private static final Map<String, Set<String>> DISABLED_STORAGE_FORMATS = new HashMap<String, Set<String>>() {{ - put(IOConstants.AVRO, new HashSet<String>() {{ - add("testReadDataBasic"); - add("testReadPartitionedBasic"); - add("testProjectionsBasic"); - add("testSchemaLoadPrimitiveTypes"); - }}); put(IOConstants.PARQUETFILE, new HashSet<String>() {{ add("testReadDataBasic"); add("testReadPartitionedBasic"); Modified: hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java (original) +++ hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java Fri Nov 7 20:41:34 2014 @@ -18,8 +18,6 @@ */ package org.apache.hive.hcatalog.pig; -import com.google.common.collect.ImmutableSet; - import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -52,7 +50,6 @@ import org.apache.pig.impl.logicalLayer. import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; @@ -76,14 +73,13 @@ public class TestHCatLoaderComplexSchema private static final Map<String, Set<String>> DISABLED_STORAGE_FORMATS = new HashMap<String, Set<String>>() {{ put(IOConstants.AVRO, new HashSet<String>() {{ - add("testSyntheticComplexSchema"); - add("testTupleInBagInTupleInBag"); - add("testMapWithComplexData"); + add("testMapNullKey"); }}); put(IOConstants.PARQUETFILE, new HashSet<String>() {{ add("testSyntheticComplexSchema"); add("testTupleInBagInTupleInBag"); add("testMapWithComplexData"); + add("testMapNullKey"); }}); }}; @@ -223,6 +219,10 @@ public class TestHCatLoaderComplexSchema private void verifyWriteRead(String tablename, String pigSchema, String tableSchema, List<Tuple> data, boolean provideSchemaToStorer) throws IOException, CommandNeedRetryException, ExecException, FrontendException { + verifyWriteRead(tablename, pigSchema, tableSchema, data, data, provideSchemaToStorer); + } + private void verifyWriteRead(String tablename, String pigSchema, String tableSchema, List<Tuple> data, List<Tuple> result, boolean provideSchemaToStorer) + throws IOException, CommandNeedRetryException, ExecException, FrontendException { MockLoader.setData(tablename + "Input", data); try { createTable(tablename, tableSchema); @@ -244,7 +244,7 @@ public class TestHCatLoaderComplexSchema Iterator<Tuple> it = server.openIterator("X"); int i = 0; while (it.hasNext()) { - Tuple input = data.get(i++); + Tuple input = result.get(i++); Tuple output = it.next(); compareTuples(input, output); LOG.info("tuple : {} ", output); @@ -354,4 +354,40 @@ public class TestHCatLoaderComplexSchema verifyWriteRead("testMapWithComplexData", pigSchema, tableSchema, data, true); verifyWriteRead("testMapWithComplexData2", pigSchema, tableSchema, data, false); } + + /** + * artificially complex nested schema to test nested schema conversion + * @throws Exception + */ + @Test + public void testMapNullKey() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); + String pigSchema = "m:map[]"; + + String tableSchema = "m map<string, string>"; + + List<Tuple> data = new ArrayList<Tuple>(); + Tuple t = t( + new HashMap<String, String>() { + { + put("ac test1", "test 1"); + put("ac test2", "test 2"); + put(null, "test 3"); + }; + }); + data.add(t); + + List<Tuple> result = new ArrayList<Tuple>(); + t = t( + new HashMap<String, String>() { + { + put("ac test1", "test 1"); + put("ac test2", "test 2"); + }; + }); + result.add(t); + + verifyWriteRead("testSyntheticComplexSchema", pigSchema, tableSchema, data, result, true); + verifyWriteRead("testSyntheticComplexSchema", pigSchema, tableSchema, data, result, false); + } } Modified: hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java (original) +++ hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java Fri Nov 7 20:41:34 2014 @@ -64,7 +64,7 @@ public class TestHCatLoaderStorer extend TestHCatLoader.executeStatementOnDriver("create external table " + tblName + " (my_small_int smallint, my_tiny_int tinyint)" + " row format delimited fields terminated by '\t' stored as textfile location '" + - dataDir + "'", driver); + dataDir.toURI().getPath() + "'", driver); TestHCatLoader.dropTable(tblName2, driver); TestHCatLoader.createTable(tblName2, "my_small_int smallint, my_tiny_int tinyint", null, driver, "textfile"); Modified: hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java (original) +++ hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java Fri Nov 7 20:41:34 2014 @@ -76,29 +76,16 @@ public class TestHCatStorer extends HCat private static final Map<String, Set<String>> DISABLED_STORAGE_FORMATS = new HashMap<String, Set<String>>() {{ put(IOConstants.AVRO, new HashSet<String>() {{ - add("testBagNStruct"); - add("testDateCharTypes"); - add("testDynamicPartitioningMultiPartColsInDataNoSpec"); - add("testDynamicPartitioningMultiPartColsInDataPartialSpec"); - add("testMultiPartColsInData"); - add("testPartColsInData"); - add("testStoreFuncAllSimpleTypes"); - add("testStoreFuncSimple"); - add("testStoreInPartiitonedTbl"); - add("testStoreMultiTables"); - add("testStoreWithNoCtorArgs"); - add("testStoreWithNoSchema"); - add("testWriteChar"); - add("testWriteDate"); - add("testWriteDate2"); - add("testWriteDate3"); - add("testWriteDecimal"); - add("testWriteDecimalX"); - add("testWriteDecimalXY"); - add("testWriteSmallint"); - add("testWriteTimestamp"); - add("testWriteTinyint"); - add("testWriteVarchar"); + add("testDateCharTypes"); // incorrect precision + // expected:<0 xxxxx yyy 5.2[]> but was:<0 xxxxx yyy 5.2[0]> + add("testWriteDecimalXY"); // incorrect precision + // expected:<1.2[]> but was:<1.2[0]> + add("testWriteSmallint"); // doesn't have a notion of small, and saves the full value as an int, so no overflow + // expected:<null> but was:<32768> + add("testWriteTimestamp"); // does not support timestamp + // TypeInfoToSchema.createAvroPrimitive : UnsupportedOperationException + add("testWriteTinyint"); // doesn't have a notion of tiny, and saves the full value as an int, so no overflow + // expected:<null> but was:<300> }}); put(IOConstants.PARQUETFILE, new HashSet<String>() {{ add("testBagNStruct"); Modified: hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java (original) +++ hive/branches/spark/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java Fri Nov 7 20:41:34 2014 @@ -70,11 +70,6 @@ public class TestHCatStorerMulti { private static final Map<String, Set<String>> DISABLED_STORAGE_FORMATS = new HashMap<String, Set<String>>() {{ - put(IOConstants.AVRO, new HashSet<String>() {{ - add("testStoreBasicTable"); - add("testStorePartitionedTable"); - add("testStoreTableMulti"); - }}); put(IOConstants.PARQUETFILE, new HashSet<String>() {{ add("testStoreBasicTable"); add("testStorePartitionedTable"); Modified: hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java (original) +++ hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java Fri Nov 7 20:41:34 2014 @@ -19,10 +19,13 @@ package org.apache.hive.hcatalog.messaging; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hive.hcatalog.messaging.json.JSONMessageFactory; @@ -131,6 +134,16 @@ public abstract class MessageFactory { public abstract AddPartitionMessage buildAddPartitionMessage(Table table, List<Partition> partitions); /** + * Factory method for AddPartitionMessage. + * @param table The Table to which the partitions are added. + * @param partitionSpec The set of Partitions being added. + * @return AddPartitionMessage instance. + */ + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + public abstract AddPartitionMessage buildAddPartitionMessage(Table table, PartitionSpecProxy partitionSpec); + + /** * Factory method for DropPartitionMessage. * @param table The Table from which the partition is dropped. * @param partition The Partition being dropped. Modified: hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java (original) +++ hive/branches/spark/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java Fri Nov 7 20:41:34 2014 @@ -19,9 +19,12 @@ package org.apache.hive.hcatalog.messaging.json; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hive.hcatalog.messaging.AddPartitionMessage; import org.apache.hive.hcatalog.messaging.CreateDatabaseMessage; import org.apache.hive.hcatalog.messaging.CreateTableMessage; @@ -87,6 +90,14 @@ public class JSONMessageFactory extends } @Override + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + public AddPartitionMessage buildAddPartitionMessage(Table table, PartitionSpecProxy partitionSpec) { + return new JSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), + table.getTableName(), getPartitionKeyValues(table, partitionSpec), System.currentTimeMillis()/1000); + } + + @Override public DropPartitionMessage buildDropPartitionMessage(Table table, Partition partition) { return new JSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, partition.getDbName(), partition.getTableName(), Arrays.asList(getPartitionKeyValues(table, partition)), @@ -107,4 +118,16 @@ public class JSONMessageFactory extends partitionList.add(getPartitionKeyValues(table, partition)); return partitionList; } + + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + private static List<Map<String, String>> getPartitionKeyValues(Table table, PartitionSpecProxy partitionSpec) { + List<Map<String, String>> partitionList = new ArrayList<Map<String, String>>(); + PartitionSpecProxy.PartitionIterator iterator = partitionSpec.getPartitionIterator(); + while (iterator.hasNext()) { + Partition partition = iterator.next(); + partitionList.add(getPartitionKeyValues(table, partition)); + } + return partitionList; + } } Modified: hive/branches/spark/hcatalog/webhcat/java-client/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/java-client/pom.xml?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/hcatalog/webhcat/java-client/pom.xml (original) +++ hive/branches/spark/hcatalog/webhcat/java-client/pom.xml Fri Nov 7 20:41:34 2014 @@ -47,6 +47,13 @@ </dependency> <!-- test intra-project --> <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.hive.hcatalog</groupId> <artifactId>hive-hcatalog-core</artifactId> <version>${project.version}</version> Modified: hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClient.java URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClient.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClient.java (original) +++ hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClient.java Fri Nov 7 20:41:34 2014 @@ -22,6 +22,8 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.metastore.api.PartitionEventType; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hive.hcatalog.common.HCatException; @@ -213,6 +215,26 @@ public abstract class HCatClient { public abstract List<HCatPartition> deserializePartitions(List<String> hcatPartitionStringReps) throws HCatException; /** + * Serializer for HCatPartitionSpec. + * @param partitionSpec HCatPartitionSpec to be serialized. + * @return A list of Strings, representing the HCatPartitionSpec as a whole. + * @throws HCatException On failure to serialize. + */ + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + public abstract List<String> serializePartitionSpec(HCatPartitionSpec partitionSpec) throws HCatException; + + /** + * Deserializer for HCatPartitionSpec. + * @param hcatPartitionSpecStrings List of strings, representing the HCatPartitionSpec as a whole. + * @return HCatPartitionSpec, reconstructed from the list of strings. + * @throws HCatException On failure to deserialize. + */ + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + public abstract HCatPartitionSpec deserializePartitionSpec(List<String> hcatPartitionSpecStrings) throws HCatException; + + /** * Creates the table like an existing table. * * @param dbName The name of the database. @@ -280,6 +302,21 @@ public abstract class HCatClient { throws HCatException; /** + * Gets partitions in terms of generic HCatPartitionSpec instances. + */ + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + public abstract HCatPartitionSpec getPartitionSpecs(String dbName, String tableName, int maxPartitions) throws HCatException; + + /** + * Gets partitions in terms of generic HCatPartitionSpec instances. + */ + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + public abstract HCatPartitionSpec getPartitionSpecs(String dbName, String tableName, Map<String, String> partitionSelector, int maxPartitions) + throws HCatException; + + /** * Gets the partition. * * @param dbName The database name. @@ -312,6 +349,17 @@ public abstract class HCatClient { throws HCatException; /** + * Adds partitions using HCatPartitionSpec. + * @param partitionSpec The HCatPartitionSpec representing the set of partitions added. + * @return The number of partitions added. + * @throws HCatException On failure to add partitions. + */ + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + public abstract int addPartitionSpec(HCatPartitionSpec partitionSpec) + throws HCatException; + + /** * Drops partition(s) that match the specified (and possibly partial) partition specification. * A partial partition-specification is one where not all partition-keys have associated values. For example, * for a table ('myDb.myTable') with 2 partition keys (dt string, region string), @@ -344,6 +392,14 @@ public abstract class HCatClient { String filter) throws HCatException; /** + * List partitions by filter, but as HCatPartitionSpecs. + */ + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + public abstract HCatPartitionSpec listPartitionSpecsByFilter(String dbName, String tblName, + String filter, int maxPartitions) throws HCatException; + + /** * Mark partition for event. * * @param dbName The database name. Modified: hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java (original) +++ hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java Fri Nov 7 20:41:34 2014 @@ -25,6 +25,8 @@ import java.util.Map; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.MetaStoreUtils; @@ -353,6 +355,31 @@ public class HCatClientHMSImpl extends H return listPartitionsByFilter(dbName, tblName, getFilterString(partitionSpec)); } + @Override + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + public HCatPartitionSpec getPartitionSpecs(String dbName, String tableName, int maxPartitions) throws HCatException { + try { + return new HCatPartitionSpec(getTable(dbName, tableName), + hmsClient.listPartitionSpecs(dbName, tableName, maxPartitions)); + } + catch (NoSuchObjectException e) { + throw new ObjectNotFoundException( + "NoSuchObjectException while retrieving partition.", e); + } catch (MetaException e) { + throw new HCatException( + "MetaException while retrieving partition.", e); + } catch (TException e) { + throw new ConnectionFailureException( + "TException while retrieving partition.", e); + } + } + + @Override + public HCatPartitionSpec getPartitionSpecs(String dbName, String tableName, Map<String, String> partitionSelector, int maxPartitions) throws HCatException { + return listPartitionSpecsByFilter(dbName, tableName, getFilterString(partitionSelector), maxPartitions); + } + private static String getFilterString(Map<String, String> partitionSpec) { final String AND = " AND "; @@ -413,7 +440,7 @@ public class HCatClientHMSImpl extends H Table tbl = null; try { tbl = hmsClient.getTable(partInfo.getDatabaseName(), - partInfo.getTableName()); + partInfo.getTableName()); // TODO: Should be moved out. if (tbl.getPartitionKeysSize() == 0) { throw new HCatException("The table " + partInfo.getTableName() @@ -511,6 +538,28 @@ public class HCatClientHMSImpl extends H } @Override + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + public HCatPartitionSpec listPartitionSpecsByFilter(String dbName, String tblName, String filter, int maxPartitions) + throws HCatException { + try { + return new HCatPartitionSpec(getTable(dbName, tblName), + hmsClient.listPartitionSpecsByFilter(dbName, tblName, filter, maxPartitions)); + } + catch(MetaException e) { + throw new HCatException("MetaException while fetching partitions.", e); + } + catch (NoSuchObjectException e) { + throw new ObjectNotFoundException( + "NoSuchObjectException while fetching partitions.", e); + } + catch (TException e) { + throw new ConnectionFailureException( + "TException while fetching partitions.", e); + } + } + + @Override public void markPartitionForEvent(String dbName, String tblName, Map<String, String> partKVs, PartitionEventType eventType) throws HCatException { @@ -572,7 +621,7 @@ public class HCatClientHMSImpl extends H String token = null; try { token = hmsClient.getDelegationToken(owner, - renewerKerberosPrincipalName); + renewerKerberosPrincipalName); } catch (MetaException e) { throw new HCatException( "MetaException while getting delegation token.", e); @@ -750,6 +799,30 @@ public class HCatClientHMSImpl extends H } @Override + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + public int addPartitionSpec(HCatPartitionSpec partitionSpec) throws HCatException { + + try { + return hmsClient.add_partitions_pspec(partitionSpec.toPartitionSpecProxy()); + } catch (InvalidObjectException e) { + throw new HCatException( + "InvalidObjectException while adding partition.", e); + } catch (AlreadyExistsException e) { + throw new HCatException( + "AlreadyExistsException while adding partition.", e); + } catch (MetaException e) { + throw new HCatException("MetaException while adding partition.", e); + } catch (NoSuchObjectException e) { + throw new ObjectNotFoundException("The table " + + "could not be found.", e); + } catch (TException e) { + throw new ConnectionFailureException( + "TException while adding partition.", e); + } + } + + @Override public String getMessageBusTopicName(String dbName, String tableName) throws HCatException { try { return hmsClient.getTable(dbName, tableName).getParameters().get(HCatConstants.HCAT_MSGBUS_TOPIC_NAME); @@ -824,4 +897,16 @@ public class HCatClientHMSImpl extends H } return partitions; } + + @Override + public List<String> serializePartitionSpec(HCatPartitionSpec partitionSpec) throws HCatException { + return MetadataSerializer.get().serializePartitionSpec(partitionSpec); + } + + @Override + public HCatPartitionSpec deserializePartitionSpec(List<String> hcatPartitionSpecStrings) throws HCatException { + HCatPartitionSpec hcatPartitionSpec = MetadataSerializer.get().deserializePartitionSpec(hcatPartitionSpecStrings); + hcatPartitionSpec.hcatTable(getTable(hcatPartitionSpec.getDbName(), hcatPartitionSpec.getTableName())); + return hcatPartitionSpec; + } } Modified: hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataJSONSerializer.java URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataJSONSerializer.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataJSONSerializer.java (original) +++ hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataJSONSerializer.java Fri Nov 7 20:41:34 2014 @@ -1,7 +1,11 @@ package org.apache.hive.hcatalog.api; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PartitionSpec; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hive.hcatalog.common.HCatException; import org.apache.thrift.TDeserializer; import org.apache.thrift.TException; @@ -10,6 +14,9 @@ import org.apache.thrift.protocol.TJSONP import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.List; + /** * MetadataSerializer implementation, that serializes HCat API elements into JSON. */ @@ -68,4 +75,38 @@ class MetadataJSONSerializer extends Met throw new HCatException("Could not de-serialize HCatPartition.", exception); } } + + @Override + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + public List<String> serializePartitionSpec(HCatPartitionSpec hcatPartitionSpec) throws HCatException { + try { + List<String> stringReps = new ArrayList<String>(); + TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); + for (PartitionSpec partitionSpec : hcatPartitionSpec.partitionSpecProxy.toPartitionSpec()) { + stringReps.add(serializer.toString(partitionSpec, "UTF-8")); + } + return stringReps; + } + catch (TException serializationException) { + throw new HCatException("Failed to serialize!", serializationException); + } + } + + @Override + public HCatPartitionSpec deserializePartitionSpec(List<String> hcatPartitionSpecStrings) throws HCatException { + try { + List<PartitionSpec> partitionSpecList = new ArrayList<PartitionSpec>(); + TDeserializer deserializer = new TDeserializer(new TJSONProtocol.Factory()); + for (String stringRep : hcatPartitionSpecStrings) { + PartitionSpec partSpec = new PartitionSpec(); + deserializer.deserialize(partSpec, stringRep, "UTF-8"); + partitionSpecList.add(partSpec); + } + return new HCatPartitionSpec(null, PartitionSpecProxy.Factory.get(partitionSpecList)); + } + catch (TException deserializationException) { + throw new HCatException("Failed to deserialize!", deserializationException); + } + } } Modified: hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataSerializer.java URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataSerializer.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataSerializer.java (original) +++ hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataSerializer.java Fri Nov 7 20:41:34 2014 @@ -1,7 +1,11 @@ package org.apache.hive.hcatalog.api; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hive.hcatalog.common.HCatException; +import java.util.List; + /** * Interface to serialize HCat API elements. */ @@ -51,4 +55,24 @@ abstract class MetadataSerializer { */ public abstract HCatPartition deserializePartition(String hcatPartitionStringRep) throws HCatException; + /** + * Serializer for HCatPartitionSpec. + * @param hcatPartitionSpec HCatPartitionSpec instance to be serialized. + * @return Serialized string-representations. + * @throws HCatException On failure to serialize. + */ + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + public abstract List<String> serializePartitionSpec(HCatPartitionSpec hcatPartitionSpec) throws HCatException; + + /** + * Deserializer for HCatPartitionSpec string-representations. + * @param hcatPartitionSpecStrings List of strings to be converted into an HCatPartitionSpec. + * @return Deserialized HCatPartitionSpec instance. + * @throws HCatException On failure to deserialize. (e.g. incompatible serialization format, etc.) + */ + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + public abstract HCatPartitionSpec deserializePartitionSpec(List<String> hcatPartitionSpecStrings) throws HCatException; + } Modified: hive/branches/spark/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java (original) +++ hive/branches/spark/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java Fri Nov 7 20:41:34 2014 @@ -31,6 +31,7 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStore; import org.apache.hadoop.hive.metastore.api.PartitionEventType; +import org.apache.hadoop.hive.ql.WindowsPathUtil; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; @@ -107,13 +108,17 @@ public class TestHCatClient { @BeforeClass public static void startMetaStoreServer() throws Exception { + hcatConf = new HiveConf(TestHCatClient.class); + if (Shell.WINDOWS) { + WindowsPathUtil.convertPathsFromWindowsToHdfs(hcatConf); + } + Thread t = new Thread(new RunMS(msPort)); t.start(); Thread.sleep(10000); securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); - hcatConf = new HiveConf(TestHCatClient.class); hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + msPort); hcatConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); @@ -970,7 +975,7 @@ public class TestHCatClient { sourceMetaStore.addPartition(HCatAddPartitionDesc.create(sourcePartition_2).build()); // The source table now has 2 partitions, one in TEXTFILE, the other in ORC. - // Test that adding these partitions to the target-table *without* replicating the table-change. + // Test adding these partitions to the target-table *without* replicating the table-change. List<HCatPartition> sourcePartitions = sourceMetaStore.getPartitions(dbName, tableName); assertEquals("Unexpected number of source partitions.", 2, sourcePartitions.size()); @@ -1002,4 +1007,139 @@ public class TestHCatClient { assertTrue("Unexpected exception! " + unexpected.getMessage(), false); } } + + /** + * Test that partition-definitions can be replicated between HCat-instances, + * independently of table-metadata replication, using PartitionSpec interfaces. + * (This is essentially the same test as testPartitionRegistrationWithCustomSchema(), + * transliterated to use the PartitionSpec APIs.) + * 2 identical tables are created on 2 different HCat instances ("source" and "target"). + * On the source instance, + * 1. One partition is added with the old format ("TEXTFILE"). + * 2. The table is updated with an additional column and the data-format changed to ORC. + * 3. Another partition is added with the new format. + * 4. The partitions' metadata is copied to the target HCat instance, without updating the target table definition. + * 5. The partitions' metadata is tested to be an exact replica of that on the source. + * @throws Exception + */ + @Test + public void testPartitionSpecRegistrationWithCustomSchema() throws Exception { + try { + startReplicationTargetMetaStoreIfRequired(); + + HCatClient sourceMetaStore = HCatClient.create(new Configuration(hcatConf)); + final String dbName = "myDb"; + final String tableName = "myTable"; + + sourceMetaStore.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); + + sourceMetaStore.createDatabase(HCatCreateDBDesc.create(dbName).build()); + List<HCatFieldSchema> columnSchema = new ArrayList<HCatFieldSchema>( + Arrays.asList(new HCatFieldSchema("foo", Type.INT, ""), + new HCatFieldSchema("bar", Type.STRING, ""))); + + List<HCatFieldSchema> partitionSchema = Arrays.asList(new HCatFieldSchema("dt", Type.STRING, ""), + new HCatFieldSchema("grid", Type.STRING, "")); + + HCatTable sourceTable = new HCatTable(dbName, tableName).cols(columnSchema) + .partCols(partitionSchema) + .comment("Source table."); + + sourceMetaStore.createTable(HCatCreateTableDesc.create(sourceTable).build()); + + // Verify that the sourceTable was created successfully. + sourceTable = sourceMetaStore.getTable(dbName, tableName); + assertNotNull("Table couldn't be queried for. ", sourceTable); + + // Partitions added now should inherit table-schema, properties, etc. + Map<String, String> partitionSpec_1 = new HashMap<String, String>(); + partitionSpec_1.put("grid", "AB"); + partitionSpec_1.put("dt", "2011_12_31"); + HCatPartition sourcePartition_1 = new HCatPartition(sourceTable, partitionSpec_1, ""); + + sourceMetaStore.addPartition(HCatAddPartitionDesc.create(sourcePartition_1).build()); + assertEquals("Unexpected number of partitions. ", + sourceMetaStore.getPartitions(dbName, tableName).size(), 1); + // Verify that partition_1 was added correctly, and properties were inherited from the HCatTable. + HCatPartition addedPartition_1 = sourceMetaStore.getPartition(dbName, tableName, partitionSpec_1); + assertEquals("Column schema doesn't match.", addedPartition_1.getColumns(), sourceTable.getCols()); + assertEquals("InputFormat doesn't match.", addedPartition_1.getInputFormat(), sourceTable.getInputFileFormat()); + assertEquals("OutputFormat doesn't match.", addedPartition_1.getOutputFormat(), sourceTable.getOutputFileFormat()); + assertEquals("SerDe doesn't match.", addedPartition_1.getSerDe(), sourceTable.getSerdeLib()); + assertEquals("SerDe params don't match.", addedPartition_1.getSerdeParams(), sourceTable.getSerdeParams()); + + // Replicate table definition. + + HCatClient targetMetaStore = HCatClient.create(new Configuration(replicationTargetHCatConf)); + targetMetaStore.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); + + targetMetaStore.createDatabase(HCatCreateDBDesc.create(dbName).build()); + // Make a copy of the source-table, as would be done across class-loaders. + HCatTable targetTable = targetMetaStore.deserializeTable(sourceMetaStore.serializeTable(sourceTable)); + targetMetaStore.createTable(HCatCreateTableDesc.create(targetTable).build()); + targetTable = targetMetaStore.getTable(dbName, tableName); + + assertEquals("Created table doesn't match the source.", + targetTable.diff(sourceTable), HCatTable.NO_DIFF); + + // Modify Table schema at the source. + List<HCatFieldSchema> newColumnSchema = new ArrayList<HCatFieldSchema>(columnSchema); + newColumnSchema.add(new HCatFieldSchema("goo_new", Type.DOUBLE, "")); + Map<String, String> tableParams = new HashMap<String, String>(1); + tableParams.put("orc.compress", "ZLIB"); + sourceTable.cols(newColumnSchema) // Add a column. + .fileFormat("orcfile") // Change SerDe, File I/O formats. + .tblProps(tableParams) + .serdeParam(serdeConstants.FIELD_DELIM, Character.toString('\001')); + sourceMetaStore.updateTableSchema(dbName, tableName, sourceTable); + sourceTable = sourceMetaStore.getTable(dbName, tableName); + + // Add another partition to the source. + Map<String, String> partitionSpec_2 = new HashMap<String, String>(); + partitionSpec_2.put("grid", "AB"); + partitionSpec_2.put("dt", "2012_01_01"); + HCatPartition sourcePartition_2 = new HCatPartition(sourceTable, partitionSpec_2, ""); + sourceMetaStore.addPartition(HCatAddPartitionDesc.create(sourcePartition_2).build()); + + // The source table now has 2 partitions, one in TEXTFILE, the other in ORC. + // Test adding these partitions to the target-table *without* replicating the table-change. + + HCatPartitionSpec sourcePartitionSpec = sourceMetaStore.getPartitionSpecs(dbName, tableName, -1); + assertEquals("Unexpected number of source partitions.", 2, sourcePartitionSpec.size()); + + // Serialize the hcatPartitionSpec. + List<String> partitionSpecString = sourceMetaStore.serializePartitionSpec(sourcePartitionSpec); + + // Deserialize the HCatPartitionSpec using the target HCatClient instance. + HCatPartitionSpec targetPartitionSpec = targetMetaStore.deserializePartitionSpec(partitionSpecString); + assertEquals("Could not add the expected number of partitions.", + sourcePartitionSpec.size(), targetMetaStore.addPartitionSpec(targetPartitionSpec)); + + // Retrieve partitions. + targetPartitionSpec = targetMetaStore.getPartitionSpecs(dbName, tableName, -1); + assertEquals("Could not retrieve the expected number of partitions.", + sourcePartitionSpec.size(), targetPartitionSpec.size()); + + // Assert that the source and target partitions are equivalent. + HCatPartitionSpec.HCatPartitionIterator sourceIterator = sourcePartitionSpec.getPartitionIterator(); + HCatPartitionSpec.HCatPartitionIterator targetIterator = targetPartitionSpec.getPartitionIterator(); + + while (targetIterator.hasNext()) { + assertTrue("Fewer target partitions than source.", sourceIterator.hasNext()); + HCatPartition sourcePartition = sourceIterator.next(); + HCatPartition targetPartition = targetIterator.next(); + assertEquals("Column schema doesn't match.", sourcePartition.getColumns(), targetPartition.getColumns()); + assertEquals("InputFormat doesn't match.", sourcePartition.getInputFormat(), targetPartition.getInputFormat()); + assertEquals("OutputFormat doesn't match.", sourcePartition.getOutputFormat(), targetPartition.getOutputFormat()); + assertEquals("SerDe doesn't match.", sourcePartition.getSerDe(), targetPartition.getSerDe()); + assertEquals("SerDe params don't match.", sourcePartition.getSerdeParams(), targetPartition.getSerdeParams()); + + } + } + catch (Exception unexpected) { + LOG.error( "Unexpected exception! ", unexpected); + assertTrue("Unexpected exception! " + unexpected.getMessage(), false); + } + } + } Modified: hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java (original) +++ hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java Fri Nov 7 20:41:34 2014 @@ -229,15 +229,17 @@ public class ExecServiceImpl implements watchdog.checkException(); } catch (Exception ex) { - LOG.error("Command: " + cmd + " failed:", ex); + LOG.error("Command: " + cmd + " failed. res=" + res, ex); } if(watchdog.killedProcess()) { String msg = " was terminated due to timeout(" + timeout + "ms). See " + AppConfig .EXEC_TIMEOUT_NAME + " property"; - LOG.warn("Command: " + cmd + msg); + LOG.warn("Command: " + cmd + msg + " res=" + res); res.stderr += " Command " + msg; } - + if(res.exitcode != 0) { + LOG.info("Command: " + cmd + " failed. res=" + res); + } return res; } Modified: hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java (original) +++ hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java Fri Nov 7 20:41:34 2014 @@ -40,7 +40,7 @@ import org.apache.thrift.TException; /** * Helper class to run jobs using Kerberos security. Always safe to - * use these methods, it's a noop if security is not enabled. + * use these methods, it's a no-op if security is not enabled. */ public class SecureProxySupport { private Path tokenPath; @@ -140,6 +140,7 @@ public class SecureProxySupport { ugi.doAs(new PrivilegedExceptionAction<Object>() { public Object run() throws IOException { FileSystem fs = FileSystem.get(conf); + //todo: according to JavaDoc this seems like private API: addDelegationToken should be used twrapper.token = fs.getDelegationToken(ugi.getShortUserName()); return null; } Modified: hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java (original) +++ hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java Fri Nov 7 20:41:34 2014 @@ -83,6 +83,20 @@ public class SqoopDelegator extends Laun args.add("-D" + TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER); args.add("-D" + TempletonControllerJob.MAPREDUCE_JOB_TAGS_ARG_PLACEHOLDER); } + if(i == 0 && TempletonUtils.isset(libdir) && TempletonUtils.isset(appConf.sqoopArchive())) { + //http://sqoop.apache.org/docs/1.4.5/SqoopUserGuide.html#_using_generic_and_specific_arguments + String libJars = null; + for(String s : args) { + if(s.startsWith(JobSubmissionConstants.Sqoop.LIB_JARS)) { + libJars = s.substring(s.indexOf("=") + 1); + break; + } + } + //the jars in libJars will be localized to CWD of the launcher task; then -libjars will + //cause them to be localized for the Sqoop MR job tasks + args.add(TempletonUtils.quoteForWindows("-libjars")); + args.add(TempletonUtils.quoteForWindows(libJars)); + } } } else if (TempletonUtils.isset(optionsFile)) { args.add("--options-file"); @@ -114,11 +128,13 @@ public class SqoopDelegator extends Laun /**Sqoop accesses databases via JDBC. This means it needs to have appropriate JDBC drivers available. Normally, the user would install Sqoop and place these jars into SQOOP_HOME/lib. When WebHCat is configured to auto-ship the Sqoop tar file, we - need to make sure that relevant JDBC jars are available on target node. + need to make sure that relevant JDBC jars are available on target node but we cannot modify + lib/ of exploded tar because Dist Cache intentionally prevents this. The user is expected to place any JDBC jars into an HDFS directory and specify this - dir in "libdir" parameter. All the files in this dir will be copied to lib/ of the - exploded Sqoop tar ball on target node. + dir in "libdir" parameter. WebHCat then ensures that these jars are localized for the launcher task + and made available to Sqoop. {@link org.apache.hive.hcatalog.templeton.tool.LaunchMapper#handleSqoop(org.apache.hadoop.conf.Configuration, java.util.Map)} + {@link #makeArgs(String, String, String, String, String, boolean, String)} */ LOG.debug("libdir=" + libdir); List<Path> jarList = TempletonUtils.hadoopFsListChildren(libdir, appConf, runAs); Modified: hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java (original) +++ hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java Fri Nov 7 20:41:34 2014 @@ -18,7 +18,6 @@ */ package org.apache.hive.hcatalog.templeton.tool; -import com.google.common.io.Files; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -101,11 +100,18 @@ public class LaunchMapper extends Mapper if(TempletonUtils.isset(conf.get(Sqoop.LIB_JARS))) { //LIB_JARS should only be set if Sqoop is auto-shipped LOG.debug(Sqoop.LIB_JARS + "=" + conf.get(Sqoop.LIB_JARS)); - //copy these (which have now been localized) jars to sqoop/lib - String destDir = conf.get(AppConfig.SQOOP_HOME_PATH) + File.separator + "lib"; String[] files = conf.getStrings(Sqoop.LIB_JARS); + StringBuilder jdbcJars = new StringBuilder(); for(String f : files) { - Files.copy(new File(f), new File(destDir + File.separator + f)); + jdbcJars.append(f).append(File.pathSeparator); + } + jdbcJars.setLength(jdbcJars.length() - 1); + //this makes the jars available to Sqoop client + if(TempletonUtils.isset(System.getenv("HADOOP_CLASSPATH"))) { + env.put("HADOOP_CLASSPATH", System.getenv("HADOOP_CLASSPATH") + File.pathSeparator + jdbcJars.toString()); + } + else { + env.put("HADOOP_CLASSPATH", jdbcJars.toString()); } } } Modified: hive/branches/spark/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java (original) +++ hive/branches/spark/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java Fri Nov 7 20:41:34 2014 @@ -171,8 +171,8 @@ public class TestJdbcWithMiniKdc { MiniHiveKdc.HIVE_TEST_USER_2); } catch (SQLException e) { // Expected error - assertTrue(e.getMessage().contains("Failed to validate proxy privilege")); - assertTrue(e.getCause().getCause().getMessage().contains("Failed to validate proxy privilege")); + assertTrue(e.getMessage().contains("Error retrieving delegation token for user")); + assertTrue(e.getCause().getCause().getMessage().contains("is not allowed to impersonate")); } finally { hs2Conn.close(); } Modified: hive/branches/spark/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestPasswordWithCredentialProvider.java URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestPasswordWithCredentialProvider.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestPasswordWithCredentialProvider.java (original) +++ hive/branches/spark/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestPasswordWithCredentialProvider.java Fri Nov 7 20:41:34 2014 @@ -81,7 +81,7 @@ public class TestPasswordWithCredentialP conf.set("hadoop.security.credential.clear-text-fallback", "true"); // Set up CredentialProvider - conf.set("hadoop.security.credential.provider.path", "jceks://file/" + tmpDir + "/test.jks"); + conf.set("hadoop.security.credential.provider.path", "jceks://file/" + tmpDir.toURI().getPath() + "/test.jks"); // CredentialProvider/CredentialProviderFactory may not exist, depending on the version of // hadoop-2 being used to build Hive. Use reflection to do the following lines Modified: hive/branches/spark/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java (original) +++ hive/branches/spark/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java Fri Nov 7 20:41:34 2014 @@ -21,14 +21,19 @@ package org.apache.hive.jdbc; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.Statement; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.security.GroupMappingServiceProvider; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; import org.apache.hive.jdbc.miniHS2.MiniHS2; import org.junit.After; import org.junit.Before; @@ -37,6 +42,26 @@ import org.junit.Test; public class TestSchedulerQueue { + // hadoop group mapping that maps user to same group + public static class HiveTestSimpleGroupMapping implements GroupMappingServiceProvider { + public static String primaryTag = ""; + @Override + public List<String> getGroups(String user) throws IOException { + List<String> results = new ArrayList<String>(); + results.add(user + primaryTag); + results.add(user + "-group"); + return results; + } + + @Override + public void cacheGroupsRefresh() throws IOException { + } + + @Override + public void cacheGroupsAdd(List<String> groups) throws IOException { + } + } + private MiniHS2 miniHS2 = null; private static HiveConf conf = new HiveConf(); private Connection hs2Conn = null; @@ -44,6 +69,8 @@ public class TestSchedulerQueue { @BeforeClass public static void beforeTest() throws Exception { Class.forName(MiniHS2.getJdbcDriverName()); + conf.set("hadoop.security.group.mapping", + HiveTestSimpleGroupMapping.class.getName()); } @Before @@ -56,6 +83,7 @@ public class TestSchedulerQueue { miniHS2.setConfProperty(YarnConfiguration.RM_SCHEDULER, "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"); miniHS2.start(new HashMap<String, String>()); + HiveTestSimpleGroupMapping.primaryTag = ""; } @After @@ -79,6 +107,7 @@ public class TestSchedulerQueue { @Test public void testFairSchedulerQueueMapping() throws Exception { hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "user1", "bar"); + verifyProperty(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "false"); verifyProperty("mapreduce.framework.name", "yarn"); verifyProperty(HiveConf.ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE.varname, "true"); @@ -88,6 +117,31 @@ public class TestSchedulerQueue { } /** + * Verify: + * Test is running with MR2 and queue mapping are set correctly for primary group rule. + * @throws Exception + */ + @Test + public void testFairSchedulerPrimaryQueueMapping() throws Exception { + miniHS2.setConfProperty(FairSchedulerConfiguration.ALLOCATION_FILE, "fair-scheduler-test.xml"); + HiveTestSimpleGroupMapping.primaryTag = "-test"; + hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "user2", "bar"); + verifyProperty("mapreduce.job.queuename", "root.user2" + HiveTestSimpleGroupMapping.primaryTag); + } + + /** + * Verify: + * Test is running with MR2 and queue mapping are set correctly for primary group rule. + * @throws Exception + */ + @Test + public void testFairSchedulerSecondaryQueueMapping() throws Exception { + miniHS2.setConfProperty(FairSchedulerConfiguration.ALLOCATION_FILE, "fair-scheduler-test.xml"); + hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "user3", "bar"); + verifyProperty("mapreduce.job.queuename", "root.user3-group"); + } + + /** * Verify that the queue refresh doesn't happen when configured to be off. * * @throws Exception Modified: hive/branches/spark/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/AbstractHiveService.java URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/AbstractHiveService.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/AbstractHiveService.java (original) +++ hive/branches/spark/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/AbstractHiveService.java Fri Nov 7 20:41:34 2014 @@ -106,7 +106,7 @@ public abstract class AbstractHiveServic } // get service host - protected String getHost() { + public String getHost() { return hostname; } @@ -127,12 +127,12 @@ public abstract class AbstractHiveServic } // Get binary service port # - protected int getBinaryPort() { + public int getBinaryPort() { return binaryPort; } // Get http service port # - protected int getHttpPort() { + public int getHttpPort() { return httpPort; } Modified: hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthorizationApiAuthorizer.java URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthorizationApiAuthorizer.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthorizationApiAuthorizer.java (original) +++ hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAuthorizationApiAuthorizer.java Fri Nov 7 20:41:34 2014 @@ -156,7 +156,7 @@ public abstract class TestAuthorizationA FunctionInvoker invoker = new FunctionInvoker() { @Override public void invoke() throws Exception { - msc.create_role(new Role()); + msc.create_role(new Role("role1", 0, "owner")); } }; testFunction(invoker); Modified: hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java (original) +++ hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java Fri Nov 7 20:41:34 2014 @@ -34,8 +34,10 @@ import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener; import org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.ql.WindowsPathUtil; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Shell; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -55,7 +57,11 @@ public class StorageBasedMetastoreTestBa } protected HiveConf createHiveConf() throws Exception { - return new HiveConf(this.getClass()); + HiveConf conf = new HiveConf(this.getClass()); + if (Shell.WINDOWS) { + WindowsPathUtil.convertPathsFromWindowsToHdfs(conf); + } + return conf; } @Before @@ -71,9 +77,8 @@ public class StorageBasedMetastoreTestBa System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname, InjectableDummyAuthenticator.class.getName()); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); - clientHiveConf = createHiveConf(); + MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), clientHiveConf); // Turn off client-side authorization clientHiveConf.setBoolVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED,false); Modified: hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationDrops.java URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationDrops.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationDrops.java (original) +++ hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationDrops.java Fri Nov 7 20:41:34 2014 @@ -18,10 +18,15 @@ package org.apache.hadoop.hive.ql.security; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; +import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim; import org.junit.Assert; import org.junit.Test; @@ -30,6 +35,41 @@ import org.junit.Test; */ public class TestStorageBasedMetastoreAuthorizationDrops extends StorageBasedMetastoreTestBase { + protected static MiniDFSShim dfs = null; + + @Override + protected HiveConf createHiveConf() throws Exception { + // Hadoop FS ACLs do not work with LocalFileSystem, so set up MiniDFS. + HiveConf conf = super.createHiveConf(); + + String currentUserName = ShimLoader.getHadoopShims().getUGIForConf(conf).getShortUserName(); + conf.set("hadoop.proxyuser." + currentUserName + ".groups", "*"); + conf.set("hadoop.proxyuser." + currentUserName + ".hosts", "*"); + dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null); + FileSystem fs = dfs.getFileSystem(); + + Path warehouseDir = new Path(new Path(fs.getUri()), "/warehouse"); + fs.mkdirs(warehouseDir); + conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, warehouseDir.toString()); + conf.setBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS, true); + + // Set up scratch directory + Path scratchDir = new Path(new Path(fs.getUri()), "/scratchdir"); + conf.setVar(HiveConf.ConfVars.SCRATCHDIR, scratchDir.toString()); + + return conf; + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + + if (dfs != null) { + dfs.shutdown(); + dfs = null; + } + } + @Test public void testDropDatabase() throws Exception { dropDatabaseByOtherUser("-rwxrwxrwx", 0); Modified: hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java (original) +++ hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java Fri Nov 7 20:41:34 2014 @@ -140,7 +140,7 @@ public class TestCompactor { executeStatementOnDriver("CREATE EXTERNAL TABLE " + tblNameStg + "(a INT, b STRING)" + " ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\t' LINES TERMINATED BY '\\n'" + " STORED AS TEXTFILE" + - " LOCATION '" + stagingFolder.newFolder() + "'", driver); + " LOCATION '" + stagingFolder.newFolder().toURI().getPath() + "'", driver); executeStatementOnDriver("load data local inpath '" + BASIC_FILE_NAME + "' overwrite into table " + tblNameStg, driver); Modified: hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java (original) +++ hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java Fri Nov 7 20:41:34 2014 @@ -43,9 +43,6 @@ public class TestZooKeeperTokenStore ext private CuratorFramework zkClient = null; private int zkPort = -1; private ZooKeeperTokenStore ts; - // connect timeout large enough for slower test environments - private final int connectTimeoutMillis = 30000; - private final int sessionTimeoutMillis = 3000; @Override protected void setUp() throws Exception { @@ -55,10 +52,9 @@ public class TestZooKeeperTokenStore ext } this.zkCluster = new MiniZooKeeperCluster(); this.zkPort = this.zkCluster.startup(zkDataDir); - - this.zkClient = CuratorFrameworkFactory.builder().connectString("localhost:" + zkPort) - .sessionTimeoutMs(sessionTimeoutMillis).connectionTimeoutMs(connectTimeoutMillis) - .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build(); + this.zkClient = + CuratorFrameworkFactory.builder().connectString("localhost:" + zkPort) + .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build(); this.zkClient.start(); } @@ -74,15 +70,9 @@ public class TestZooKeeperTokenStore ext private Configuration createConf(String zkPath) { Configuration conf = new Configuration(); - conf.set( - HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR, - "localhost:" + this.zkPort); - conf.set( - HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ZNODE, - zkPath); - conf.setLong( - HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_TIMEOUTMILLIS, - connectTimeoutMillis); + conf.set(HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR, "localhost:" + + this.zkPort); + conf.set(HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ZNODE, zkPath); return conf; } Modified: hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java (original) +++ hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java Fri Nov 7 20:41:34 2014 @@ -1805,6 +1805,7 @@ public void testParseUrlHttpMode() throw ResultSet rs = stmt.executeQuery("SELECT 1 AS a, 2 AS a from " + tableName); assertTrue(rs.next()); assertEquals(1, rs.getInt("a")); + rs.close(); } Modified: hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java (original) +++ hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java Fri Nov 7 20:41:34 2014 @@ -31,15 +31,21 @@ import java.util.HashMap; import java.util.Map; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hive.jdbc.miniHS2.MiniHS2; import org.junit.After; +import org.junit.Assert; +import org.junit.Assume; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TestSSL { + private static final Logger LOG = LoggerFactory.getLogger(TestSSL.class); private static final String KEY_STORE_NAME = "keystore.jks"; private static final String TRUST_STORE_NAME = "truststore.jks"; private static final String KEY_STORE_PASSWORD = "HiveJdbc"; @@ -87,6 +93,73 @@ public class TestSSL { System.clearProperty(JAVA_TRUST_STORE_PASS_PROP); } + private int execCommand(String cmd) throws Exception { + int exitCode; + try { + String output = Shell.execCommand("bash", "-c", cmd); + LOG.info("Output from '" + cmd + "': " + output) ; + exitCode = 0; + } catch (Shell.ExitCodeException e) { + exitCode = e.getExitCode(); + LOG.info("Error executing '" + cmd + "', exitCode = " + exitCode, e); + } + return exitCode; + } + + /*** + * Tests to ensure SSLv2 and SSLv3 are disabled + */ + @Test + public void testSSLVersion() throws Exception { + Assume.assumeTrue(execCommand("which openssl") == 0); // we need openssl + Assume.assumeTrue(System.getProperty("os.name").toLowerCase() + .contains("linux")); // we depend on linux openssl exit codes + + setSslConfOverlay(confOverlay); + // Test in binary mode + setBinaryConfOverlay(confOverlay); + // Start HS2 with SSL + miniHS2.start(confOverlay); + + // make SSL connection + hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL() + ";ssl=true;sslTrustStore=" + + dataFileDir + File.separator + TRUST_STORE_NAME + ";trustStorePassword=" + + KEY_STORE_PASSWORD, System.getProperty("user.name"), "bar"); + hs2Conn.close(); + Assert.assertEquals("Expected exit code of 1", 1, + execCommand("openssl s_client -connect " + miniHS2.getHost() + ":" + miniHS2.getBinaryPort() + + " -ssl2 < /dev/null")); + Assert.assertEquals("Expected exit code of 1", 1, + execCommand("openssl s_client -connect " + miniHS2.getHost() + ":" + miniHS2.getBinaryPort() + + " -ssl3 < /dev/null")); + miniHS2.stop(); + + // Test in http mode + setHttpConfOverlay(confOverlay); + miniHS2.start(confOverlay); + // make SSL connection + try { + hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL() + + ";ssl=true;sslTrustStore=" + dataFileDir + File.separator + + TRUST_STORE_NAME + ";trustStorePassword=" + KEY_STORE_PASSWORD + + "?hive.server2.transport.mode=" + HS2_HTTP_MODE + + ";hive.server2.thrift.http.path=" + HS2_HTTP_ENDPOINT, + System.getProperty("user.name"), "bar"); + Assert.fail("Expected SQLException during connect"); + } catch (SQLException e) { + LOG.info("Expected exception: " + e, e); + Assert.assertEquals("08S01", e.getSQLState().trim()); + Throwable cause = e.getCause(); + Assert.assertNotNull(cause); + while (cause.getCause() != null) { + cause = cause.getCause(); + } + Assert.assertEquals("org.apache.http.NoHttpResponseException", cause.getClass().getName()); + Assert.assertEquals("The target server failed to respond", cause.getMessage()); + } + miniHS2.stop(); + } + /*** * Test SSL client with non-SSL server fails * @throws Exception Modified: hive/branches/spark/itests/src/test/resources/testconfiguration.properties URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/src/test/resources/testconfiguration.properties?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/itests/src/test/resources/testconfiguration.properties (original) +++ hive/branches/spark/itests/src/test/resources/testconfiguration.properties Fri Nov 7 20:41:34 2014 @@ -55,7 +55,20 @@ minitez.query.files.shared=alter_merge_2 bucket2.q,\ bucket3.q,\ bucket4.q,\ - cbo_correctness.q,\ + cbo_gby.q,\ + cbo_gby_empty.q,\ + cbo_join.q,\ + cbo_limit.q,\ + cbo_semijoin.q,\ + cbo_simple_select.q,\ + cbo_stats.q,\ + cbo_subq_exists.q,\ + cbo_subq_in.q,\ + cbo_subq_not_in.q,\ + cbo_udf_udaf.q,\ + cbo_union.q,\ + cbo_views.q,\ + cbo_windowing.q,\ correlationoptimizer1.q,\ count.q,\ create_merge_compressed.q,\ @@ -161,9 +174,25 @@ minitez.query.files.shared=alter_merge_2 vector_cast_constant.q,\ vector_char_4.q,\ vector_char_simple.q,\ + vector_coalesce.q,\ vector_count_distinct.q,\ vector_data_types.q,\ + vector_decimal_1.q,\ + vector_decimal_10_0.q,\ + vector_decimal_2.q,\ + vector_decimal_3.q,\ + vector_decimal_4.q,\ + vector_decimal_5.q,\ + vector_decimal_6.q,\ vector_decimal_aggregate.q,\ + vector_decimal_cast.q,\ + vector_decimal_expressions.q,\ + vector_decimal_mapjoin.q,\ + vector_decimal_math_funcs.q,\ + vector_decimal_precision.q,\ + vector_decimal_trailing.q,\ + vector_decimal_udf.q,\ + vector_decimal_udf2.q,\ vector_distinct_2.q,\ vector_elt.q,\ vector_groupby_3.q,\ @@ -196,6 +225,7 @@ minitez.query.files.shared=alter_merge_2 vectorization_9.q,\ vectorization_decimal_date.q,\ vectorization_div0.q,\ + vectorization_limit.q,\ vectorization_nested_udf.q,\ vectorization_not.q,\ vectorization_part.q,\ @@ -204,7 +234,10 @@ minitez.query.files.shared=alter_merge_2 vectorization_short_regress.q,\ vectorized_bucketmapjoin1.q,\ vectorized_case.q,\ + vectorized_casts.q,\ vectorized_context.q,\ + vectorized_date_funcs.q,\ + vectorized_distinct_gby.q,\ vectorized_mapjoin.q,\ vectorized_math_funcs.q,\ vectorized_nested_mapjoin.q,\ Modified: hive/branches/spark/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java (original) +++ hive/branches/spark/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java Fri Nov 7 20:41:34 2014 @@ -952,7 +952,7 @@ public class QTestUtil { for (Task<? extends Serializable> plan : tasks) { Utilities.serializePlan(plan, ofs, conf); } - + ofs.close(); fixXml4JDK7(outf.getPath()); maskPatterns(xmlPlanMask, outf.getPath()); @@ -964,6 +964,7 @@ public class QTestUtil { return exitVal; } finally { conf.set(HiveConf.ConfVars.PLAN_SERIALIZATION.varname, "kryo"); + IOUtils.closeQuietly(ofs); } } Modified: hive/branches/spark/jdbc/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/spark/jdbc/pom.xml?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/jdbc/pom.xml (original) +++ hive/branches/spark/jdbc/pom.xml Fri Nov 7 20:41:34 2014 @@ -29,6 +29,7 @@ <properties> <hive.path.to.root>..</hive.path.to.root> + <packaging.minimizeJar>false</packaging.minimizeJar> </properties> <dependencies> @@ -97,6 +98,11 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-framework</artifactId> + <version>${curator.version}</version> + </dependency> </dependencies> <profiles> @@ -122,6 +128,12 @@ </dependency> </dependencies> </profile> + <profile> + <id>dist</id> + <properties> + <packaging.minimizeJar>true</packaging.minimizeJar> + </properties> + </profile> </profiles> @@ -138,7 +150,7 @@ <goal>shade</goal> </goals> <configuration> - <minimizeJar>true</minimizeJar> + <minimizeJar>${packaging.minimizeJar}</minimizeJar> <shadedArtifactAttached>true</shadedArtifactAttached> <shadedClassifierName>${hive.jdbc.driver.classifier}</shadedClassifierName> <filters> @@ -163,6 +175,12 @@ </includes> </filter> <filter> + <artifact>commons-logging:commons-logging</artifact> + <includes> + <include>**</include> + </includes> + </filter> + <filter> <artifact>*:*</artifact> <excludes> <exclude>META-INF/*.SF</exclude> Modified: hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java URL: http://svn.apache.org/viewvc/hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java (original) +++ hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java Fri Nov 7 20:41:34 2014 @@ -99,6 +99,7 @@ public class HiveConnection implements j private JdbcConnectionParams connParams; private final boolean isEmbeddedMode; private TTransport transport; + private boolean assumeSubject; // TODO should be replaced by CliServiceClient private TCLIService.Iface client; private boolean isClosed = true; @@ -177,6 +178,9 @@ public class HiveConnection implements j private void openTransport() throws SQLException { while (true) { try { + assumeSubject = + JdbcConnectionParams.AUTH_KERBEROS_AUTH_TYPE_FROM_SUBJECT.equals(sessConfMap + .get(JdbcConnectionParams.AUTH_KERBEROS_AUTH_TYPE)); transport = isHttpTransportMode() ? createHttpTransport() : createBinaryTransport(); if (!transport.isOpen()) { LOG.info("Will try to open client transport with JDBC Uri: " + jdbcUriString); @@ -265,8 +269,9 @@ public class HiveConnection implements j * In https mode, the entire information is encrypted * TODO: Optimize this with a mix of kerberos + using cookie. */ - requestInterceptor = new HttpKerberosRequestInterceptor( - sessConfMap.get(JdbcConnectionParams.AUTH_PRINCIPAL), host, getServerHttpUrl(false)); + requestInterceptor = + new HttpKerberosRequestInterceptor(sessConfMap.get(JdbcConnectionParams.AUTH_PRINCIPAL), + host, getServerHttpUrl(useSsl), assumeSubject); } else { /** @@ -351,8 +356,6 @@ public class HiveConnection implements j } saslProps.put(Sasl.QOP, saslQOP.toString()); saslProps.put(Sasl.SERVER_AUTH, "true"); - boolean assumeSubject = JdbcConnectionParams.AUTH_KERBEROS_AUTH_TYPE_FROM_SUBJECT.equals(sessConfMap - .get(JdbcConnectionParams.AUTH_KERBEROS_AUTH_TYPE)); transport = KerberosSaslHelper.getKerberosTransport( sessConfMap.get(JdbcConnectionParams.AUTH_PRINCIPAL), host, HiveAuthFactory.getSocketTransport(host, port, loginTimeout), saslProps, Modified: hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/HttpKerberosRequestInterceptor.java URL: http://svn.apache.org/viewvc/hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/HttpKerberosRequestInterceptor.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/HttpKerberosRequestInterceptor.java (original) +++ hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/HttpKerberosRequestInterceptor.java Fri Nov 7 20:41:34 2014 @@ -39,15 +39,17 @@ public class HttpKerberosRequestIntercep String principal; String host; String serverHttpUrl; + boolean assumeSubject; // A fair reentrant lock private static ReentrantLock kerberosLock = new ReentrantLock(true); public HttpKerberosRequestInterceptor(String principal, String host, - String serverHttpUrl) { + String serverHttpUrl, boolean assumeSubject) { this.principal = principal; this.host = host; this.serverHttpUrl = serverHttpUrl; + this.assumeSubject = assumeSubject; } @Override @@ -59,7 +61,7 @@ public class HttpKerberosRequestIntercep // Locking ensures the tokens are unique in case of concurrent requests kerberosLock.lock(); kerberosAuthHeader = HttpAuthUtils.getKerberosServiceTicket( - principal, host, serverHttpUrl); + principal, host, serverHttpUrl, assumeSubject); // Set the session key token (Base64 encoded) in the headers httpRequest.addHeader(HttpAuthUtils.AUTHORIZATION + ": " + HttpAuthUtils.NEGOTIATE + " ", kerberosAuthHeader); Modified: hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/Utils.java URL: http://svn.apache.org/viewvc/hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/Utils.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/Utils.java (original) +++ hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/Utils.java Fri Nov 7 20:41:34 2014 @@ -100,8 +100,6 @@ public class Utils { static final String ZOOKEEPER_DEFAULT_NAMESPACE = "hiveserver2"; // Non-configurable params: - // ZOOKEEPER_SESSION_TIMEOUT is not exposed as client configurable - static final int ZOOKEEPER_SESSION_TIMEOUT = 600 * 1000; // Currently supports JKS keystore format static final String SSL_TRUST_STORE_TYPE = "JKS"; Modified: hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java URL: http://svn.apache.org/viewvc/hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java?rev=1637444&r1=1637443&r2=1637444&view=diff ============================================================================== --- hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java (original) +++ hive/branches/spark/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java Fri Nov 7 20:41:34 2014 @@ -25,9 +25,11 @@ import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.hive.jdbc.Utils.JdbcConnectionParams; import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.ZooKeeper; public class ZooKeeperHiveClientHelper { public static final Log LOG = LogFactory.getLog(ZooKeeperHiveClientHelper.class.getName()); @@ -59,14 +61,12 @@ public class ZooKeeperHiveClientHelper { List<String> serverHosts; Random randomizer = new Random(); String serverNode; - ZooKeeper zooKeeperClient = null; - // Pick a random HiveServer2 host from the ZooKeeper namspace + CuratorFramework zooKeeperClient = + CuratorFrameworkFactory.builder().connectString(zooKeeperEnsemble) + .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build(); + zooKeeperClient.start(); try { - zooKeeperClient = - new ZooKeeper(zooKeeperEnsemble, JdbcConnectionParams.ZOOKEEPER_SESSION_TIMEOUT, - new ZooKeeperHiveClientHelper.DummyWatcher()); - // All the HiveServer2 host nodes that are in ZooKeeper currently - serverHosts = zooKeeperClient.getChildren("/" + zooKeeperNamespace, false); + serverHosts = zooKeeperClient.getChildren().forPath("/" + zooKeeperNamespace); // Remove the znodes we've already tried from this list serverHosts.removeAll(connParams.getRejectedHostZnodePaths()); if (serverHosts.isEmpty()) { @@ -76,22 +76,18 @@ public class ZooKeeperHiveClientHelper { // Now pick a host randomly serverNode = serverHosts.get(randomizer.nextInt(serverHosts.size())); connParams.setCurrentHostZnodePath(serverNode); - // Read the value from the node (UTF-8 enoded byte array) and convert it to a String String serverUri = - new String(zooKeeperClient.getData("/" + zooKeeperNamespace + "/" + serverNode, false, - null), Charset.forName("UTF-8")); + new String( + zooKeeperClient.getData().forPath("/" + zooKeeperNamespace + "/" + serverNode), + Charset.forName("UTF-8")); LOG.info("Selected HiveServer2 instance with uri: " + serverUri); return serverUri; } catch (Exception e) { throw new ZooKeeperHiveClientException("Unable to read HiveServer2 uri from ZooKeeper", e); } finally { - // Try to close the client connection with ZooKeeper + // Close the client connection with ZooKeeper if (zooKeeperClient != null) { - try { - zooKeeperClient.close(); - } catch (Exception e) { - // No-op - } + zooKeeperClient.close(); } } }