This is an automated email from the ASF dual-hosted git repository. pvary pushed a commit to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-2.3 by this push: new eaea1f0e1e8 HIVE-26882: Allow transactional check of Table parameter before altering the Table (#3888) (Peter Vary reviewed by Prasanth Jayachandran and Szehon Ho) (#3947) eaea1f0e1e8 is described below commit eaea1f0e1e84f9f66ba9f08cbe5d3c491d5f14c7 Author: pvary <peter.vary.apa...@gmail.com> AuthorDate: Sun Jan 15 10:32:06 2023 +0100 HIVE-26882: Allow transactional check of Table parameter before altering the Table (#3888) (Peter Vary reviewed by Prasanth Jayachandran and Szehon Ho) (#3947) * HIVE-17981 Create a set of builders for Thrift classes. This closes #274. (Alan Gates, reviewed by Peter Vary) * HIVE-18355: Add builder for metastore Thrift classes missed in the first pass - FunctionBuilder (Peter Vary, reviewed by Alan Gates) * HIVE-18372: Create testing infra to test different HMS instances (Peter Vary, reviewed by Marta Kuczora, Vihang Karajgaonkar and Adam Szita) * HIVE-26882: Allow transactional check of Table parameter before altering the Table (#3888) (Peter Vary reviewed by Prasanth Jayachandran and Szehon Ho) Co-authored-by: Alan Gates <ga...@hortonworks.com> Co-authored-by: Peter Vary <pv...@cloudera.com> Co-authored-by: Peter Vary <peter_va...@apple.com> --- .../hcatalog/listener/DummyRawStoreFailEvent.java | 5 + metastore/if/hive_metastore.thrift | 3 + .../thrift/gen-cpp/hive_metastore_constants.cpp | 4 + .../gen/thrift/gen-cpp/hive_metastore_constants.h | 2 + .../metastore/api/hive_metastoreConstants.java | 4 + .../src/gen/thrift/gen-php/metastore/Types.php | 10 + .../gen/thrift/gen-py/hive_metastore/constants.py | 2 + .../gen/thrift/gen-rb/hive_metastore_constants.rb | 4 + .../metastore/DefaultPartitionExpressionProxy.java | 56 ++ .../hadoop/hive/metastore/HiveAlterHandler.java | 20 +- .../apache/hadoop/hive/metastore/ObjectStore.java | 28 +- .../org/apache/hadoop/hive/metastore/RawStore.java | 14 +- .../client/builder/ConstraintBuilder.java | 98 ++++ .../metastore/client/builder/DatabaseBuilder.java | 88 +++ .../metastore/client/builder/FunctionBuilder.java | 115 ++++ .../GrantRevokePrivilegeRequestBuilder.java | 63 +++ .../client/builder/HiveObjectPrivilegeBuilder.java | 63 +++ .../client/builder/HiveObjectRefBuilder.java | 64 +++ .../metastore/client/builder/IndexBuilder.java | 104 ++++ .../metastore/client/builder/PartitionBuilder.java | 102 ++++ .../client/builder/PrivilegeGrantInfoBuilder.java | 83 +++ .../hive/metastore/client/builder/RoleBuilder.java | 55 ++ .../client/builder/SQLForeignKeyBuilder.java | 83 +++ .../client/builder/SQLPrimaryKeyBuilder.java | 42 ++ .../client/builder/StorageDescriptorBuilder.java | 210 +++++++ .../metastore/client/builder/TableBuilder.java | 155 +++++ .../hadoop/hive/metastore/hbase/HBaseStore.java | 6 + .../hadoop/hive/metastore/utils/SecurityUtils.java | 313 +++++++++++ .../metastore/DummyRawStoreControlledCommit.java | 5 + .../metastore/DummyRawStoreForJdoConnection.java | 6 + .../metastore/client/MetaStoreFactoryForTests.java | 107 ++++ .../hive/metastore/client/TestDatabases.java | 622 +++++++++++++++++++++ .../client/TestTablesCreateDropAlterTruncate.java | 232 ++++++++ .../minihms/AbstractMetaStoreService.java | 153 +++++ .../minihms/ClusterMetaStoreForTests.java | 33 ++ .../minihms/EmbeddedMetaStoreForTests.java | 34 ++ .../hadoop/hive/metastore/minihms/MiniHMS.java | 69 +++ .../metastore/minihms/RemoteMetaStoreForTests.java | 41 ++ 38 files changed, 3090 insertions(+), 8 deletions(-) diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index 9871083ac74..a5cc94e846c 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -119,6 +119,11 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { return objectStore.openTransaction(); } + @Override + public boolean openTransaction(String isolationLevel) { + return objectStore.openTransaction(isolationLevel); + } + @Override public void rollbackTransaction() { objectStore.rollbackTransaction(); diff --git a/metastore/if/hive_metastore.thrift b/metastore/if/hive_metastore.thrift index 9df27319ffa..161fd8c5261 100755 --- a/metastore/if/hive_metastore.thrift +++ b/metastore/if/hive_metastore.thrift @@ -1548,3 +1548,6 @@ const string TABLE_NO_AUTO_COMPACT = "no_auto_compaction", const string TABLE_TRANSACTIONAL_PROPERTIES = "transactional_properties", +// Keys for alter table environment context parameters +const string EXPECTED_PARAMETER_KEY = "expected_parameter_key", +const string EXPECTED_PARAMETER_VALUE = "expected_parameter_value", \ No newline at end of file diff --git a/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp b/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp index 1cbd176597b..a24bfd86f8d 100644 --- a/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp +++ b/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp @@ -59,6 +59,10 @@ hive_metastoreConstants::hive_metastoreConstants() { TABLE_TRANSACTIONAL_PROPERTIES = "transactional_properties"; + EXPECTED_PARAMETER_KEY = "expected_parameter_key"; + + EXPECTED_PARAMETER_VALUE = "expected_parameter_value"; + } }}} // namespace diff --git a/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h b/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h index 3d068c3ec9e..3798cdd9d9c 100644 --- a/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h +++ b/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h @@ -39,6 +39,8 @@ class hive_metastoreConstants { std::string TABLE_IS_TRANSACTIONAL; std::string TABLE_NO_AUTO_COMPACT; std::string TABLE_TRANSACTIONAL_PROPERTIES; + std::string EXPECTED_PARAMETER_KEY; + std::string EXPECTED_PARAMETER_VALUE; }; extern const hive_metastoreConstants g_hive_metastore_constants; diff --git a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java index 8de8896bff4..f54726a01e6 100644 --- a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java +++ b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java @@ -84,4 +84,8 @@ public class hive_metastoreConstants { public static final String TABLE_TRANSACTIONAL_PROPERTIES = "transactional_properties"; + public static final String EXPECTED_PARAMETER_KEY = "expected_parameter_key"; + + public static final String EXPECTED_PARAMETER_VALUE = "expected_parameter_value"; + } diff --git a/metastore/src/gen/thrift/gen-php/metastore/Types.php b/metastore/src/gen/thrift/gen-php/metastore/Types.php index d81ff27e178..a82ee5e8e88 100644 --- a/metastore/src/gen/thrift/gen-php/metastore/Types.php +++ b/metastore/src/gen/thrift/gen-php/metastore/Types.php @@ -20239,6 +20239,8 @@ final class Constant extends \Thrift\Type\TConstant { static protected $TABLE_IS_TRANSACTIONAL; static protected $TABLE_NO_AUTO_COMPACT; static protected $TABLE_TRANSACTIONAL_PROPERTIES; + static protected $EXPECTED_PARAMETER_KEY; + static protected $EXPECTED_PARAMETER_VALUE; static protected function init_DDL_TIME() { return "transient_lastDdlTime"; @@ -20335,6 +20337,14 @@ final class Constant extends \Thrift\Type\TConstant { static protected function init_TABLE_TRANSACTIONAL_PROPERTIES() { return "transactional_properties"; } + + static protected function init_EXPECTED_PARAMETER_KEY() { + return "expected_parameter_key"; + } + + static protected function init_EXPECTED_PARAMETER_VALUE() { + return "expected_parameter_value"; + } } diff --git a/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py b/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py index 5100236afa2..0e16bb3e787 100644 --- a/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py +++ b/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py @@ -33,3 +33,5 @@ META_TABLE_STORAGE = "storage_handler" TABLE_IS_TRANSACTIONAL = "transactional" TABLE_NO_AUTO_COMPACT = "no_auto_compaction" TABLE_TRANSACTIONAL_PROPERTIES = "transactional_properties" +EXPECTED_PARAMETER_KEY = "expected_parameter_key" +EXPECTED_PARAMETER_VALUE = "expected_parameter_value" diff --git a/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb b/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb index 6aa7143c76b..ad6cd226bb7 100644 --- a/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb +++ b/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb @@ -55,3 +55,7 @@ TABLE_NO_AUTO_COMPACT = %q"no_auto_compaction" TABLE_TRANSACTIONAL_PROPERTIES = %q"transactional_properties" +EXPECTED_PARAMETER_KEY = %q"expected_parameter_key" + +EXPECTED_PARAMETER_VALUE = %q"expected_parameter_value" + diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultPartitionExpressionProxy.java b/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultPartitionExpressionProxy.java new file mode 100644 index 00000000000..2b4a8f099ab --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultPartitionExpressionProxy.java @@ -0,0 +1,56 @@ +package org.apache.hadoop.hive.metastore; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; + +import java.util.List; + +/** + * Default implementation of PartitionExpressionProxy. Eventually this should use the SARGs in + * Hive's storage-api. For now it just throws UnsupportedOperationException. + */ +public class DefaultPartitionExpressionProxy implements PartitionExpressionProxy { + @Override + public String convertExprToFilter(byte[] expr) throws MetaException { + throw new UnsupportedOperationException(); + } + + @Override + public boolean filterPartitionsByExpr(List<String> partColumnNames, List<PrimitiveTypeInfo> partColumnTypeInfos, byte[] expr, String defaultPartitionName, List<String> partitionNames) throws MetaException { + throw new UnsupportedOperationException(); + } + + @Override + public FileMetadataExprType getMetadataType(String inputFormat) { + throw new UnsupportedOperationException(); + } + + @Override + public FileFormatProxy getFileFormatProxy(FileMetadataExprType type) { + throw new UnsupportedOperationException(); + } + + @Override + public SearchArgument createSarg(byte[] expr) { + throw new UnsupportedOperationException(); + } +} \ No newline at end of file diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 3fefe5502b8..971c23f751a 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.ipc.RemoteException; import org.apache.hive.common.util.HiveStringUtils; +import javax.jdo.Constants; import java.io.IOException; import java.net.URI; import java.util.ArrayList; @@ -126,7 +127,18 @@ public class HiveAlterHandler implements AlterHandler { } try { - msdb.openTransaction(); + String expectedKey = environmentContext != null && environmentContext.getProperties() != null ? + environmentContext.getProperties().get(hive_metastoreConstants.EXPECTED_PARAMETER_KEY) : null; + String expectedValue = environmentContext != null && environmentContext.getProperties() != null ? + environmentContext.getProperties().get(hive_metastoreConstants.EXPECTED_PARAMETER_VALUE) : null; + + if (expectedKey != null) { + // If we have to check the expected state of the table we have to prevent nonrepeatable reads. + msdb.openTransaction(Constants.TX_REPEATABLE_READ); + } else { + msdb.openTransaction(); + } + name = name.toLowerCase(); dbname = dbname.toLowerCase(); @@ -146,6 +158,12 @@ public class HiveAlterHandler implements AlterHandler { throw new InvalidOperationException("table " + dbname + "." + name + " doesn't exist"); } + if (expectedKey != null && expectedValue != null + && !expectedValue.equals(oldt.getParameters().get(expectedKey))) { + throw new MetaException("The table has been modified. The parameter value for key '" + expectedKey + "' is '" + + oldt.getParameters().get(expectedKey) + "'. The expected was value was '" + expectedValue + "'"); + } + // Views derive the column type from the base table definition. So the view definition // can be altered to change the column types. The column type compatibility checks should // be done only for non-views. diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index bb69d07db89..eab348f76f3 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -561,17 +561,31 @@ public class ObjectStore implements RawStore, Configurable { } /** - * Opens a new one or the one already created Every call of this function must - * have corresponding commit or rollback function call + * Opens a new one or the one already created. Every call of this function must + * have corresponding commit or rollback function call. * * @return an active transaction */ - @Override public boolean openTransaction() { + return openTransaction(null); + } + + /** + * Opens a new one or the one already created. Every call of this function must + * have corresponding commit or rollback function call. + * + * @param isolationLevel The transaction isolation level. Only possible to set on the first call. + * @return an active transaction + */ + @Override + public boolean openTransaction(String isolationLevel) { openTrasactionCalls++; if (openTrasactionCalls == 1) { currentTransaction = pm.currentTransaction(); + if (isolationLevel != null) { + currentTransaction.setIsolationLevel(isolationLevel); + } currentTransaction.begin(); transactionStatus = TXN_STATUS.OPEN; } else { @@ -581,10 +595,16 @@ public class ObjectStore implements RawStore, Configurable { throw new RuntimeException("openTransaction called in an interior" + " transaction scope, but currentTransaction is not active."); } + + // Can not change the isolation level on an already open transaction + if (isolationLevel != null && !isolationLevel.equals(currentTransaction.getIsolationLevel())) { + throw new RuntimeException("Can not set isolation level on an open transaction"); + } } boolean result = currentTransaction.isActive(); - debugLog("Open transaction: count = " + openTrasactionCalls + ", isActive = " + result); + debugLog("Open transaction: count = " + openTrasactionCalls + ", isActive = " + result + ", isolationLevel = " + + currentTransaction.getIsolationLevel()); return result; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java index 639a1d8d60c..3e50db451f9 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -77,13 +77,21 @@ public interface RawStore extends Configurable { public abstract void shutdown(); /** - * Opens a new one or the one already created Every call of this function must - * have corresponding commit or rollback function call + * Opens a new one or the one already created. Every call of this function must + * have corresponding commit or rollback function call. * * @return an active transaction */ + boolean openTransaction(); - public abstract boolean openTransaction(); + /** + * Opens a new one or the one already created. Every call of this function must + * have corresponding commit or rollback function call. + * + * @param isolationLevel The transaction isolation level. Only possible to set on the first call. + * @return an active transaction + */ + boolean openTransaction(String isolationLevel); /** * if this is the commit of the first open call then an actual commit is diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/ConstraintBuilder.java b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/ConstraintBuilder.java new file mode 100644 index 00000000000..50e779a22bf --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/ConstraintBuilder.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.client.builder; + +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Table; + +/** + * Base builder for all types of constraints. Database name, table name, and column name + * must be provided. + * @param <T> Type of builder extending this. + */ +abstract class ConstraintBuilder<T> { + protected String dbName, tableName, columnName, constraintName; + protected int keySeq; + protected boolean enable, validate, rely; + private T child; + + protected ConstraintBuilder() { + keySeq = 1; + enable = true; + validate = rely = false; + } + + protected void setChild(T child) { + this.child = child; + } + + protected void checkBuildable(String defaultConstraintName) throws MetaException { + if (dbName == null || tableName == null || columnName == null) { + throw new MetaException("You must provide database name, table name, and column name"); + } + if (constraintName == null) { + constraintName = dbName + "_" + tableName + "_" + columnName + "_" + defaultConstraintName; + } + } + + public T setDbName(String dbName) { + this.dbName = dbName; + return child; + } + + public T setTableName(String tableName) { + this.tableName = tableName; + return child; + } + + public T setDbAndTableName(Table table) { + this.dbName = table.getDbName(); + this.tableName = table.getTableName(); + return child; + } + + public T setColumnName(String columnName) { + this.columnName = columnName; + return child; + } + + public T setConstraintName(String constraintName) { + this.constraintName = constraintName; + return child; + } + + public T setKeySeq(int keySeq) { + this.keySeq = keySeq; + return child; + } + + public T setEnable(boolean enable) { + this.enable = enable; + return child; + } + + public T setValidate(boolean validate) { + this.validate = validate; + return child; + } + + public T setRely(boolean rely) { + this.rely = rely; + return child; + } +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/DatabaseBuilder.java b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/DatabaseBuilder.java new file mode 100644 index 00000000000..99addedc162 --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/DatabaseBuilder.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.client.builder; + +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.PrincipalType; +import org.apache.hadoop.hive.metastore.utils.SecurityUtils; +import org.apache.thrift.TException; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * A builder for {@link Database}. The name of the new database is required. Everything else + * selects reasonable defaults. + */ +public class DatabaseBuilder { + private String name, description, location; + private Map<String, String> params = new HashMap<>(); + private String ownerName; + private PrincipalType ownerType; + + public DatabaseBuilder setName(String name) { + this.name = name; + return this; + } + + public DatabaseBuilder setDescription(String description) { + this.description = description; + return this; + } + + public DatabaseBuilder setLocation(String location) { + this.location = location; + return this; + } + + public DatabaseBuilder setParams(Map<String, String> params) { + this.params = params; + return this; + } + + public DatabaseBuilder addParam(String key, String value) { + params.put(key, value); + return this; + } + + public DatabaseBuilder setOwnerName(String ownerName) { + this.ownerName = ownerName; + return this; + } + + public DatabaseBuilder setOwnerType(PrincipalType ownerType) { + this.ownerType = ownerType; + return this; + } + + public Database build() throws TException { + if (name == null) throw new MetaException("You must name the database"); + Database db = new Database(name, description, location, params); + try { + if (ownerName != null) ownerName = SecurityUtils.getUser(); + db.setOwnerName(ownerName); + if (ownerType == null) ownerType = PrincipalType.USER; + db.setOwnerType(ownerType); + return db; + } catch (IOException e) { + throw new MetaException(e.getMessage()); + } + } +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/FunctionBuilder.java b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/FunctionBuilder.java new file mode 100644 index 00000000000..378b56b3792 --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/FunctionBuilder.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.client.builder; + +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Function; +import org.apache.hadoop.hive.metastore.api.FunctionType; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.PrincipalType; +import org.apache.hadoop.hive.metastore.api.ResourceUri; +import org.apache.hadoop.hive.metastore.utils.SecurityUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Class for creating Thrift Function objects for tests, and API usage. + */ +public class FunctionBuilder { + private String dbName = "default"; + private String funcName = null; + private String className = null; + private String owner = null; + private PrincipalType ownerType; + private int createTime; + private FunctionType funcType; + private List<ResourceUri> resourceUris; + + public FunctionBuilder() { + // Set some reasonable defaults + ownerType = PrincipalType.USER; + createTime = (int) (System.currentTimeMillis() / 1000); + funcType = FunctionType.JAVA; + resourceUris = new ArrayList<ResourceUri>(); + } + + public FunctionBuilder setDbName(String dbName) { + this.dbName = dbName; + return this; + } + + public FunctionBuilder setDbName(Database db) { + this.dbName = db.getName(); + return this; + } + + public FunctionBuilder setName(String funcName) { + this.funcName = funcName; + return this; + } + + public FunctionBuilder setClass(String className) { + this.className = className; + return this; + } + + public FunctionBuilder setOwner(String owner) { + this.owner = owner; + return this; + } + + public FunctionBuilder setOwnerType(PrincipalType ownerType) { + this.ownerType = ownerType; + return this; + } + + public FunctionBuilder setCreateTime(int createTime) { + this.createTime = createTime; + return this; + } + + public FunctionBuilder setFunctionType(FunctionType funcType) { + this.funcType = funcType; + return this; + } + + public FunctionBuilder setResourceUris(List<ResourceUri> resourceUris) { + this.resourceUris = resourceUris; + return this; + } + + public FunctionBuilder addResourceUri(ResourceUri resourceUri) { + this.resourceUris.add(resourceUri); + return this; + } + + public Function build() throws MetaException { + try { + if (owner != null) { + owner = SecurityUtils.getUser(); + } + } catch (IOException e) { + throw new MetaException(e.getMessage()); + } + return new Function(funcName, dbName, className, owner, ownerType, createTime, funcType, + resourceUris); + } +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/GrantRevokePrivilegeRequestBuilder.java b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/GrantRevokePrivilegeRequestBuilder.java new file mode 100644 index 00000000000..26cea191e16 --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/GrantRevokePrivilegeRequestBuilder.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.client.builder; + +import org.apache.hadoop.hive.metastore.api.GrantRevokePrivilegeRequest; +import org.apache.hadoop.hive.metastore.api.GrantRevokeType; +import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.PrivilegeBag; + +/** + * A builder for {@link GrantRevokePrivilegeRequest}. The revoke of grant option defaults to + * false. The request Type and the privileges must be provided. + */ +public class GrantRevokePrivilegeRequestBuilder { + private GrantRevokeType requestType; + private PrivilegeBag privileges; + private boolean revokeGrantOption; + + public GrantRevokePrivilegeRequestBuilder() { + privileges = new PrivilegeBag(); + revokeGrantOption = false; + } + + public GrantRevokePrivilegeRequestBuilder setRequestType(GrantRevokeType requestType) { + this.requestType = requestType; + return this; + } + + public GrantRevokePrivilegeRequestBuilder setRevokeGrantOption(boolean revokeGrantOption) { + this.revokeGrantOption = revokeGrantOption; + return this; + } + + public GrantRevokePrivilegeRequestBuilder addPrivilege(HiveObjectPrivilege privilege) { + privileges.addToPrivileges(privilege); + return this; + } + + public GrantRevokePrivilegeRequest build() throws MetaException { + if (requestType == null || privileges.getPrivilegesSize() == 0) { + throw new MetaException("The request type and at least one privilege must be provided."); + } + GrantRevokePrivilegeRequest rqst = new GrantRevokePrivilegeRequest(requestType, privileges); + if (revokeGrantOption) rqst.setRevokeGrantOption(revokeGrantOption); + return rqst; + } +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/HiveObjectPrivilegeBuilder.java b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/HiveObjectPrivilegeBuilder.java new file mode 100644 index 00000000000..d802e1a971a --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/HiveObjectPrivilegeBuilder.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.client.builder; + +import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege; +import org.apache.hadoop.hive.metastore.api.HiveObjectRef; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.PrincipalType; +import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo; + +/** + * Builder for {@link HiveObjectPrivilege}. All values must be set. + */ +public class HiveObjectPrivilegeBuilder { + private HiveObjectRef hiveObjectRef; + private String principleName; + private PrincipalType principalType; + private PrivilegeGrantInfo grantInfo; + + public HiveObjectPrivilegeBuilder setHiveObjectRef(HiveObjectRef hiveObjectRef) { + this.hiveObjectRef = hiveObjectRef; + return this; + } + + public HiveObjectPrivilegeBuilder setPrincipleName(String principleName) { + this.principleName = principleName; + return this; + } + + public HiveObjectPrivilegeBuilder setPrincipalType(PrincipalType principalType) { + this.principalType = principalType; + return this; + } + + public HiveObjectPrivilegeBuilder setGrantInfo(PrivilegeGrantInfo grantInfo) { + this.grantInfo = grantInfo; + return this; + } + + public HiveObjectPrivilege build() throws MetaException { + if (hiveObjectRef == null || principleName == null || principalType == null || + grantInfo == null) { + throw new MetaException("hive object reference, principle name and type, and grant info " + + "must all be provided"); + } + return new HiveObjectPrivilege(hiveObjectRef, principleName, principalType, grantInfo); + } +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/HiveObjectRefBuilder.java b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/HiveObjectRefBuilder.java new file mode 100644 index 00000000000..94c871354d2 --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/HiveObjectRefBuilder.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.client.builder; + +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.HiveObjectRef; +import org.apache.hadoop.hive.metastore.api.HiveObjectType; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * A builder for {@link HiveObjectRef}. Unlike most builders (which allow a gradual building up + * of the values) this gives a number of methods that take the object to be referenced and then + * build the appropriate reference. This is intended primarily for use with + * {@link HiveObjectPrivilegeBuilder} + */ +public class HiveObjectRefBuilder { + private HiveObjectType objectType; + private String dbName, objectName, columnName; + private List<String> partValues; + + public HiveObjectRef buildGlobalReference() { + return new HiveObjectRef(HiveObjectType.GLOBAL, null, null, new ArrayList<String>(), null); + } + + public HiveObjectRef buildDatabaseReference(Database db) { + return new + HiveObjectRef(HiveObjectType.DATABASE, db.getName(), null, new ArrayList<String>(), null); + } + + public HiveObjectRef buildTableReference(Table table) { + return new HiveObjectRef(HiveObjectType.TABLE, table.getDbName(), table.getTableName(), + new ArrayList<String>(), null); + } + + public HiveObjectRef buildPartitionReference(Partition part) { + return new HiveObjectRef(HiveObjectType.PARTITION, part.getDbName(), part.getTableName(), + part.getValues(), null); + } + + public HiveObjectRef buildColumnReference(Table table, String columnName) { + return new HiveObjectRef(HiveObjectType.TABLE, table.getDbName(), table.getTableName(), + new ArrayList<String>(), columnName); + } +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/IndexBuilder.java b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/IndexBuilder.java new file mode 100644 index 00000000000..6c8b1d80a90 --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/IndexBuilder.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.client.builder; + +import org.apache.hadoop.hive.metastore.api.Index; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Table; + +import java.util.HashMap; +import java.util.Map; + +/** + * Builder for indices. You must supply the database name and table name (or table reference), a + * name for the index, and whatever StorageDescriptorBuilder requires. All other fields will be + * given reasonable defaults. + */ +public class IndexBuilder extends StorageDescriptorBuilder<IndexBuilder> { + private String dbName, tableName, indexName, indexTableName, handlerClass; + private int createTime, lastAccessTime; + private Map<String, String> indexParams; + private boolean deferredRebuild; + + public IndexBuilder() { + // Set some reasonable defaults + indexParams = new HashMap<>(); + createTime = lastAccessTime = (int)(System.currentTimeMillis() / 1000); + super.setChild(this); + } + + public IndexBuilder setDbName(String dbName) { + this.dbName = dbName; + return this; + } + + public IndexBuilder setTableName(String tableName) { + this.tableName = tableName; + return this; + } + + public IndexBuilder setDbAndTableName(Table table) { + this.dbName = table.getDbName(); + this.tableName = table.getTableName(); + return this; + } + + public IndexBuilder setCreateTime(int createTime) { + this.createTime = createTime; + return this; + } + + public IndexBuilder setLastAccessTime(int lastAccessTime) { + this.lastAccessTime = lastAccessTime; + return this; + } + + public IndexBuilder setIndexParams(Map<String, String> indexParams) { + this.indexParams = indexParams; + return this; + } + + public IndexBuilder setIndexName(String indexName) { + this.indexName = indexName; + return this; + } + + public IndexBuilder setIndexTableName(String indexTableName) { + this.indexTableName = indexTableName; + return this; + } + + public IndexBuilder setHandlerClass(String handlerClass) { + this.handlerClass = handlerClass; + return this; + } + + public IndexBuilder setDeferredRebuild(boolean deferredRebuild) { + this.deferredRebuild = deferredRebuild; + return this; + } + + public Index build() throws MetaException { + if (dbName == null || tableName == null || indexName == null) { + throw new MetaException("You must provide database name, table name, and index name"); + } + if (indexTableName == null) indexTableName = tableName + "_" + indexName + "_table"; + return new Index(indexName, handlerClass, dbName, tableName, createTime, lastAccessTime, + indexTableName, buildSd(), indexParams, deferredRebuild); + } +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/PartitionBuilder.java b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/PartitionBuilder.java new file mode 100644 index 00000000000..265625f95ca --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/PartitionBuilder.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.client.builder; + +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Builder for {@link Partition}. The only requirements are 1. (database name and table name) or table + * reference; 2. partition values; 3. whatever {@link StorageDescriptorBuilder} requires. + */ +public class PartitionBuilder extends StorageDescriptorBuilder<PartitionBuilder> { + private String dbName, tableName; + private int createTime, lastAccessTime; + private Map<String, String> partParams; + private List<String> values; + + public PartitionBuilder() { + // Set some reasonable defaults + partParams = new HashMap<>(); + createTime = lastAccessTime = (int)(System.currentTimeMillis() / 1000); + super.setChild(this); + } + + public PartitionBuilder setDbName(String dbName) { + this.dbName = dbName; + return this; + } + + public PartitionBuilder setTableName(String tableName) { + this.tableName = tableName; + return this; + } + + public PartitionBuilder setDbAndTableName(Table table) { + this.dbName = table.getDbName(); + this.tableName = table.getTableName(); + return this; + } + + public PartitionBuilder setValues(List<String> values) { + this.values = values; + return this; + } + + public PartitionBuilder addValue(String value) { + if (values == null) values = new ArrayList<>(); + values.add(value); + return this; + } + + public PartitionBuilder setCreateTime(int createTime) { + this.createTime = createTime; + return this; + } + + public PartitionBuilder setLastAccessTime(int lastAccessTime) { + this.lastAccessTime = lastAccessTime; + return this; + } + + public PartitionBuilder setPartParams(Map<String, String> partParams) { + this.partParams = partParams; + return this; + } + + public PartitionBuilder addPartParam(String key, String value) { + if (partParams == null) partParams = new HashMap<>(); + partParams.put(key, value); + return this; + } + + public Partition build() throws MetaException { + if (dbName == null || tableName == null) { + throw new MetaException("database name and table name must be provided"); + } + if (values == null) throw new MetaException("You must provide partition values"); + return new Partition(values, dbName, tableName, createTime, lastAccessTime, buildSd(), + partParams); + } +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/PrivilegeGrantInfoBuilder.java b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/PrivilegeGrantInfoBuilder.java new file mode 100644 index 00000000000..213798e262d --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/PrivilegeGrantInfoBuilder.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.client.builder; + +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.PrincipalType; +import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo; +import org.apache.hadoop.hive.metastore.utils.SecurityUtils; + +import java.io.IOException; + +/** + * Builder for {@link PrivilegeGrantInfo}. The privilege is required. If not provided the grantor + * is + * assumed to be the current user. This is really intended for use by the + * {@link HiveObjectPrivilegeBuilder}. + */ +public class PrivilegeGrantInfoBuilder { + private String privilege, grantor; + private int createTime; + private PrincipalType grantorType; + private boolean grantOption; + + public PrivilegeGrantInfoBuilder() { + createTime = (int)(System.currentTimeMillis() / 1000); + grantOption = false; + } + + public PrivilegeGrantInfoBuilder setPrivilege(String privilege) { + this.privilege = privilege; + return this; + } + + public PrivilegeGrantInfoBuilder setGrantor(String grantor) { + this.grantor = grantor; + return this; + } + + public PrivilegeGrantInfoBuilder setCreateTime(int createTime) { + this.createTime = createTime; + return this; + } + + public PrivilegeGrantInfoBuilder setGrantorType(PrincipalType grantorType) { + this.grantorType = grantorType; + return this; + } + + public PrivilegeGrantInfoBuilder setGrantOption(boolean grantOption) { + this.grantOption = grantOption; + return this; + } + + public PrivilegeGrantInfo build() throws MetaException { + if (privilege == null) { + throw new MetaException("Privilege must be provided."); + } + if (grantor == null) { + try { + grantor = SecurityUtils.getUser(); + grantorType = PrincipalType.USER; + } catch (IOException e) { + throw new MetaException(e.getMessage()); + } + } + return new PrivilegeGrantInfo(privilege, createTime, grantor, grantorType, grantOption); + } +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/RoleBuilder.java b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/RoleBuilder.java new file mode 100644 index 00000000000..0b8d189f31a --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/RoleBuilder.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.client.builder; + +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Role; + +/** + * A builder for {@link Role}. The roleName and the ownerName must be provided. + */ +public class RoleBuilder { + private String roleName, ownerName; + private int createTime; + + public RoleBuilder() { + createTime = (int)(System.currentTimeMillis() / 1000); + } + + public RoleBuilder setRoleName(String roleName) { + this.roleName = roleName; + return this; + } + + public RoleBuilder setOwnerName(String ownerName) { + this.ownerName = ownerName; + return this; + } + + public RoleBuilder setCreateTime(int createTime) { + this.createTime = createTime; + return this; + } + + public Role build() throws MetaException { + if (roleName == null || ownerName == null) { + throw new MetaException("role name and owner name must be provided."); + } + return new Role(roleName, createTime, ownerName); + } +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/SQLForeignKeyBuilder.java b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/SQLForeignKeyBuilder.java new file mode 100644 index 00000000000..a39319a1e4d --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/SQLForeignKeyBuilder.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.client.builder; + +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.SQLForeignKey; +import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; + +/** + * Builder for {@link SQLForeignKey}. Requires what {@link ConstraintBuilder} requires, plus + * primary key + * database, table, column and name. + */ +public class SQLForeignKeyBuilder extends ConstraintBuilder<SQLForeignKeyBuilder> { + private String pkDb, pkTable, pkColumn, pkName; + private int updateRule, deleteRule; + + public SQLForeignKeyBuilder() { + updateRule = deleteRule = 0; + } + + public SQLForeignKeyBuilder setPkDb(String pkDb) { + this.pkDb = pkDb; + return this; + } + + public SQLForeignKeyBuilder setPkTable(String pkTable) { + this.pkTable = pkTable; + return this; + } + + public SQLForeignKeyBuilder setPkColumn(String pkColumn) { + this.pkColumn = pkColumn; + return this; + } + + public SQLForeignKeyBuilder setPkName(String pkName) { + this.pkName = pkName; + return this; + } + + public SQLForeignKeyBuilder setPrimaryKey(SQLPrimaryKey pk) { + pkDb = pk.getTable_db(); + pkTable = pk.getTable_name(); + pkColumn = pk.getColumn_name(); + pkName = pk.getPk_name(); + return this; + } + + public SQLForeignKeyBuilder setUpdateRule(int updateRule) { + this.updateRule = updateRule; + return this; + } + + public SQLForeignKeyBuilder setDeleteRule(int deleteRule) { + this.deleteRule = deleteRule; + return this; + } + + public SQLForeignKey build() throws MetaException { + checkBuildable("foreign_key"); + if (pkDb == null || pkTable == null || pkColumn == null || pkName == null) { + throw new MetaException("You must provide the primary key database, table, column, and name"); + } + return new SQLForeignKey(pkDb, pkTable, pkColumn, dbName, tableName, columnName, keySeq, + updateRule, deleteRule, constraintName, pkName, enable, validate, rely); + } +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/SQLPrimaryKeyBuilder.java b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/SQLPrimaryKeyBuilder.java new file mode 100644 index 00000000000..9000f861676 --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/SQLPrimaryKeyBuilder.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.client.builder; + +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; + +/** + * Builder for {@link SQLPrimaryKey}. Only requires what {@link ConstraintBuilder} requires. + */ +public class SQLPrimaryKeyBuilder extends ConstraintBuilder<SQLPrimaryKeyBuilder> { + + public SQLPrimaryKeyBuilder() { + super.setChild(this); + } + + // Just to translate + public SQLPrimaryKeyBuilder setPrimaryKeyName(String name) { + return setConstraintName(name); + } + + public SQLPrimaryKey build() throws MetaException { + checkBuildable("primary_key"); + return new SQLPrimaryKey(dbName, tableName, columnName, keySeq, constraintName, enable, + validate, rely); + } +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/StorageDescriptorBuilder.java b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/StorageDescriptorBuilder.java new file mode 100644 index 00000000000..39d1fa210ce --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/StorageDescriptorBuilder.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.client.builder; + +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Order; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.SkewedInfo; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Builds a {@link StorageDescriptor}. Only requires that columns be set. It picks reasonable + * defaults for everything else. This is intended for use just by objects that have a StorageDescriptor, + * not direct use. + */ +abstract class StorageDescriptorBuilder<T> { + private static final String SERDE_LIB = "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"; + private static final String INPUT_FORMAT = "org.apache.hadoop.hive.ql.io.HiveInputFormat"; + private static final String OUTPUT_FORMAT = "org.apache.hadoop.hive.ql.io.HiveOutputFormat"; + + private String location, inputFormat, outputFormat, serdeName, serdeLib; + private List<FieldSchema> cols; + private int numBuckets; + private Map<String, String> storageDescriptorParams, serdeParams; + private boolean compressed, storedAsSubDirectories; + private List<String> bucketCols, skewedColNames; + private List<Order> sortCols; + private List<List<String>> skewedColValues; + private Map<List<String>, String> skewedColValueLocationMaps; + // This enables us to return the correct type from the builder + private T child; + + protected StorageDescriptorBuilder() { + // Set some reasonable defaults + storageDescriptorParams = new HashMap<>(); + serdeParams = new HashMap<>(); + bucketCols = new ArrayList<>(); + sortCols = new ArrayList<>(); + numBuckets = 0; + compressed = false; + inputFormat = INPUT_FORMAT; + outputFormat = OUTPUT_FORMAT; + serdeLib = SERDE_LIB; + skewedColNames = new ArrayList<>(); + skewedColValues = new ArrayList<>(); + skewedColValueLocationMaps = new HashMap<>(); + } + + protected StorageDescriptor buildSd() throws MetaException { + if (cols == null) throw new MetaException("You must provide the columns"); + SerDeInfo serdeInfo = new SerDeInfo(serdeName, serdeLib, serdeParams); + StorageDescriptor sd = new StorageDescriptor(cols, location, inputFormat, outputFormat, + compressed, numBuckets, serdeInfo, bucketCols, sortCols, storageDescriptorParams); + sd.setStoredAsSubDirectories(storedAsSubDirectories); + if (skewedColNames != null) { + SkewedInfo skewed = new SkewedInfo(skewedColNames, skewedColValues, + skewedColValueLocationMaps); + sd.setSkewedInfo(skewed); + } + return sd; + } + + protected void setChild(T child) { + this.child = child; + } + + public T setLocation(String location) { + this.location = location; + return child; + } + + public T setInputFormat(String inputFormat) { + this.inputFormat = inputFormat; + return child; + } + + public T setOutputFormat(String outputFormat) { + this.outputFormat = outputFormat; + return child; + } + + public T setSerdeName(String serdeName) { + this.serdeName = serdeName; + return child; + } + + public T setSerdeLib(String serdeLib) { + this.serdeLib = serdeLib; + return child; + } + public T setCols(List<FieldSchema> cols) { + this.cols = cols; + return child; + } + + public T addCol(String name, String type, String comment) { + if (cols == null) cols = new ArrayList<>(); + cols.add(new FieldSchema(name, type, comment)); + return child; + } + + public T addCol(String name, String type) { + return addCol(name, type, ""); + } + + public T setNumBuckets(int numBuckets) { + this.numBuckets = numBuckets; + return child; + } + + public T setStorageDescriptorParams( + Map<String, String> storageDescriptorParams) { + this.storageDescriptorParams = storageDescriptorParams; + return child; + } + + public T addStorageDescriptorParam(String key, String value) { + if (storageDescriptorParams == null) storageDescriptorParams = new HashMap<>(); + storageDescriptorParams.put(key, value); + return child; + } + + public T setSerdeParams(Map<String, String> serdeParams) { + this.serdeParams = serdeParams; + return child; + } + + public T addSerdeParam(String key, String value) { + if (serdeParams == null) serdeParams = new HashMap<>(); + serdeParams.put(key, value); + return child; + } + + public T setCompressed(boolean compressed) { + this.compressed = compressed; + return child; + } + + public T setStoredAsSubDirectories(boolean storedAsSubDirectories) { + this.storedAsSubDirectories = storedAsSubDirectories; + return child; + } + + public T setBucketCols(List<String> bucketCols) { + this.bucketCols = bucketCols; + return child; + } + + public T addBucketCol(String bucketCol) { + if (bucketCols == null) bucketCols = new ArrayList<>(); + bucketCols.add(bucketCol); + return child; + } + + public T setSkewedColNames(List<String> skewedColNames) { + this.skewedColNames = skewedColNames; + return child; + } + + public T addSkewedColName(String skewedColName) { + if (skewedColNames == null) skewedColNames = new ArrayList<>(); + skewedColNames.add(skewedColName); + return child; + } + + public T setSortCols(List<Order> sortCols) { + this.sortCols = sortCols; + return child; + } + + public T addSortCol(String col, int order) { + if (sortCols == null) sortCols = new ArrayList<>(); + sortCols.add(new Order(col, order)); + return child; + } + + // It is not at all clear how to flatten these last two out in a useful way, and no one uses + // these anyway. + public T setSkewedColValues(List<List<String>> skewedColValues) { + this.skewedColValues = skewedColValues; + return child; + } + + public T setSkewedColValueLocationMaps( + Map<List<String>, String> skewedColValueLocationMaps) { + this.skewedColValueLocationMaps = skewedColValueLocationMaps; + return child; + } +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/TableBuilder.java b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/TableBuilder.java new file mode 100644 index 00000000000..0b005858c06 --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/client/builder/TableBuilder.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.client.builder; + +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.utils.SecurityUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Build a {@link Table}. The database name and table name must be provided, plus whatever is + * needed by the underlying {@link StorageDescriptorBuilder}. + */ +public class TableBuilder extends StorageDescriptorBuilder<TableBuilder> { + private String dbName, tableName, owner, viewOriginalText, viewExpandedText, type; + private List<FieldSchema> partCols; + private int createTime, lastAccessTime, retention; + private Map<String, String> tableParams; + private boolean rewriteEnabled, temporary; + + public TableBuilder() { + // Set some reasonable defaults + tableParams = new HashMap<>(); + createTime = lastAccessTime = (int)(System.currentTimeMillis() / 1000); + retention = 0; + super.setChild(this); + } + + public TableBuilder setDbName(String dbName) { + this.dbName = dbName; + return this; + } + + public TableBuilder setDbName(Database db) { + this.dbName = db.getName(); + return this; + } + + public TableBuilder setTableName(String tableName) { + this.tableName = tableName; + return this; + } + + public TableBuilder setOwner(String owner) { + this.owner = owner; + return this; + } + + public TableBuilder setViewOriginalText(String viewOriginalText) { + this.viewOriginalText = viewOriginalText; + return this; + } + + public TableBuilder setViewExpandedText(String viewExpandedText) { + this.viewExpandedText = viewExpandedText; + return this; + } + + public TableBuilder setType(String type) { + this.type = type; + return this; + } + + public TableBuilder setPartCols(List<FieldSchema> partCols) { + this.partCols = partCols; + return this; + } + + public TableBuilder addPartCol(String name, String type, String comment) { + if (partCols == null) partCols = new ArrayList<>(); + partCols.add(new FieldSchema(name, type, comment)); + return this; + } + + public TableBuilder addPartCol(String name, String type) { + return addPartCol(name, type, ""); + } + + public TableBuilder setCreateTime(int createTime) { + this.createTime = createTime; + return this; + } + + public TableBuilder setLastAccessTime(int lastAccessTime) { + this.lastAccessTime = lastAccessTime; + return this; + } + + public TableBuilder setRetention(int retention) { + this.retention = retention; + return this; + } + + public TableBuilder setTableParams(Map<String, String> tableParams) { + this.tableParams = tableParams; + return this; + } + + public TableBuilder addTableParam(String key, String value) { + if (tableParams == null) tableParams = new HashMap<>(); + tableParams.put(key, value); + return this; + } + + public TableBuilder setRewriteEnabled(boolean rewriteEnabled) { + this.rewriteEnabled = rewriteEnabled; + return this; + } + + public TableBuilder setTemporary(boolean temporary) { + this.temporary = temporary; + return this; + } + + public Table build() throws MetaException { + if (dbName == null || tableName == null) { + throw new MetaException("You must set the database and table name"); + } + if (owner == null) { + try { + owner = SecurityUtils.getUser(); + } catch (IOException e) { + throw new MetaException(e.getMessage()); + } + } + Table t = new Table(tableName, dbName, owner, createTime, lastAccessTime, retention, buildSd(), + partCols, tableParams, viewOriginalText, viewExpandedText, type); + if (rewriteEnabled) t.setRewriteEnabled(true); + if (temporary) t.setTemporary(temporary); + return t; + } + +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java index 83cefb73d19..17f296066e6 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java @@ -130,6 +130,12 @@ public class HBaseStore implements RawStore { return true; } + + @Override + public boolean openTransaction(String isolationLevel) { + throw new UnsupportedOperationException("Not supported"); + } + @Override public boolean commitTransaction() { if (--txnNestLevel == 0) { diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java new file mode 100644 index 00000000000..77ef4543413 --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.utils; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.thrift.DBTokenStore; +import org.apache.hadoop.hive.thrift.DelegationTokenIdentifier; +import org.apache.hadoop.hive.thrift.DelegationTokenSelector; +import org.apache.hadoop.hive.thrift.MemoryTokenStore; +import org.apache.hadoop.hive.thrift.ZooKeeperTokenStore; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosUtil; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.TokenSelector; +import org.apache.zookeeper.client.ZooKeeperSaslClient; + +import javax.security.auth.login.AppConfigurationEntry; +import org.apache.thrift.transport.TSSLTransportFactory; +import org.apache.thrift.transport.TServerSocket; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLServerSocket; +import javax.net.ssl.SSLSocket; +import javax.security.auth.login.LoginException; +import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class SecurityUtils { + private static final Logger LOG = LoggerFactory.getLogger(SecurityUtils.class); + + public static UserGroupInformation getUGI() throws LoginException, IOException { + String doAs = System.getenv("HADOOP_USER_NAME"); + if (doAs != null && doAs.length() > 0) { + /* + * this allows doAs (proxy user) to be passed along across process boundary where + * delegation tokens are not supported. For example, a DDL stmt via WebHCat with + * a doAs parameter, forks to 'hcat' which needs to start a Session that + * proxies the end user + */ + return UserGroupInformation.createProxyUser(doAs, UserGroupInformation.getLoginUser()); + } + return UserGroupInformation.getCurrentUser(); + } + /** + * Dynamically sets up the JAAS configuration that uses kerberos + * @param principal + * @param keyTabFile + * @throws IOException + */ + public static void setZookeeperClientKerberosJaasConfig(String principal, String keyTabFile) throws IOException { + // ZooKeeper property name to pick the correct JAAS conf section + final String SASL_LOGIN_CONTEXT_NAME = "HiveZooKeeperClient"; + System.setProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY, SASL_LOGIN_CONTEXT_NAME); + + principal = SecurityUtil.getServerPrincipal(principal, "0.0.0.0"); + JaasConfiguration jaasConf = new JaasConfiguration(SASL_LOGIN_CONTEXT_NAME, principal, keyTabFile); + + // Install the Configuration in the runtime. + javax.security.auth.login.Configuration.setConfiguration(jaasConf); + } + + /** + * A JAAS configuration for ZooKeeper clients intended to use for SASL + * Kerberos. + */ + private static class JaasConfiguration extends javax.security.auth.login.Configuration { + // Current installed Configuration + private static final boolean IBM_JAVA = System.getProperty("java.vendor") + .contains("IBM"); + private final javax.security.auth.login.Configuration baseConfig = javax.security.auth.login.Configuration + .getConfiguration(); + private final String loginContextName; + private final String principal; + private final String keyTabFile; + + public JaasConfiguration(String hiveLoginContextName, String principal, String keyTabFile) { + this.loginContextName = hiveLoginContextName; + this.principal = principal; + this.keyTabFile = keyTabFile; + } + + @Override + public AppConfigurationEntry[] getAppConfigurationEntry(String appName) { + if (loginContextName.equals(appName)) { + Map<String, String> krbOptions = new HashMap<String, String>(); + if (IBM_JAVA) { + krbOptions.put("credsType", "both"); + krbOptions.put("useKeytab", keyTabFile); + } else { + krbOptions.put("doNotPrompt", "true"); + krbOptions.put("storeKey", "true"); + krbOptions.put("useKeyTab", "true"); + krbOptions.put("keyTab", keyTabFile); + } + krbOptions.put("principal", principal); + krbOptions.put("refreshKrb5Config", "true"); + AppConfigurationEntry hiveZooKeeperClientEntry = new AppConfigurationEntry( + KerberosUtil.getKrb5LoginModuleName(), LoginModuleControlFlag.REQUIRED, krbOptions); + return new AppConfigurationEntry[] { hiveZooKeeperClientEntry }; + } + // Try the base config + if (baseConfig != null) { + return baseConfig.getAppConfigurationEntry(appName); + } + return null; + } + } + + /** + * Get the string form of the token given a token signature. The signature is used as the value of + * the "service" field in the token for lookup. Ref: AbstractDelegationTokenSelector in Hadoop. If + * there exists such a token in the token cache (credential store) of the job, the lookup returns + * that. This is relevant only when running against a "secure" hadoop release The method gets hold + * of the tokens if they are set up by hadoop - this should happen on the map/reduce tasks if the + * client added the tokens into hadoop's credential store in the front end during job submission. + * The method will select the hive delegation token among the set of tokens and return the string + * form of it + * + * @param tokenSignature + * @return the string form of the token found + * @throws IOException + */ + public static String getTokenStrForm(String tokenSignature) throws IOException { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + TokenSelector<? extends TokenIdentifier> tokenSelector = new DelegationTokenSelector(); + + Token<? extends TokenIdentifier> token = tokenSelector.selectToken( + tokenSignature == null ? new Text() : new Text(tokenSignature), ugi.getTokens()); + return token != null ? token.encodeToUrlString() : null; + } + + /** + * Create a delegation token object for the given token string and service. Add the token to given + * UGI + * + * @param ugi + * @param tokenStr + * @param tokenService + * @throws IOException + */ + public static void setTokenStr(UserGroupInformation ugi, String tokenStr, String tokenService) + throws IOException { + Token<DelegationTokenIdentifier> delegationToken = createToken(tokenStr, tokenService); + ugi.addToken(delegationToken); + } + + /** + * Create a new token using the given string and service + * + * @param tokenStr + * @param tokenService + * @return + * @throws IOException + */ + private static Token<DelegationTokenIdentifier> createToken(String tokenStr, String tokenService) + throws IOException { + Token<DelegationTokenIdentifier> delegationToken = new Token<DelegationTokenIdentifier>(); + delegationToken.decodeFromUrlString(tokenStr); + delegationToken.setService(new Text(tokenService)); + return delegationToken; + } + + private static final String DELEGATION_TOKEN_STORE_CLS = "hive.cluster.delegation.token.store.class"; + + /** + * This method should be used to return the metastore specific tokenstore class name to main + * backwards compatibility + * + * @param conf - HiveConf object + * @return the tokenStoreClass name from the HiveConf. It maps the hive specific tokenstoreclass + * name to metastore module specific class name. For eg: + * hive.cluster.delegation.token.store.class is set to + * org.apache.hadoop.hive.thrift.MemoryTokenStore it returns the equivalent tokenstore + * class defined in the metastore module which is + * org.apache.hadoop.hive.metastore.security.MemoryTokenStore Similarly, + * org.apache.hadoop.hive.thrift.DBTokenStore maps to + * org.apache.hadoop.hive.metastore.security.DBTokenStore and + * org.apache.hadoop.hive.thrift.ZooKeeperTokenStore maps to + * org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore + */ + public static String getTokenStoreClassName(Configuration conf) { + String tokenStoreClass = conf.get(DELEGATION_TOKEN_STORE_CLS, ""); + if (StringUtils.isBlank(tokenStoreClass)) { + // default tokenstore is MemoryTokenStore + return MemoryTokenStore.class.getName(); + } + switch (tokenStoreClass) { + case "org.apache.hadoop.hive.thrift.DBTokenStore": + return DBTokenStore.class.getName(); + case "org.apache.hadoop.hive.thrift.MemoryTokenStore": + return MemoryTokenStore.class.getName(); + case "org.apache.hadoop.hive.thrift.ZooKeeperTokenStore": + return ZooKeeperTokenStore.class.getName(); + default: + return tokenStoreClass; + } + } + + + /** + * @return the user name set in hadoop.job.ugi param or the current user from System + * @throws IOException if underlying Hadoop call throws LoginException + */ + public static String getUser() throws IOException { + try { + UserGroupInformation ugi = getUGI(); + return ugi.getUserName(); + } catch (LoginException le) { + throw new IOException(le); + } + } + + public static TServerSocket getServerSocket(String hiveHost, int portNum) throws TTransportException { + InetSocketAddress serverAddress; + if (hiveHost == null || hiveHost.isEmpty()) { + // Wildcard bind + serverAddress = new InetSocketAddress(portNum); + } else { + serverAddress = new InetSocketAddress(hiveHost, portNum); + } + return new TServerSocket(serverAddress); + } + + public static TServerSocket getServerSSLSocket(String hiveHost, int portNum, String keyStorePath, + String keyStorePassWord, List<String> sslVersionBlacklist) throws TTransportException, + UnknownHostException { + TSSLTransportFactory.TSSLTransportParameters params = + new TSSLTransportFactory.TSSLTransportParameters(); + params.setKeyStore(keyStorePath, keyStorePassWord); + InetSocketAddress serverAddress; + if (hiveHost == null || hiveHost.isEmpty()) { + // Wildcard bind + serverAddress = new InetSocketAddress(portNum); + } else { + serverAddress = new InetSocketAddress(hiveHost, portNum); + } + TServerSocket thriftServerSocket = + TSSLTransportFactory.getServerSocket(portNum, 0, serverAddress.getAddress(), params); + if (thriftServerSocket.getServerSocket() instanceof SSLServerSocket) { + List<String> sslVersionBlacklistLocal = new ArrayList<>(); + for (String sslVersion : sslVersionBlacklist) { + sslVersionBlacklistLocal.add(sslVersion.trim().toLowerCase()); + } + SSLServerSocket sslServerSocket = (SSLServerSocket) thriftServerSocket.getServerSocket(); + List<String> enabledProtocols = new ArrayList<>(); + for (String protocol : sslServerSocket.getEnabledProtocols()) { + if (sslVersionBlacklistLocal.contains(protocol.toLowerCase())) { + LOG.debug("Disabling SSL Protocol: " + protocol); + } else { + enabledProtocols.add(protocol); + } + } + sslServerSocket.setEnabledProtocols(enabledProtocols.toArray(new String[0])); + LOG.info("SSL Server Socket Enabled Protocols: " + + Arrays.toString(sslServerSocket.getEnabledProtocols())); + } + return thriftServerSocket; + } + + public static TTransport getSSLSocket(String host, int port, int loginTimeout, + String trustStorePath, String trustStorePassWord) throws TTransportException { + TSSLTransportFactory.TSSLTransportParameters params = + new TSSLTransportFactory.TSSLTransportParameters(); + params.setTrustStore(trustStorePath, trustStorePassWord); + params.requireClientAuth(true); + // The underlying SSLSocket object is bound to host:port with the given SO_TIMEOUT and + // SSLContext created with the given params + TSocket tSSLSocket = TSSLTransportFactory.getClientSocket(host, port, loginTimeout, params); + return getSSLSocketWithHttps(tSSLSocket); + } + + // Using endpoint identification algorithm as HTTPS enables us to do + // CNAMEs/subjectAltName verification + private static TSocket getSSLSocketWithHttps(TSocket tSSLSocket) throws TTransportException { + SSLSocket sslSocket = (SSLSocket) tSSLSocket.getSocket(); + SSLParameters sslParams = sslSocket.getSSLParameters(); + sslParams.setEndpointIdentificationAlgorithm("HTTPS"); + sslSocket.setSSLParameters(sslParams); + return new TSocket(sslSocket); + } +} \ No newline at end of file diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index 2babda20627..ec3b2376dd7 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -123,6 +123,11 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { return objectStore.openTransaction(); } + @Override + public boolean openTransaction(String isolationLevel) { + return objectStore.openTransaction(isolationLevel); + } + @Override public void rollbackTransaction() { objectStore.rollbackTransaction(); diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 52391bea6b3..a0f071a042b 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -103,6 +103,12 @@ public class DummyRawStoreForJdoConnection implements RawStore { return false; } + + @Override + public boolean openTransaction(String isolationLevel) { + return false; + } + @Override public boolean commitTransaction() { diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/client/MetaStoreFactoryForTests.java b/metastore/src/test/org/apache/hadoop/hive/metastore/client/MetaStoreFactoryForTests.java new file mode 100644 index 00000000000..e57c6a1f670 --- /dev/null +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/client/MetaStoreFactoryForTests.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.client; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.DefaultPartitionExpressionProxy; +import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService; +import org.apache.hadoop.hive.metastore.minihms.MiniHMS; + +import java.util.ArrayList; +import java.util.List; + +/** + * Factory for creating specific + * {@link org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService} implementations for + * tests. + */ +public class MetaStoreFactoryForTests { + private static final int DEFAULT_LIMIT_PARTITION_REQUEST = 100; + + /** + * We would like to run the tests with 2 MetaStore configurations + * - Embedded - Where the MetaStore is running in the same thread, and does not use Thrift + * - Remote - Where the MetaStore is started in a different thread, and uses Thrift for + * communication + * + * Or if the test.hms.client.configs system property is set, it would return a single test + * MetaStoreService which uses these configs. In this case the MetaStore should be created + * manually or by an external application. + * @return The list of the test MetaStoreService implementations usable by @Parameterized + * .Parameters + */ + public static List<Object[]> getMetaStores() throws Exception { + List<Object[]> metaStores = new ArrayList<Object[]>(); + + HiveConf conf = new HiveConf(); + // set some values to use for getting conf. vars + HiveConf.setBoolVar(conf, HiveConf.ConfVars.METASTORE_METRICS, true); + HiveConf.setIntVar(conf, HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX, 2); + HiveConf.setIntVar(conf, HiveConf.ConfVars.METASTORE_LIMIT_PARTITION_REQUEST, + DEFAULT_LIMIT_PARTITION_REQUEST); + HiveConf.setVar(conf, HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS, + DefaultPartitionExpressionProxy.class.getName()); + + // Do this only on your own peril, and never in the production code + conf.set("datanucleus.autoCreateTables", "false"); + + // Example for using cluster configuration xml-s + // -Dtest.hms.client.configs=/tmp/conf/core-site.xml,/tmp/conf/hive-site.xml + String testHMSClientConfiguration = System.getProperty("test.hms.client.configs"); + if (testHMSClientConfiguration != null) { + HiveConf clusterConf = new HiveConf(conf); + // Loading the extra configuration options + String[] configurationFiles = testHMSClientConfiguration.split(","); + for(String configurationFile : configurationFiles) { + clusterConf.addResource(new Path(configurationFile)); + } + + // Using MetaStore running in an existing cluster + AbstractMetaStoreService cluster = + new MiniHMS.Builder() + .setConf(clusterConf) + .setType(MiniHMS.MiniHMSType.CLUSTER) + .build(); + metaStores.add(new Object[]{"Cluster", cluster}); + } + + // Create Embedded MetaStore + conf.set("javax.jdo.option.ConnectionURL", + "jdbc:derby:memory:${test.tmp.dir}/junit_metastore_db1;create=true"); + AbstractMetaStoreService embedded = + new MiniHMS.Builder() + .setConf(conf) + .setType(MiniHMS.MiniHMSType.EMBEDDED) + .build(); + metaStores.add(new Object[] { "Embedded", embedded}); + + // Create Remote MetaStore + conf.set("javax.jdo.option.ConnectionURL", + "jdbc:derby:memory:${test.tmp.dir}/junit_metastore_db2;create=true"); + AbstractMetaStoreService remote = + new MiniHMS.Builder() + .setConf(conf) + .setType(MiniHMS.MiniHMSType.REMOTE) + .build(); + metaStores.add(new Object[] { "Remote", remote}); + + return metaStores; + } +} diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/client/TestDatabases.java b/metastore/src/test/org/apache/hadoop/hive/metastore/client/TestDatabases.java new file mode 100644 index 00000000000..2b41881f534 --- /dev/null +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/client/TestDatabases.java @@ -0,0 +1,622 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.client; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Function; +import org.apache.hadoop.hive.metastore.api.Index; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.PrincipalType; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; +import org.apache.hadoop.hive.metastore.client.builder.FunctionBuilder; +import org.apache.hadoop.hive.metastore.client.builder.IndexBuilder; +import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; +import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService; +import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransportException; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +@RunWith(Parameterized.class) +public class TestDatabases { + private static final Logger LOG = LoggerFactory.getLogger(TestDatabases.class); + // Needed until there is no junit release with @BeforeParam, @AfterParam (junit 4.13) + // https://github.com/junit-team/junit4/commit/1bf8438b65858565dbb64736bfe13aae9cfc1b5a + // Then we should remove our own copy + private static Set<AbstractMetaStoreService> metaStoreServices = null; + private static final String DEFAULT_DATABASE = "default"; + private final AbstractMetaStoreService metaStore; + private IMetaStoreClient client; + private Database[] testDatabases = new Database[4]; + + @Parameterized.Parameters(name = "{0}") + public static List<Object[]> getMetaStoreToTest() throws Exception { + List<Object[]> result = MetaStoreFactoryForTests.getMetaStores(); + metaStoreServices = new HashSet<>(result.size()); + for(Object[] test: result) { + metaStoreServices.add((AbstractMetaStoreService) test[1]); + } + return result; + } + + public TestDatabases(String name, AbstractMetaStoreService metaStore) throws Exception { + this.metaStore = metaStore; + this.metaStore.start(); + } + + // Needed until there is no junit release with @BeforeParam, @AfterParam (junit 4.13) + // https://github.com/junit-team/junit4/commit/1bf8438b65858565dbb64736bfe13aae9cfc1b5a + // Then we should move this to @AfterParam + @AfterClass + public static void stopMetaStores() throws Exception { + for(AbstractMetaStoreService metaStoreService : metaStoreServices) { + try { + metaStoreService.stop(); + } catch(Exception e) { + // Catch the exceptions, so every other metastore could be stopped as well + // Log it, so at least there is a slight possibility we find out about this :) + LOG.error("Error stopping MetaStoreService", e); + } + } + } + + @Before + public void setUp() throws Exception { + // Get new client + client = metaStore.getClient(); + + // Clean up the databases + for(String databaseName : client.getAllDatabases()) { + if (!databaseName.equals(DEFAULT_DATABASE)) { + client.dropDatabase(databaseName, true, true, true); + } + } + + testDatabases[0] = + new DatabaseBuilder().setName("test_database_1").build(); + testDatabases[1] = + new DatabaseBuilder().setName("test_database_to_find_1").build(); + testDatabases[2] = + new DatabaseBuilder().setName("test_database_to_find_2").build(); + testDatabases[3] = + new DatabaseBuilder().setName("test_database_hidden_1").build(); + + // Create the databases, and reload them from the MetaStore + for(int i=0; i < testDatabases.length; i++) { + client.createDatabase(testDatabases[i]); + testDatabases[i] = client.getDatabase(testDatabases[i].getName()); + } + } + + @After + public void tearDown() throws Exception { + try { + if (client != null) { + client.close(); + } + } finally { + client = null; + } + } + + /** + * This test creates and queries a database and then drops it. Good for testing the happy path. + * @throws Exception + */ + @Test + public void testCreateGetDeleteDatabase() throws Exception { + Database database = getDatabaseWithAllParametersSet(); + client.createDatabase(database); + Database createdDatabase = client.getDatabase(database.getName()); + + // The createTime will be set on the server side, so the comparison should skip it + Assert.assertEquals("Comparing databases", database, createdDatabase); + Assert.assertTrue("The directory should be created", metaStore.isPathExists( + new Path(database.getLocationUri()))); + client.dropDatabase(database.getName()); + Assert.assertFalse("The directory should be removed", + metaStore.isPathExists(new Path(database.getLocationUri()))); + try { + client.getDatabase(database.getName()); + Assert.fail("Expected a NoSuchObjectException to be thrown"); + } catch (NoSuchObjectException exception) { + // Expected exception + } + } + + @Test + public void testCreateDatabaseDefaultValues() throws Exception { + Database database = new Database(); + database.setName("dummy"); + + client.createDatabase(database); + Database createdDatabase = client.getDatabase(database.getName()); + + Assert.assertNull("Comparing description", createdDatabase.getDescription()); + Assert.assertEquals("Comparing location", metaStore.getWarehouseRoot() + "/" + + createdDatabase.getName() + ".db", createdDatabase.getLocationUri()); + Assert.assertEquals("Comparing parameters", new HashMap<String, String>(), + createdDatabase.getParameters()); + Assert.assertNull("Comparing privileges", createdDatabase.getPrivileges()); + Assert.assertNull("Comparing owner name", createdDatabase.getOwnerName()); + Assert.assertEquals("Comparing owner type", PrincipalType.USER, createdDatabase.getOwnerType()); + } + + @Test(expected = MetaException.class) + public void testCreateDatabaseNullName() throws Exception { + Database database = testDatabases[0]; + + // Missing class setting field + database.setName(null); + + client.createDatabase(database); + // Throwing InvalidObjectException would be more appropriate, but we do not change the API + } + + @Test(expected = InvalidObjectException.class) + public void testCreateDatabaseInvalidName() throws Exception { + Database database = testDatabases[0]; + + // Invalid character in new database name + database.setName("test_database_1;"); + client.createDatabase(database); + } + + @Test(expected = InvalidObjectException.class) + public void testCreateDatabaseEmptyName() throws Exception { + Database database = testDatabases[0]; + + // Empty new database name + database.setName(""); + client.createDatabase(database); + // Throwing InvalidObjectException would be more appropriate, but we do not change the API + } + + @Test(expected = AlreadyExistsException.class) + public void testCreateDatabaseAlreadyExists() throws Exception { + Database database = testDatabases[0]; + + // Already existing database + client.createDatabase(database); + } + + @Test + public void testDefaultDatabaseData() throws Exception { + Database database = client.getDatabase(DEFAULT_DATABASE); + Assert.assertEquals("Default database name", "default", database.getName()); + Assert.assertEquals("Default database description", "Default Hive database", + database.getDescription()); + Assert.assertEquals("Default database location", metaStore.getWarehouseRoot(), + new Path(database.getLocationUri())); + Assert.assertEquals("Default database parameters", new HashMap<String, String>(), + database.getParameters()); + Assert.assertEquals("Default database owner", "public", database.getOwnerName()); + Assert.assertEquals("Default database owner type", PrincipalType.ROLE, database.getOwnerType()); + Assert.assertNull("Default database privileges", database.getPrivileges()); + } + + @Test + public void testGetDatabaseCaseInsensitive() throws Exception { + Database database = testDatabases[0]; + + // Test in upper case + Database resultUpper = client.getDatabase(database.getName().toUpperCase()); + Assert.assertEquals("Comparing databases", database, resultUpper); + + // Test in mixed case + Database resultMix = client.getDatabase("teST_dAtABase_1"); + Assert.assertEquals("Comparing databases", database, resultMix); + } + + @Test(expected = NoSuchObjectException.class) + public void testGetDatabaseNoSuchDatabase() throws Exception { + client.getDatabase("no_such_database"); + } + + @Test + public void testGetDatabaseNullName() throws Exception { + // Missing database name in the query + try { + client.getDatabase(null); + // TODO: Should have a check on the server side. + Assert.fail("Expected a NullPointerException or TTransportException to be thrown"); + } catch (NullPointerException exception) { + // Expected exception - Embedded MetaStore + } catch (TTransportException exception) { + // Expected exception - Remote MetaStore + } + } + + @Test(expected = NoSuchObjectException.class) + public void testDropDatabaseNoSuchDatabase() throws Exception { + client.dropDatabase("no_such_database"); + } + + @Test + public void testDropDatabaseNullName() throws Exception { + // Missing database in the query + try { + client.dropDatabase(null); + // TODO: Should be checked on server side + Assert.fail("Expected an NullPointerException or TTransportException to be thrown"); + } catch (NullPointerException exception) { + // Expected exception - Embedded MetaStore + } catch (TTransportException exception) { + // Expected exception - Remote MetaStore + } + } + + @Test + public void testDropDatabaseDefaultDatabase() throws Exception { + // Check if it is possible to drop default database + try { + client.dropDatabase(DEFAULT_DATABASE); + // TODO: Should be checked on server side + Assert.fail("Expected an MetaException or TTransportException to be thrown"); + } catch (MetaException exception) { + // Expected exception - Embedded MetaStore + } catch (TTransportException exception) { + // Expected exception - Remote MetaStore + } + } + + @Test + public void testDropDatabaseCaseInsensitive() throws Exception { + Database database = testDatabases[0]; + + // Test in upper case + client.dropDatabase(database.getName().toUpperCase()); + List<String> allDatabases = client.getAllDatabases(); + Assert.assertEquals("All databases size", 4, allDatabases.size()); + + // Test in mixed case + client.createDatabase(database); + client.dropDatabase("TesT_DatABaSe_1"); + allDatabases = client.getAllDatabases(); + Assert.assertEquals("All databases size", 4, allDatabases.size()); + } + + @Test + public void testDropDatabaseDeleteData() throws Exception { + Database database = testDatabases[0]; + Path dataFile = new Path(database.getLocationUri().toString() + "/dataFile"); + metaStore.createFile(dataFile, "100"); + + // Do not delete the data + client.dropDatabase(database.getName(), false, false); + // Check that the data still exist + Assert.assertTrue("The data file should still exist", metaStore.isPathExists(dataFile)); + + // Recreate the database + client.createDatabase(database); + Assert.assertTrue("The data file should still exist", metaStore.isPathExists(dataFile)); + + // Delete the data + client.dropDatabase(database.getName(), true, false); + // Check that the data is removed + Assert.assertFalse("The data file should not exist", metaStore.isPathExists(dataFile)); + } + + @Test(expected = NoSuchObjectException.class) + public void testDropDatabaseIgnoreUnknownFalse() throws Exception { + // No such database + client.dropDatabase("no_such_database", false, false); + } + + @Test + public void testDropDatabaseIgnoreUnknownTrue() throws Exception { + // No such database + client.dropDatabase("no_such_database", false, true); + } + + @Test(expected = InvalidOperationException.class) + public void testDropDatabaseWithTable() throws Exception { + Database database = testDatabases[0]; + Table testTable = + new TableBuilder() + .setDbName(database.getName()) + .setTableName("test_table") + .addCol("test_col", "int") + .build(); + client.createTable(testTable); + + client.dropDatabase(database.getName(), true, true, false); + } + + @Test + public void testDropDatabaseWithTableCascade() throws Exception { + Database database = testDatabases[0]; + Table testTable = + new TableBuilder() + .setDbName(database.getName()) + .setTableName("test_table") + .addCol("test_col", "int") + .build(); + client.createTable(testTable); + + client.dropDatabase(database.getName(), true, true, true); + Assert.assertFalse("The directory should be removed", + metaStore.isPathExists(new Path(database.getLocationUri()))); + } + + @Test(expected = InvalidOperationException.class) + public void testDropDatabaseWithFunction() throws Exception { + Database database = testDatabases[0]; + + Function testFunction = + new FunctionBuilder() + .setDbName(database.getName()) + .setName("test_function") + .setClass("org.apache.hadoop.hive.ql.udf.generic.GenericUDFUpper") + .build(); + + client.createFunction(testFunction); + + client.dropDatabase(database.getName(), true, true, false); + } + + @Test + public void testDropDatabaseWithFunctionCascade() throws Exception { + Database database = testDatabases[0]; + + Function testFunction = + new FunctionBuilder() + .setDbName(database.getName()) + .setName("test_function") + .setClass("org.apache.hadoop.hive.ql.udf.generic.GenericUDFUpper") + .build(); + + client.createFunction(testFunction); + + client.dropDatabase(database.getName(), true, true, true); + Assert.assertFalse("The directory should be removed", + metaStore.isPathExists(new Path(database.getLocationUri()))); + } + + /** + * Creates an index in the given database for testing purposes + * @param databaseName The database name in which the index should be creatd + * @throws TException If there is an error during the index creation + */ + private void createIndex(String databaseName) throws TException { + Table testTable = + new TableBuilder() + .setDbName(databaseName) + .setTableName("test_table") + .addCol("test_col", "int") + .build(); + + Index testIndex = + new IndexBuilder() + .setIndexName("test_index") + .setIndexTableName("test_index_table") + .setDbAndTableName(testTable) + .addCol("test_col", "int") + .build(); + Table testIndexTable = + new TableBuilder() + .setDbName(databaseName) + .setTableName("test_index_table") + .addCol("test_col", "int") + .build(); + + // Drop database with index + client.createTable(testTable); + client.createIndex(testIndex, testIndexTable); + } + + @Test + public void testDropDatabaseWithIndex() throws Exception { + Database database = testDatabases[0]; + createIndex(database.getName()); + + // TODO: Known error, should be fixed + // client.dropDatabase(database.getName(), true, true, true); + // Need to drop index to clean up the mess + try { + // Without cascade + client.dropDatabase(database.getName(), true, true, false); + Assert.fail("Expected an InvalidOperationException to be thrown"); + } catch (InvalidOperationException exception) { + // Expected exception + } + client.dropIndex(database.getName(), "test_table", "test_index", true); + // TODO: End index hack + } + + @Test + public void testDropDatabaseWithIndexCascade() throws Exception { + Database database = testDatabases[0]; + createIndex(database.getName()); + + // With cascade + // TODO: Known error, should be fixed + // client.dropDatabase(database.getName(), true, true, true); + // Need to drop index to clean up the mess + client.dropIndex(database.getName(), "test_table", "test_index", true); + client.dropDatabase(database.getName(), true, true, true); + Assert.assertFalse("The directory should be removed", + metaStore.isPathExists(new Path(database.getLocationUri()))); + } + + @Test + public void testGetAllDatabases() throws Exception { + List<String> allDatabases = client.getAllDatabases(); + Assert.assertEquals("All databases size", 5, allDatabases.size()); + for(Database database : testDatabases) { + Assert.assertTrue("Checking database names", allDatabases.contains(database.getName())); + } + Assert.assertTrue("Checnking that default database is returned", + allDatabases.contains(DEFAULT_DATABASE)); + + // Drop one database, see what remains + client.dropDatabase(testDatabases[1].getName()); + allDatabases = client.getAllDatabases(); + Assert.assertEquals("All databases size", 4, allDatabases.size()); + for(Database database : testDatabases) { + if (!database.getName().equals(testDatabases[1].getName())) { + Assert.assertTrue("Checking database names", allDatabases.contains(database.getName())); + } + } + Assert.assertTrue("Checnking that default database is returned", + allDatabases.contains(DEFAULT_DATABASE)); + Assert.assertFalse("Checking that the deleted database is not returned", + allDatabases.contains(testDatabases[1].getName())); + } + + @Test + public void testGetDatabases() throws Exception { + // Find databases which name contains _to_find_ + List<String> databases = client.getDatabases("*_to_find_*"); + Assert.assertEquals("Found databases size", 2, databases.size()); + Assert.assertTrue("Should contain", databases.contains("test_database_to_find_1")); + Assert.assertTrue("Should contain", databases.contains("test_database_to_find_2")); + + // Find databases which name contains _to_find_ or _hidden_ + databases = client.getDatabases("*_to_find_*|*_hidden_*"); + Assert.assertEquals("Found databases size", 3, databases.size()); + Assert.assertTrue("Should contain", databases.contains("test_database_to_find_1")); + Assert.assertTrue("Should contain", databases.contains("test_database_to_find_2")); + Assert.assertTrue("Should contain", databases.contains("test_database_hidden_1")); + + // Look for databases but do not find any + databases = client.getDatabases("*_not_such_database_*"); + Assert.assertEquals("No such databases size", 0, databases.size()); + + // Look for databases without pattern + databases = client.getDatabases(null); + Assert.assertEquals("Search databases without pattern size", 5, databases.size()); + } + + @Test + public void testGetDatabasesCaseInsensitive() throws Exception { + // Check case insensitive search + List<String> databases = client.getDatabases("*_tO_FiND*"); + Assert.assertEquals("Found databases size", 2, databases.size()); + Assert.assertTrue("Should contain", databases.contains("test_database_to_find_1")); + Assert.assertTrue("Should contain", databases.contains("test_database_to_find_2")); + } + +// HIVE-8472 - is missing on branch-2.3 which would fix this +// @Test +// public void testAlterDatabase() throws Exception { +// Database originalDatabase = testDatabases[0]; +// Database newDatabase = +// new DatabaseBuilder() +// // The database name is not changed during alter +// .setName(originalDatabase.getName()) +// .setOwnerType(PrincipalType.GROUP) +// .setOwnerName("owner2") +// .setLocation(metaStore.getWarehouseRoot() + "/database_location_2") +// .setDescription("dummy description 2") +// .addParam("param_key_1", "param_value_1_2") +// .addParam("param_key_2_3", "param_value_2_3") +// .build(); +// +// client.alterDatabase(originalDatabase.getName(), newDatabase); +// Database alteredDatabase = client.getDatabase(newDatabase.getName()); +// Assert.assertEquals("Comparing Databases", newDatabase, alteredDatabase); +// } + + @Test + public void testAlterDatabaseNotNullableFields() throws Exception { + Database database = getDatabaseWithAllParametersSet(); + client.createDatabase(database); + Database originalDatabase = client.getDatabase(database.getName()); + Database newDatabase = new Database(); + newDatabase.setName("new_name"); + + client.alterDatabase(originalDatabase.getName(), newDatabase); + // The name should not be changed, so reload the db with the original name + Database alteredDatabase = client.getDatabase(originalDatabase.getName()); + Assert.assertEquals("Database name should not change", originalDatabase.getName(), + alteredDatabase.getName()); + Assert.assertEquals("Database description should not change", originalDatabase.getDescription(), + alteredDatabase.getDescription()); + Assert.assertEquals("Database location should not change", originalDatabase.getLocationUri(), + alteredDatabase.getLocationUri()); + Assert.assertEquals("Database parameters should be empty", new HashMap<String, String>(), + alteredDatabase.getParameters()); + Assert.assertNull("Database owner should be empty", alteredDatabase.getOwnerName()); + Assert.assertEquals("Database owner type should not change", originalDatabase.getOwnerType(), + alteredDatabase.getOwnerType()); + Assert.assertNull("Database privileges should be empty", alteredDatabase.getPrivileges()); + } + +// HIVE-8472 - is missing on branch-2.3 which would fix this +// @Test(expected = NoSuchObjectException.class) +// public void testAlterDatabaseNoSuchDatabase() throws Exception { +// Database newDatabase = new DatabaseBuilder().setName("test_database_altered").build(); +// +// client.alterDatabase("no_such_database", newDatabase); +// } + +// HIVE-8472 - is missing on branch-2.3 which would fix this +// @Test +// public void testAlterDatabaseCaseInsensitive() throws Exception { +// Database originalDatabase = testDatabases[0]; +// Database newDatabase = originalDatabase.deepCopy(); +// newDatabase.setDescription("Altered database"); +// +// // Test in upper case +// client.alterDatabase(originalDatabase.getName().toUpperCase(), newDatabase); +// Database alteredDatabase = client.getDatabase(newDatabase.getName()); +// Assert.assertEquals("Comparing databases", newDatabase, alteredDatabase); +// +// // Test in mixed case +// originalDatabase = testDatabases[2]; +// newDatabase = originalDatabase.deepCopy(); +// newDatabase.setDescription("Altered database 2"); +// client.alterDatabase("TeST_daTAbaSe_TO_FiNd_2", newDatabase); +// alteredDatabase = client.getDatabase(newDatabase.getName()); +// Assert.assertEquals("Comparing databases", newDatabase, alteredDatabase); +// } + + private Database getDatabaseWithAllParametersSet() throws Exception { + return new DatabaseBuilder() + .setName("dummy") + .setOwnerType(PrincipalType.ROLE) + .setOwnerName("owner") + .setLocation(metaStore.getWarehouseRoot() + "/database_location") + .setDescription("dummy description") + .addParam("param_key_1", "param_value_1") + .addParam("param_key_2", "param_value_2") + .build(); + } +} diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java b/metastore/src/test/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java new file mode 100644 index 00000000000..978df5e4142 --- /dev/null +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.client; + +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.RawStore; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; +import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import static org.junit.Assert.assertTrue; + +/** + * Test class for IMetaStoreClient API. Testing the Table related functions for metadata + * manipulation, like creating, dropping and altering tables. + */ +@RunWith(Parameterized.class) +public class TestTablesCreateDropAlterTruncate { + private static final Logger LOG = LoggerFactory.getLogger(TestTablesCreateDropAlterTruncate.class); + // Needed until there is no junit release with @BeforeParam, @AfterParam (junit 4.13) + // https://github.com/junit-team/junit4/commit/1bf8438b65858565dbb64736bfe13aae9cfc1b5a + // Then we should remove our own copy + private static Set<AbstractMetaStoreService> metaStoreServices = null; + private static final String DEFAULT_DATABASE = "default"; + private final AbstractMetaStoreService metaStore; + private IMetaStoreClient client; + private Table[] testTables = new Table[6]; + + @Parameterized.Parameters(name = "{0}") + public static List<Object[]> getMetaStoreToTest() throws Exception { + List<Object[]> result = MetaStoreFactoryForTests.getMetaStores(); + metaStoreServices = new HashSet<>(result.size()); + for(Object[] test: result) { + metaStoreServices.add((AbstractMetaStoreService) test[1]); + } + return result; + } + + public TestTablesCreateDropAlterTruncate(String name, AbstractMetaStoreService metaStore) throws Exception { + this.metaStore = metaStore; + this.metaStore.start(); + } + + // Needed until there is no junit release with @BeforeParam, @AfterParam (junit 4.13) + // https://github.com/junit-team/junit4/commit/1bf8438b65858565dbb64736bfe13aae9cfc1b5a + // Then we should move this to @AfterParam + @AfterClass + public static void stopMetaStores() throws Exception { + for (AbstractMetaStoreService metaStoreService : metaStoreServices) { + try { + metaStoreService.stop(); + } catch (Exception e) { + // Catch the exceptions, so every other metastore could be stopped as well + // Log it, so at least there is a slight possibility we find out about this :) + LOG.error("Error stopping MetaStoreService", e); + } + } + } + + @Before + public void setUp() throws Exception { + // Get new client + client = metaStore.getClient(); + + // Drop every table in the default database + for (String tableName : client.getAllTables(DEFAULT_DATABASE)) { + client.dropTable(DEFAULT_DATABASE, tableName, true, true, true); + } + + // Clean up trash + metaStore.cleanWarehouseDirs(); + + testTables[0] = + new TableBuilder() + .setDbName(DEFAULT_DATABASE) + .setTableName("test_table") + .addCol("test_col", "int") + .build(); + + client.createTable(testTables[0]); + testTables[0] = client.getTable(testTables[0].getDbName(), testTables[0].getTableName()); + } + + @After + public void tearDown() throws Exception { + try { + if (client != null) { + try { + client.close(); + } catch (Exception e) { + // HIVE-19729: Shallow the exceptions based on the discussion in the Jira + } + } + } finally { + client = null; + } + } + + @Test + public void testAlterTableExpectedPropertyMatch() throws Exception { + Table originalTable = testTables[0]; + + EnvironmentContext context = new EnvironmentContext(); + context.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_KEY, "transient_lastDdlTime"); + context.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_VALUE, + originalTable.getParameters().get("transient_lastDdlTime")); + + client.alter_table_with_environmentContext(originalTable.getDbName(), originalTable.getTableName(), + originalTable, context); + } + + @Test(expected = MetaException.class) + public void testAlterTableExpectedPropertyDifferent() throws Exception { + Table originalTable = testTables[0]; + + EnvironmentContext context = new EnvironmentContext(); + context.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_KEY, "transient_lastDdlTime"); + context.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_VALUE, "alma"); + + client.alter_table_with_environmentContext(originalTable.getDbName(), originalTable.getTableName(), + originalTable, context); + } + + /** + * This tests ensures that concurrent Iceberg commits will fail. Acceptable as a first sanity check. + * <p> + * I have not found a good way to check that HMS side database commits are parallel in the + * {@link org.apache.hadoop.hive.metastore.HiveAlterHandler#alterTable(RawStore, Warehouse, String, String, Table, EnvironmentContext)} + * call, but this test could be used to manually ensure that using breakpoints. + */ + @Test + public void testAlterTableExpectedPropertyConcurrent() throws Exception { + Table originalTable = testTables[0]; + + originalTable.getParameters().put("snapshot", "0"); + client.alter_table_with_environmentContext(originalTable.getDbName(), originalTable.getTableName(), + originalTable, null); + + ExecutorService threads = null; + try { + threads = Executors.newFixedThreadPool(2); + for (int i = 0; i < 3; i++) { + EnvironmentContext context = new EnvironmentContext(); + context.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_KEY, "snapshot"); + context.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_VALUE, String.valueOf(i)); + + Table newTable = originalTable.deepCopy(); + newTable.getParameters().put("snapshot", String.valueOf(i + 1)); + + IMetaStoreClient client1 = metaStore.getClient(); + IMetaStoreClient client2 = metaStore.getClient(); + + Collection<Callable<Boolean>> concurrentTasks = new ArrayList<>(2); + concurrentTasks.add(alterTask(client1, newTable, context)); + concurrentTasks.add(alterTask(client2, newTable, context)); + + Collection<Future<Boolean>> results = threads.invokeAll(concurrentTasks); + + boolean foundSuccess = false; + boolean foundFailure = false; + + for (Future<Boolean> result : results) { + if (result.get()) { + foundSuccess = true; + } else { + foundFailure = true; + } + } + + assertTrue("At least one success is expected", foundSuccess); + assertTrue("At least one failure is expected", foundFailure); + } + } finally { + if (threads != null) { + threads.shutdown(); + } + } + } + + private Callable<Boolean> alterTask(final IMetaStoreClient hmsClient, final Table newTable, final EnvironmentContext context) { + return new Callable<Boolean>() { + @Override + public Boolean call() throws Exception { + try { + hmsClient.alter_table_with_environmentContext(newTable.getDbName(), newTable.getTableName(), + newTable, context); + } catch (Throwable e) { + return false; + } + return true; + } + }; + } +} \ No newline at end of file diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java b/metastore/src/test/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java new file mode 100644 index 00000000000..9b4fd704e96 --- /dev/null +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.minihms; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.TrashPolicy; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaException; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.MetaException; + +import java.io.IOException; +import java.util.Map; + +/** + * The tests should use this abstract class to access the MetaStore services. + * This abstract class ensures, that the same tests could be run against the different MetaStore + * configurations. + */ +public abstract class AbstractMetaStoreService { + protected HiveConf configuration; + private Warehouse warehouse; + private FileSystem warehouseRootFs; + private Path trashDir; + + public AbstractMetaStoreService(HiveConf configuration) { + this.configuration = new HiveConf(configuration); + } + + /** + * Starts the MetaStoreService. Be aware, as the current MetaStore does not implement clean + * shutdown, starting MetaStoreService is possible only once per test. + * + * @throws Exception if any Exception occurs + */ + public void start() throws Exception { + warehouse = new Warehouse(configuration); + warehouseRootFs = warehouse.getFs(warehouse.getWhRoot()); + TrashPolicy trashPolicy = TrashPolicy.getInstance(configuration, warehouseRootFs, warehouse.getWhRoot()); + trashDir = trashPolicy.getCurrentTrashDir(); + } + + /** + * Starts the service with adding extra configuration to the default ones. Be aware, as the + * current MetaStore does not implement clean shutdown, starting MetaStoreService is possible + * only once per test. + * + * @param confOverlay The extra parameters which should be set before starting the service + * @throws Exception if any Exception occurs + */ + public void start(Map<HiveConf.ConfVars, String> confOverlay) throws Exception { + // Set confOverlay parameters + for (Map.Entry<HiveConf.ConfVars, String> entry : confOverlay.entrySet()) { + HiveConf.setVar(configuration, entry.getKey(), entry.getValue()); + } + // Start the service + start(); + } + + /** + * Returns the MetaStoreClient for this MetaStoreService. + * + * @return The client connected to this service + * @throws HiveMetaException if any Exception occurs during client configuration + */ + public IMetaStoreClient getClient() throws MetaException { + return new HiveMetaStoreClient(configuration); + } + + /** + * Returns the MetaStore Warehouse root directory name. + * + * @return The warehouse root directory + * @throws HiveMetaException IO failure + */ + public Path getWarehouseRoot() throws MetaException { + return warehouse.getWhRoot(); + } + + /** + * Check if a path exists. + * + * @param path The path to check + * @return true if the path exists + * @throws IOException IO failure + */ + public boolean isPathExists(Path path) throws IOException { + return warehouseRootFs.exists(path); + } + + /** + * Check if a path exists in the thrash directory. + * + * @param path The path to check + * @return True if the path exists + * @throws IOException IO failure + */ + public boolean isPathExistsInTrash(Path path) throws IOException { + Path pathInTrash = new Path(trashDir.toUri().getScheme(), trashDir.toUri().getAuthority(), + trashDir.toUri().getPath() + path.toUri().getPath()); + return isPathExists(pathInTrash); + } + + /** + * Creates a file on the given path. + * + * @param path Destination path + * @param content The content of the file + * @throws IOException IO failure + */ + public void createFile(Path path, String content) throws IOException { + FSDataOutputStream outputStream = warehouseRootFs.create(path); + outputStream.write(content.getBytes()); + outputStream.close(); + } + + /** + * Cleans the warehouse and the thrash dirs in preparation for the tests. + * + * @throws HiveMetaException IO failure + */ + public void cleanWarehouseDirs() throws MetaException { + warehouse.deleteDir(getWarehouseRoot(), true, true); + warehouse.deleteDir(trashDir, true, true); + } + + /** + * Stops the MetaStoreService. When MetaStore will implement clean shutdown, this method will + * call shutdown on MetaStore. Currently this does nothing :( + */ + public void stop() { + } +} diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/minihms/ClusterMetaStoreForTests.java b/metastore/src/test/org/apache/hadoop/hive/metastore/minihms/ClusterMetaStoreForTests.java new file mode 100644 index 00000000000..b4c859de5fb --- /dev/null +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/minihms/ClusterMetaStoreForTests.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.minihms; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; + +/** + * The AbstractMetaStore implementation which is used when the tests are running against a cluster + * created manually or by an external application (See the + * {@link org.apache.hadoop.hive.metastore.client.MetaStoreFactoryForTests} class). + */ +public class ClusterMetaStoreForTests extends AbstractMetaStoreService { + public ClusterMetaStoreForTests(HiveConf configuration) { + super(configuration); + } +} diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/minihms/EmbeddedMetaStoreForTests.java b/metastore/src/test/org/apache/hadoop/hive/metastore/minihms/EmbeddedMetaStoreForTests.java new file mode 100644 index 00000000000..7d2b34b8bea --- /dev/null +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/minihms/EmbeddedMetaStoreForTests.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.minihms; + + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; + +/** + * The AbstractMetaStore implementation which is used when the tests are running against an embedded + * MetaStore in the same thread as the client without Thrift communication (See the + * {@link org.apache.hadoop.hive.metastore.client.MetaStoreFactoryForTests} class). + */ +public class EmbeddedMetaStoreForTests extends AbstractMetaStoreService { + public EmbeddedMetaStoreForTests(HiveConf configuration) { + super(configuration); + } +} \ No newline at end of file diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/minihms/MiniHMS.java b/metastore/src/test/org/apache/hadoop/hive/metastore/minihms/MiniHMS.java new file mode 100644 index 00000000000..1d2f5c3fd58 --- /dev/null +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/minihms/MiniHMS.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.minihms; + +import org.apache.hadoop.hive.conf.HiveConf; + +/** + * Mini HMS implementation, which can be used to run tests against different HMS configurations. + * Currently it supports 3 types: + * - EMBEDDED - MetaStore running in embedded mode + * - REMOTE - MetaStore running in the same process but in a dedicated thread and accessed + * through the Thrift interface + * - CLUSTER - In this case the MiniHMS is only a wrapper around the HMS running on a cluster, + * so the same tests could be run against a real cluster + */ +public class MiniHMS { + public enum MiniHMSType { + EMBEDDED, + REMOTE, + CLUSTER + } + + public static class Builder { + private HiveConf metaStoreConf = new HiveConf(); + private MiniHMSType miniHMSType = MiniHMSType.EMBEDDED; + + public Builder() { + } + + public Builder setConf(HiveConf conf) { + this.metaStoreConf = new HiveConf(conf); + return this; + } + + public Builder setType(MiniHMSType type) { + this.miniHMSType = type; + return this; + } + + public AbstractMetaStoreService build() throws Exception { + switch (miniHMSType) { + case REMOTE: + return new RemoteMetaStoreForTests(metaStoreConf); + case EMBEDDED: + return new EmbeddedMetaStoreForTests(metaStoreConf); + case CLUSTER: + return new ClusterMetaStoreForTests(metaStoreConf); + default: + throw new IllegalArgumentException("Unexpected miniHMSType: " + miniHMSType); + } + } + } +} diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/minihms/RemoteMetaStoreForTests.java b/metastore/src/test/org/apache/hadoop/hive/metastore/minihms/RemoteMetaStoreForTests.java new file mode 100644 index 00000000000..41a45b8ca3a --- /dev/null +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/minihms/RemoteMetaStoreForTests.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.minihms; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; + +/** + * The AbstractMetaStore implementation which is used when the tests are running against a + * MetaStore which running in a dedicated thread and accessed through the Thrift interface (See the + * {@link org.apache.hadoop.hive.metastore.client.MetaStoreFactoryForTests} class). + */ +public class RemoteMetaStoreForTests extends AbstractMetaStoreService { + + public RemoteMetaStoreForTests(HiveConf configuration) { + super(configuration); + } + + public void start() throws Exception { + HiveConf.setBoolVar(configuration, HiveConf.ConfVars.METASTORE_EXECUTE_SET_UGI, false); + int port = MetaStoreUtils.startMetaStore(configuration); + HiveConf.setVar(configuration, HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); + super.start(); + } +} \ No newline at end of file