http://git-wip-us.apache.org/repos/asf/hive/blob/651e7950/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- diff --cc standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 0000000,47f819b..285f7fb mode 000000,100644..100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@@ -1,0 -1,9353 +1,9602 @@@ + /* * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package org.apache.hadoop.hive.metastore; + + import static org.apache.commons.lang.StringUtils.join; + import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_COMMENT; + import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME; + import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; + import static org.apache.hadoop.hive.metastore.Warehouse.getCatalogQualifiedTableName; + import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; + import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.parseDbName; + import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.CAT_NAME; + import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.DB_NAME; + import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependCatalogToDbName; + import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependNotNullCatToDbName; + + import java.io.IOException; + import java.net.InetAddress; + import java.net.UnknownHostException; + import java.nio.ByteBuffer; + import java.security.PrivilegedExceptionAction; + import java.util.AbstractMap; + import java.util.ArrayList; + import java.util.Arrays; + import java.util.Collection; + import java.util.Collections; + import java.util.HashMap; + import java.util.HashSet; + import java.util.Iterator; + import java.util.LinkedHashMap; + import java.util.LinkedList; + import java.util.List; + import java.util.Map; + import java.util.Map.Entry; + import java.util.Objects; + import java.util.Properties; + import java.util.Set; + import java.util.concurrent.ConcurrentHashMap; + import java.util.concurrent.ExecutionException; + import java.util.concurrent.ExecutorService; + import java.util.concurrent.Executors; + import java.util.concurrent.Future; + import java.util.concurrent.TimeUnit; + import java.util.concurrent.atomic.AtomicBoolean; + import java.util.concurrent.atomic.AtomicInteger; + import java.util.concurrent.locks.Condition; + import java.util.concurrent.locks.Lock; + import java.util.concurrent.locks.ReentrantLock; + import java.util.regex.Pattern; + + import javax.jdo.JDOException; + + import com.codahale.metrics.Counter; + import com.google.common.collect.ImmutableList; + import com.google.common.collect.ImmutableListMultimap; + import com.google.common.collect.Lists; + import com.google.common.collect.Multimaps; + + import org.apache.commons.cli.OptionBuilder; + import org.apache.hadoop.conf.Configuration; + import org.apache.hadoop.fs.FileStatus; + import org.apache.hadoop.fs.FileSystem; + import org.apache.hadoop.fs.Path; + import org.apache.hadoop.hive.common.StatsSetupConst; + import org.apache.hadoop.hive.common.TableName; + import org.apache.hadoop.hive.metastore.api.*; + import org.apache.hadoop.hive.metastore.events.AddForeignKeyEvent; + import org.apache.hadoop.hive.metastore.events.AcidWriteEvent; + import org.apache.hadoop.hive.metastore.conf.MetastoreConf; + import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; + import org.apache.hadoop.hive.metastore.conf.MetastoreConf.StatsUpdateMode; + import org.apache.hadoop.hive.metastore.events.AbortTxnEvent; + import org.apache.hadoop.hive.metastore.events.AddNotNullConstraintEvent; + import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; + import org.apache.hadoop.hive.metastore.events.AddPrimaryKeyEvent; + import org.apache.hadoop.hive.metastore.events.AddUniqueConstraintEvent; + import org.apache.hadoop.hive.metastore.events.AllocWriteIdEvent; + import org.apache.hadoop.hive.metastore.events.AlterCatalogEvent; + import org.apache.hadoop.hive.metastore.events.AlterDatabaseEvent; + import org.apache.hadoop.hive.metastore.events.AlterISchemaEvent; + import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; + import org.apache.hadoop.hive.metastore.events.AlterSchemaVersionEvent; + import org.apache.hadoop.hive.metastore.events.AlterTableEvent; + import org.apache.hadoop.hive.metastore.events.CommitTxnEvent; + import org.apache.hadoop.hive.metastore.events.ConfigChangeEvent; + import org.apache.hadoop.hive.metastore.events.CreateCatalogEvent; + import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; + import org.apache.hadoop.hive.metastore.events.CreateFunctionEvent; + import org.apache.hadoop.hive.metastore.events.CreateISchemaEvent; + import org.apache.hadoop.hive.metastore.events.AddSchemaVersionEvent; + import org.apache.hadoop.hive.metastore.events.CreateTableEvent; + import org.apache.hadoop.hive.metastore.events.DropCatalogEvent; + import org.apache.hadoop.hive.metastore.events.DropConstraintEvent; + import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; + import org.apache.hadoop.hive.metastore.events.DropFunctionEvent; + import org.apache.hadoop.hive.metastore.events.DropISchemaEvent; + import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; + import org.apache.hadoop.hive.metastore.events.DropSchemaVersionEvent; + import org.apache.hadoop.hive.metastore.events.DropTableEvent; + import org.apache.hadoop.hive.metastore.events.InsertEvent; + import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent; + import org.apache.hadoop.hive.metastore.events.OpenTxnEvent; + import org.apache.hadoop.hive.metastore.events.PreAddPartitionEvent; + import org.apache.hadoop.hive.metastore.events.PreAlterCatalogEvent; + import org.apache.hadoop.hive.metastore.events.PreAlterDatabaseEvent; + import org.apache.hadoop.hive.metastore.events.PreAlterISchemaEvent; + import org.apache.hadoop.hive.metastore.events.PreAlterPartitionEvent; + import org.apache.hadoop.hive.metastore.events.PreAlterSchemaVersionEvent; + import org.apache.hadoop.hive.metastore.events.PreAlterTableEvent; + import org.apache.hadoop.hive.metastore.events.PreAuthorizationCallEvent; + import org.apache.hadoop.hive.metastore.events.PreCreateCatalogEvent; + import org.apache.hadoop.hive.metastore.events.PreCreateDatabaseEvent; + import org.apache.hadoop.hive.metastore.events.PreCreateISchemaEvent; + import org.apache.hadoop.hive.metastore.events.PreAddSchemaVersionEvent; + import org.apache.hadoop.hive.metastore.events.PreCreateTableEvent; + import org.apache.hadoop.hive.metastore.events.PreDropCatalogEvent; + import org.apache.hadoop.hive.metastore.events.PreDropDatabaseEvent; + import org.apache.hadoop.hive.metastore.events.PreDropISchemaEvent; + import org.apache.hadoop.hive.metastore.events.PreDropPartitionEvent; + import org.apache.hadoop.hive.metastore.events.PreDropSchemaVersionEvent; + import org.apache.hadoop.hive.metastore.events.PreDropTableEvent; + import org.apache.hadoop.hive.metastore.events.PreEventContext; + import org.apache.hadoop.hive.metastore.events.PreLoadPartitionDoneEvent; + import org.apache.hadoop.hive.metastore.events.PreReadCatalogEvent; + import org.apache.hadoop.hive.metastore.events.PreReadDatabaseEvent; + import org.apache.hadoop.hive.metastore.events.PreReadISchemaEvent; + import org.apache.hadoop.hive.metastore.events.PreReadTableEvent; + import org.apache.hadoop.hive.metastore.events.PreReadhSchemaVersionEvent; + import org.apache.hadoop.hive.metastore.messaging.EventMessage.EventType; + import org.apache.hadoop.hive.metastore.metrics.JvmPauseMonitor; + import org.apache.hadoop.hive.metastore.metrics.Metrics; + import org.apache.hadoop.hive.metastore.metrics.MetricsConstants; + import org.apache.hadoop.hive.metastore.metrics.PerfLogger; + import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; + import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; + import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager; + import org.apache.hadoop.hive.metastore.security.TUGIContainingTransport; + import org.apache.hadoop.hive.metastore.txn.TxnStore; + import org.apache.hadoop.hive.metastore.txn.TxnUtils; + import org.apache.hadoop.security.SecurityUtil; + import org.apache.hadoop.hive.metastore.utils.CommonCliOptions; + import org.apache.hadoop.hive.metastore.utils.FileUtils; + import org.apache.hadoop.hive.metastore.utils.HdfsUtils; + import org.apache.hadoop.hive.metastore.utils.JavaUtils; + import org.apache.hadoop.hive.metastore.utils.LogUtils; + import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; + import org.apache.hadoop.hive.metastore.utils.MetastoreVersionInfo; + import org.apache.hadoop.hive.metastore.utils.SecurityUtils; + import org.apache.hadoop.security.UserGroupInformation; + import org.apache.hadoop.util.ReflectionUtils; + import org.apache.hadoop.util.ShutdownHookManager; + import org.apache.hadoop.util.StringUtils; + import org.apache.logging.log4j.LogManager; + import org.apache.logging.log4j.core.LoggerContext; + import org.apache.thrift.TException; + import org.apache.thrift.TProcessor; + import org.apache.thrift.protocol.TBinaryProtocol; + import org.apache.thrift.protocol.TCompactProtocol; + import org.apache.thrift.protocol.TProtocol; + import org.apache.thrift.protocol.TProtocolFactory; + import org.apache.thrift.server.ServerContext; + import org.apache.thrift.server.TServer; + import org.apache.thrift.server.TServerEventHandler; + import org.apache.thrift.server.TThreadPoolServer; + import org.apache.thrift.transport.TFramedTransport; + import org.apache.thrift.transport.TServerSocket; + import org.apache.thrift.transport.TTransport; + import org.apache.thrift.transport.TTransportFactory; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; + + import com.facebook.fb303.FacebookBase; + import com.facebook.fb303.fb_status; + import com.google.common.annotations.VisibleForTesting; + import com.google.common.base.Preconditions; + import com.google.common.base.Splitter; + import com.google.common.util.concurrent.ThreadFactoryBuilder; + + /** + * TODO:pc remove application logic to a separate interface. + */ + public class HiveMetaStore extends ThriftHiveMetastore { + public static final Logger LOG = LoggerFactory.getLogger(HiveMetaStore.class); + public static final String PARTITION_NUMBER_EXCEED_LIMIT_MSG = + "Number of partitions scanned (=%d) on table '%s' exceeds limit (=%d). This is controlled on the metastore server by %s."; + + // boolean that tells if the HiveMetaStore (remote) server is being used. + // Can be used to determine if the calls to metastore api (HMSHandler) are being made with + // embedded metastore or a remote one + private static boolean isMetaStoreRemote = false; + + // Used for testing to simulate method timeout. + @VisibleForTesting + static boolean TEST_TIMEOUT_ENABLED = false; + @VisibleForTesting + static long TEST_TIMEOUT_VALUE = -1; + + private static ShutdownHookManager shutdownHookMgr; + + public static final String ADMIN = "admin"; + public static final String PUBLIC = "public"; + /** MM write states. */ + public static final char MM_WRITE_OPEN = 'o', MM_WRITE_COMMITTED = 'c', MM_WRITE_ABORTED = 'a'; + + private static HadoopThriftAuthBridge.Server saslServer; + private static MetastoreDelegationTokenManager delegationTokenManager; + private static boolean useSasl; + + static final String NO_FILTER_STRING = ""; + static final int UNLIMITED_MAX_PARTITIONS = -1; + + private static final class ChainedTTransportFactory extends TTransportFactory { + private final TTransportFactory parentTransFactory; + private final TTransportFactory childTransFactory; + + private ChainedTTransportFactory( + TTransportFactory parentTransFactory, + TTransportFactory childTransFactory) { + this.parentTransFactory = parentTransFactory; + this.childTransFactory = childTransFactory; + } + + @Override + public TTransport getTransport(TTransport trans) { + return childTransFactory.getTransport(parentTransFactory.getTransport(trans)); + } + } + + public static boolean isRenameAllowed(Database srcDB, Database destDB) { + if (!srcDB.getName().equalsIgnoreCase(destDB.getName())) { + if (ReplChangeManager.isSourceOfReplication(srcDB) || ReplChangeManager.isSourceOfReplication(destDB)) { + return false; + } + } + return true; + } + + public static class HMSHandler extends FacebookBase implements IHMSHandler { + public static final Logger LOG = HiveMetaStore.LOG; + private final Configuration conf; // stores datastore (jpox) properties, + // right now they come from jpox.properties + + // Flag to control that always threads are initialized only once + // instead of multiple times + private final static AtomicBoolean alwaysThreadsInitialized = + new AtomicBoolean(false); + + private static String currentUrl; + private FileMetadataManager fileMetadataManager; + private PartitionExpressionProxy expressionProxy; + private StorageSchemaReader storageSchemaReader; + + // Variables for metrics + // Package visible so that HMSMetricsListener can see them. + static AtomicInteger databaseCount, tableCount, partCount; + + private Warehouse wh; // hdfs warehouse + private static final ThreadLocal<RawStore> threadLocalMS = + new ThreadLocal<RawStore>() { + @Override + protected RawStore initialValue() { + return null; + } + }; + + private static final ThreadLocal<TxnStore> threadLocalTxn = new ThreadLocal<TxnStore>() { + @Override + protected TxnStore initialValue() { + return null; + } + }; + + private static final ThreadLocal<Map<String, com.codahale.metrics.Timer.Context>> timerContexts = + new ThreadLocal<Map<String, com.codahale.metrics.Timer.Context>>() { + @Override + protected Map<String, com.codahale.metrics.Timer.Context> initialValue() { + return new HashMap<>(); + } + }; + + public static RawStore getRawStore() { + return threadLocalMS.get(); + } + + static void removeRawStore() { + threadLocalMS.remove(); + } + + // Thread local configuration is needed as many threads could make changes + // to the conf using the connection hook + private static final ThreadLocal<Configuration> threadLocalConf = + new ThreadLocal<Configuration>() { + @Override + protected Configuration initialValue() { + return null; + } + }; + + /** + * Thread local HMSHandler used during shutdown to notify meta listeners + */ + private static final ThreadLocal<HMSHandler> threadLocalHMSHandler = new ThreadLocal<>(); + + /** + * Thread local Map to keep track of modified meta conf keys + */ + private static final ThreadLocal<Map<String, String>> threadLocalModifiedConfig = + new ThreadLocal<>(); + + private static ExecutorService threadPool; + + static final Logger auditLog = LoggerFactory.getLogger( + HiveMetaStore.class.getName() + ".audit"); + + private static void logAuditEvent(String cmd) { + if (cmd == null) { + return; + } + + UserGroupInformation ugi; + try { + ugi = SecurityUtils.getUGI(); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + + String address = getIPAddress(); + if (address == null) { + address = "unknown-ip-addr"; + } + + auditLog.info("ugi={} ip={} cmd={} ", ugi.getUserName(), address, cmd); + } + + private static String getIPAddress() { + if (useSasl) { + if (saslServer != null && saslServer.getRemoteAddress() != null) { + return saslServer.getRemoteAddress().getHostAddress(); + } + } else { + // if kerberos is not enabled + return getThreadLocalIpAddress(); + } + return null; + } + + private static AtomicInteger nextSerialNum = new AtomicInteger(); + private static ThreadLocal<Integer> threadLocalId = new ThreadLocal<Integer>() { + @Override + protected Integer initialValue() { + return nextSerialNum.getAndIncrement(); + } + }; + + // This will only be set if the metastore is being accessed from a metastore Thrift server, + // not if it is from the CLI. Also, only if the TTransport being used to connect is an + // instance of TSocket. This is also not set when kerberos is used. + private static ThreadLocal<String> threadLocalIpAddress = new ThreadLocal<String>() { + @Override + protected String initialValue() { + return null; + } + }; + + /** + * Internal function to notify listeners for meta config change events + */ + private void notifyMetaListeners(String key, String oldValue, String newValue) throws MetaException { + for (MetaStoreEventListener listener : listeners) { + listener.onConfigChange(new ConfigChangeEvent(this, key, oldValue, newValue)); + } + + if (transactionalListeners.size() > 0) { + // All the fields of this event are final, so no reason to create a new one for each + // listener + ConfigChangeEvent cce = new ConfigChangeEvent(this, key, oldValue, newValue); + for (MetaStoreEventListener transactionalListener : transactionalListeners) { + transactionalListener.onConfigChange(cce); + } + } + } + + /** + * Internal function to notify listeners to revert back to old values of keys + * that were modified during setMetaConf. This would get called from HiveMetaStore#cleanupRawStore + */ + private void notifyMetaListenersOnShutDown() { + Map<String, String> modifiedConf = threadLocalModifiedConfig.get(); + if (modifiedConf == null) { + // Nothing got modified + return; + } + try { + Configuration conf = threadLocalConf.get(); + if (conf == null) { + throw new MetaException("Unexpected: modifiedConf is non-null but conf is null"); + } + // Notify listeners of the changed value + for (Entry<String, String> entry : modifiedConf.entrySet()) { + String key = entry.getKey(); + // curr value becomes old and vice-versa + String currVal = entry.getValue(); + String oldVal = conf.get(key); + if (!Objects.equals(oldVal, currVal)) { + notifyMetaListeners(key, oldVal, currVal); + } + } + logInfo("Meta listeners shutdown notification completed."); + } catch (MetaException e) { + LOG.error("Failed to notify meta listeners on shutdown: ", e); + } + } + + static void setThreadLocalIpAddress(String ipAddress) { + threadLocalIpAddress.set(ipAddress); + } + + // This will return null if the metastore is not being accessed from a metastore Thrift server, + // or if the TTransport being used to connect is not an instance of TSocket, or if kereberos + // is used + static String getThreadLocalIpAddress() { + return threadLocalIpAddress.get(); + } + + // Make it possible for tests to check that the right type of PartitionExpressionProxy was + // instantiated. + @VisibleForTesting + PartitionExpressionProxy getExpressionProxy() { + return expressionProxy; + } + + /** + * Use {@link #getThreadId()} instead. + * @return thread id + */ + @Deprecated + public static Integer get() { + return threadLocalId.get(); + } + + @Override + public int getThreadId() { + return threadLocalId.get(); + } + + public HMSHandler(String name) throws MetaException { + this(name, MetastoreConf.newMetastoreConf(), true); + } + + public HMSHandler(String name, Configuration conf) throws MetaException { + this(name, conf, true); + } + + public HMSHandler(String name, Configuration conf, boolean init) throws MetaException { + super(name); + this.conf = conf; + isInTest = MetastoreConf.getBoolVar(this.conf, ConfVars.HIVE_IN_TEST); + if (threadPool == null) { + synchronized (HMSHandler.class) { + int numThreads = MetastoreConf.getIntVar(conf, ConfVars.FS_HANDLER_THREADS_COUNT); + threadPool = Executors.newFixedThreadPool(numThreads, + new ThreadFactoryBuilder().setDaemon(true) + .setNameFormat("HMSHandler #%d").build()); + } + } + if (init) { + init(); + } + } + + /** + * Use {@link #getConf()} instead. + * @return Configuration object + */ + @Deprecated + public Configuration getHiveConf() { + return conf; + } + + private ClassLoader classLoader; + private AlterHandler alterHandler; + private List<MetaStorePreEventListener> preListeners; + private List<MetaStoreEventListener> listeners; + private List<TransactionalMetaStoreEventListener> transactionalListeners; + private List<MetaStoreEndFunctionListener> endFunctionListeners; + private List<MetaStoreInitListener> initListeners; + private Pattern partitionValidationPattern; + private final boolean isInTest; + + { + classLoader = Thread.currentThread().getContextClassLoader(); + if (classLoader == null) { + classLoader = Configuration.class.getClassLoader(); + } + } + + @Override + public List<TransactionalMetaStoreEventListener> getTransactionalListeners() { + return transactionalListeners; + } + + @Override + public List<MetaStoreEventListener> getListeners() { + return listeners; + } + + @Override + public void init() throws MetaException { + initListeners = MetaStoreUtils.getMetaStoreListeners( + MetaStoreInitListener.class, conf, MetastoreConf.getVar(conf, ConfVars.INIT_HOOKS)); + for (MetaStoreInitListener singleInitListener: initListeners) { + MetaStoreInitContext context = new MetaStoreInitContext(); + singleInitListener.onInit(context); + } + + String alterHandlerName = MetastoreConf.getVar(conf, ConfVars.ALTER_HANDLER); + alterHandler = ReflectionUtils.newInstance(JavaUtils.getClass( + alterHandlerName, AlterHandler.class), conf); + wh = new Warehouse(conf); + + synchronized (HMSHandler.class) { + if (currentUrl == null || !currentUrl.equals(MetaStoreInit.getConnectionURL(conf))) { + createDefaultDB(); + createDefaultRoles(); + addAdminUsers(); + currentUrl = MetaStoreInit.getConnectionURL(conf); + } + } + + //Start Metrics + if (MetastoreConf.getBoolVar(conf, ConfVars.METRICS_ENABLED)) { + LOG.info("Begin calculating metadata count metrics."); + Metrics.initialize(conf); + databaseCount = Metrics.getOrCreateGauge(MetricsConstants.TOTAL_DATABASES); + tableCount = Metrics.getOrCreateGauge(MetricsConstants.TOTAL_TABLES); + partCount = Metrics.getOrCreateGauge(MetricsConstants.TOTAL_PARTITIONS); + updateMetrics(); + + } + + preListeners = MetaStoreUtils.getMetaStoreListeners(MetaStorePreEventListener.class, + conf, MetastoreConf.getVar(conf, ConfVars.PRE_EVENT_LISTENERS)); + preListeners.add(0, new TransactionalValidationListener(conf)); + listeners = MetaStoreUtils.getMetaStoreListeners(MetaStoreEventListener.class, conf, + MetastoreConf.getVar(conf, ConfVars.EVENT_LISTENERS)); + listeners.add(new SessionPropertiesListener(conf)); + listeners.add(new AcidEventListener(conf)); + transactionalListeners = MetaStoreUtils.getMetaStoreListeners(TransactionalMetaStoreEventListener.class, + conf, MetastoreConf.getVar(conf, ConfVars.TRANSACTIONAL_EVENT_LISTENERS)); + if (Metrics.getRegistry() != null) { + listeners.add(new HMSMetricsListener(conf)); + } + + endFunctionListeners = MetaStoreUtils.getMetaStoreListeners( + MetaStoreEndFunctionListener.class, conf, MetastoreConf.getVar(conf, ConfVars.END_FUNCTION_LISTENERS)); + + String partitionValidationRegex = + MetastoreConf.getVar(conf, ConfVars.PARTITION_NAME_WHITELIST_PATTERN); + if (partitionValidationRegex != null && !partitionValidationRegex.isEmpty()) { + partitionValidationPattern = Pattern.compile(partitionValidationRegex); + } else { + partitionValidationPattern = null; + } + + // We only initialize once the tasks that need to be run periodically + if (alwaysThreadsInitialized.compareAndSet(false, true)) { + ThreadPool.initialize(conf); + Collection<String> taskNames = + MetastoreConf.getStringCollection(conf, ConfVars.TASK_THREADS_ALWAYS); + for (String taskName : taskNames) { + MetastoreTaskThread task = + JavaUtils.newInstance(JavaUtils.getClass(taskName, MetastoreTaskThread.class)); + task.setConf(conf); + long freq = task.runFrequency(TimeUnit.MILLISECONDS); + // For backwards compatibility, since some threads used to be hard coded but only run if + // frequency was > 0 + if (freq > 0) { + ThreadPool.getPool().scheduleAtFixedRate(task, freq, freq, TimeUnit.MILLISECONDS); + } + } + } + expressionProxy = PartFilterExprUtil.createExpressionProxy(conf); + fileMetadataManager = new FileMetadataManager(this.getMS(), conf); + } + + private static String addPrefix(String s) { + return threadLocalId.get() + ": " + s; + } + + /** + * Set copy of invoking HMSHandler on thread local + */ + private static void setHMSHandler(HMSHandler handler) { + if (threadLocalHMSHandler.get() == null) { + threadLocalHMSHandler.set(handler); + } + } + @Override + public void setConf(Configuration conf) { + threadLocalConf.set(conf); + RawStore ms = threadLocalMS.get(); + if (ms != null) { + ms.setConf(conf); // reload if DS related configuration is changed + } + } + + @Override + public Configuration getConf() { + Configuration conf = threadLocalConf.get(); + if (conf == null) { + conf = new Configuration(this.conf); + threadLocalConf.set(conf); + } + return conf; + } + + private Map<String, String> getModifiedConf() { + Map<String, String> modifiedConf = threadLocalModifiedConfig.get(); + if (modifiedConf == null) { + modifiedConf = new HashMap<>(); + threadLocalModifiedConfig.set(modifiedConf); + } + return modifiedConf; + } + + @Override + public Warehouse getWh() { + return wh; + } + + @Override + public void setMetaConf(String key, String value) throws MetaException { + ConfVars confVar = MetastoreConf.getMetaConf(key); + if (confVar == null) { + throw new MetaException("Invalid configuration key " + key); + } + try { + confVar.validate(value); + } catch (IllegalArgumentException e) { + throw new MetaException("Invalid configuration value " + value + " for key " + key + + " by " + e.getMessage()); + } + Configuration configuration = getConf(); + String oldValue = MetastoreConf.get(configuration, key); + // Save prev val of the key on threadLocal + Map<String, String> modifiedConf = getModifiedConf(); + if (!modifiedConf.containsKey(key)) { + modifiedConf.put(key, oldValue); + } + // Set invoking HMSHandler on threadLocal, this will be used later to notify + // metaListeners in HiveMetaStore#cleanupRawStore + setHMSHandler(this); + configuration.set(key, value); + notifyMetaListeners(key, oldValue, value); + + if (ConfVars.TRY_DIRECT_SQL == confVar) { + HMSHandler.LOG.info("Direct SQL optimization = {}", value); + } + } + + @Override + public String getMetaConf(String key) throws MetaException { + ConfVars confVar = MetastoreConf.getMetaConf(key); + if (confVar == null) { + throw new MetaException("Invalid configuration key " + key); + } + return getConf().get(key, confVar.getDefaultVal().toString()); + } + + /** + * Get a cached RawStore. + * + * @return the cached RawStore + * @throws MetaException + */ + @Override + public RawStore getMS() throws MetaException { + Configuration conf = getConf(); + return getMSForConf(conf); + } + + public static RawStore getMSForConf(Configuration conf) throws MetaException { + RawStore ms = threadLocalMS.get(); + if (ms == null) { + ms = newRawStoreForConf(conf); + ms.verifySchema(); + threadLocalMS.set(ms); + ms = threadLocalMS.get(); + } + return ms; + } + + @Override + public TxnStore getTxnHandler() { ++ return getMsThreadTxnHandler(conf); ++ } ++ ++ public static TxnStore getMsThreadTxnHandler(Configuration conf) { + TxnStore txn = threadLocalTxn.get(); + if (txn == null) { + txn = TxnUtils.getTxnStore(conf); + threadLocalTxn.set(txn); + } + return txn; + } + + static RawStore newRawStoreForConf(Configuration conf) throws MetaException { + Configuration newConf = new Configuration(conf); + String rawStoreClassName = MetastoreConf.getVar(newConf, ConfVars.RAW_STORE_IMPL); + LOG.info(addPrefix("Opening raw store with implementation class:" + rawStoreClassName)); + return RawStoreProxy.getProxy(newConf, conf, rawStoreClassName, threadLocalId.get()); + } + + @VisibleForTesting + public static void createDefaultCatalog(RawStore ms, Warehouse wh) throws MetaException, + InvalidOperationException { + try { + Catalog defaultCat = ms.getCatalog(DEFAULT_CATALOG_NAME); + // Null check because in some test cases we get a null from ms.getCatalog. + if (defaultCat !=null && defaultCat.getLocationUri().equals("TBD")) { + // One time update issue. When the new 'hive' catalog is created in an upgrade the + // script does not know the location of the warehouse. So we need to update it. + LOG.info("Setting location of default catalog, as it hasn't been done after upgrade"); + defaultCat.setLocationUri(wh.getWhRoot().toString()); + ms.alterCatalog(defaultCat.getName(), defaultCat); + } + + } catch (NoSuchObjectException e) { + Catalog cat = new Catalog(DEFAULT_CATALOG_NAME, wh.getWhRoot().toString()); + cat.setDescription(Warehouse.DEFAULT_CATALOG_COMMENT); + ms.createCatalog(cat); + } + } + + private void createDefaultDB_core(RawStore ms) throws MetaException, InvalidObjectException { + try { + ms.getDatabase(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME); + } catch (NoSuchObjectException e) { + Database db = new Database(DEFAULT_DATABASE_NAME, DEFAULT_DATABASE_COMMENT, + wh.getDefaultDatabasePath(DEFAULT_DATABASE_NAME).toString(), null); + db.setOwnerName(PUBLIC); + db.setOwnerType(PrincipalType.ROLE); + db.setCatalogName(DEFAULT_CATALOG_NAME); + ms.createDatabase(db); + } + } + + /** + * create default database if it doesn't exist. + * + * This is a potential contention when HiveServer2 using embedded metastore and Metastore + * Server try to concurrently invoke createDefaultDB. If one failed, JDOException was caught + * for one more time try, if failed again, simply ignored by warning, which meant another + * succeeds. + * + * @throws MetaException + */ + private void createDefaultDB() throws MetaException { + try { + RawStore ms = getMS(); + createDefaultCatalog(ms, wh); + createDefaultDB_core(ms); + } catch (JDOException e) { + LOG.warn("Retrying creating default database after error: " + e.getMessage(), e); + try { + createDefaultDB_core(getMS()); + } catch (InvalidObjectException e1) { + throw new MetaException(e1.getMessage()); + } + } catch (InvalidObjectException|InvalidOperationException e) { + throw new MetaException(e.getMessage()); + } + } + + /** + * create default roles if they don't exist. + * + * This is a potential contention when HiveServer2 using embedded metastore and Metastore + * Server try to concurrently invoke createDefaultRoles. If one failed, JDOException was caught + * for one more time try, if failed again, simply ignored by warning, which meant another + * succeeds. + * + * @throws MetaException + */ + private void createDefaultRoles() throws MetaException { + try { + createDefaultRoles_core(); + } catch (JDOException e) { + LOG.warn("Retrying creating default roles after error: " + e.getMessage(), e); + createDefaultRoles_core(); + } + } + + private void createDefaultRoles_core() throws MetaException { + + RawStore ms = getMS(); + try { + ms.addRole(ADMIN, ADMIN); + } catch (InvalidObjectException e) { + LOG.debug(ADMIN +" role already exists",e); + } catch (NoSuchObjectException e) { + // This should never be thrown. + LOG.warn("Unexpected exception while adding " +ADMIN+" roles" , e); + } + LOG.info("Added "+ ADMIN+ " role in metastore"); + try { + ms.addRole(PUBLIC, PUBLIC); + } catch (InvalidObjectException e) { + LOG.debug(PUBLIC + " role already exists",e); + } catch (NoSuchObjectException e) { + // This should never be thrown. + LOG.warn("Unexpected exception while adding "+PUBLIC +" roles" , e); + } + LOG.info("Added "+PUBLIC+ " role in metastore"); + // now grant all privs to admin + PrivilegeBag privs = new PrivilegeBag(); + privs.addToPrivileges(new HiveObjectPrivilege( new HiveObjectRef(HiveObjectType.GLOBAL, null, + null, null, null), ADMIN, PrincipalType.ROLE, new PrivilegeGrantInfo("All", 0, ADMIN, + PrincipalType.ROLE, true), "SQL")); + try { + ms.grantPrivileges(privs); + } catch (InvalidObjectException e) { + // Surprisingly these privs are already granted. + LOG.debug("Failed while granting global privs to admin", e); + } catch (NoSuchObjectException e) { + // Unlikely to be thrown. + LOG.warn("Failed while granting global privs to admin", e); + } + } + + /** + * add admin users if they don't exist. + * + * This is a potential contention when HiveServer2 using embedded metastore and Metastore + * Server try to concurrently invoke addAdminUsers. If one failed, JDOException was caught for + * one more time try, if failed again, simply ignored by warning, which meant another succeeds. + * + * @throws MetaException + */ + private void addAdminUsers() throws MetaException { + try { + addAdminUsers_core(); + } catch (JDOException e) { + LOG.warn("Retrying adding admin users after error: " + e.getMessage(), e); + addAdminUsers_core(); + } + } + + private void addAdminUsers_core() throws MetaException { + + // now add pre-configured users to admin role + String userStr = MetastoreConf.getVar(conf,ConfVars.USERS_IN_ADMIN_ROLE,"").trim(); + if (userStr.isEmpty()) { + LOG.info("No user is added in admin role, since config is empty"); + return; + } + // Since user names need to be valid unix user names, per IEEE Std 1003.1-2001 they cannot + // contain comma, so we can safely split above string on comma. + + Iterator<String> users = Splitter.on(",").trimResults().omitEmptyStrings().split(userStr).iterator(); + if (!users.hasNext()) { + LOG.info("No user is added in admin role, since config value "+ userStr + + " is in incorrect format. We accept comma separated list of users."); + return; + } + Role adminRole; + RawStore ms = getMS(); + try { + adminRole = ms.getRole(ADMIN); + } catch (NoSuchObjectException e) { + LOG.error("Failed to retrieve just added admin role",e); + return; + } + while (users.hasNext()) { + String userName = users.next(); + try { + ms.grantRole(adminRole, userName, PrincipalType.USER, ADMIN, PrincipalType.ROLE, true); + LOG.info("Added " + userName + " to admin role"); + } catch (NoSuchObjectException e) { + LOG.error("Failed to add "+ userName + " in admin role",e); + } catch (InvalidObjectException e) { + LOG.debug(userName + " already in admin role", e); + } + } + } + + private static void logInfo(String m) { + LOG.info(threadLocalId.get().toString() + ": " + m); + logAuditEvent(m); + } + + private String startFunction(String function, String extraLogInfo) { + incrementCounter(function); + logInfo((getThreadLocalIpAddress() == null ? "" : "source:" + getThreadLocalIpAddress() + " ") + + function + extraLogInfo); + com.codahale.metrics.Timer timer = + Metrics.getOrCreateTimer(MetricsConstants.API_PREFIX + function); + if (timer != null) { + // Timer will be null we aren't using the metrics + timerContexts.get().put(function, timer.time()); + } + Counter counter = Metrics.getOrCreateCounter(MetricsConstants.ACTIVE_CALLS + function); + if (counter != null) { + counter.inc(); + } + return function; + } + + private String startFunction(String function) { + return startFunction(function, ""); + } + + private void startTableFunction(String function, String catName, String db, String tbl) { + startFunction(function, " : tbl=" + + TableName.getQualified(catName, db, tbl)); + } + + private void startMultiTableFunction(String function, String db, List<String> tbls) { + String tableNames = join(tbls, ","); + startFunction(function, " : db=" + db + " tbls=" + tableNames); + } + + private void startPartitionFunction(String function, String cat, String db, String tbl, + List<String> partVals) { + startFunction(function, " : tbl=" + + TableName.getQualified(cat, db, tbl) + "[" + join(partVals, ",") + "]"); + } + + private void startPartitionFunction(String function, String catName, String db, String tbl, + Map<String, String> partName) { + startFunction(function, " : tbl=" + + TableName.getQualified(catName, db, tbl) + "partition=" + partName); + } + + private void endFunction(String function, boolean successful, Exception e) { + endFunction(function, successful, e, null); + } + private void endFunction(String function, boolean successful, Exception e, + String inputTableName) { + endFunction(function, new MetaStoreEndFunctionContext(successful, e, inputTableName)); + } + + private void endFunction(String function, MetaStoreEndFunctionContext context) { + com.codahale.metrics.Timer.Context timerContext = timerContexts.get().remove(function); + if (timerContext != null) { + timerContext.close(); + } + Counter counter = Metrics.getOrCreateCounter(MetricsConstants.ACTIVE_CALLS + function); + if (counter != null) { + counter.dec(); + } + + for (MetaStoreEndFunctionListener listener : endFunctionListeners) { + listener.onEndFunction(function, context); + } + } + + @Override + public fb_status getStatus() { + return fb_status.ALIVE; + } + + @Override + public void shutdown() { + cleanupRawStore(); + PerfLogger.getPerfLogger(false).cleanupPerfLogMetrics(); + } + + @Override + public AbstractMap<String, Long> getCounters() { + AbstractMap<String, Long> counters = super.getCounters(); + + // Allow endFunctionListeners to add any counters they have collected + if (endFunctionListeners != null) { + for (MetaStoreEndFunctionListener listener : endFunctionListeners) { + listener.exportCounters(counters); + } + } + + return counters; + } + + @Override + public void create_catalog(CreateCatalogRequest rqst) + throws AlreadyExistsException, InvalidObjectException, MetaException { + Catalog catalog = rqst.getCatalog(); + startFunction("create_catalog", ": " + catalog.toString()); + boolean success = false; + Exception ex = null; + try { + try { + getMS().getCatalog(catalog.getName()); + throw new AlreadyExistsException("Catalog " + catalog.getName() + " already exists"); + } catch (NoSuchObjectException e) { + // expected + } + + if (!MetaStoreUtils.validateName(catalog.getName(), null)) { + throw new InvalidObjectException(catalog.getName() + " is not a valid catalog name"); + } + + if (catalog.getLocationUri() == null) { + throw new InvalidObjectException("You must specify a path for the catalog"); + } + + RawStore ms = getMS(); + Path catPath = new Path(catalog.getLocationUri()); + boolean madeDir = false; + Map<String, String> transactionalListenersResponses = Collections.emptyMap(); + try { + firePreEvent(new PreCreateCatalogEvent(this, catalog)); + if (!wh.isDir(catPath)) { + if (!wh.mkdirs(catPath)) { + throw new MetaException("Unable to create catalog path " + catPath + + ", failed to create catalog " + catalog.getName()); + } + madeDir = true; + } + + ms.openTransaction(); + ms.createCatalog(catalog); + + // Create a default database inside the catalog + Database db = new Database(DEFAULT_DATABASE_NAME, "Default database for catalog " + + catalog.getName(), catalog.getLocationUri(), Collections.emptyMap()); + db.setCatalogName(catalog.getName()); + create_database_core(ms, db); + + if (!transactionalListeners.isEmpty()) { + transactionalListenersResponses = + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, + EventType.CREATE_CATALOG, + new CreateCatalogEvent(true, this, catalog)); + } + + success = ms.commitTransaction(); + } finally { + if (!success) { + ms.rollbackTransaction(); + if (madeDir) { + wh.deleteDir(catPath, true, false, false); + } + } + + if (!listeners.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(listeners, + EventType.CREATE_CATALOG, + new CreateCatalogEvent(success, this, catalog), + null, + transactionalListenersResponses, ms); + } + } + success = true; + } catch (AlreadyExistsException|InvalidObjectException|MetaException e) { + ex = e; + throw e; + } finally { + endFunction("create_catalog", success, ex); + } + } + + @Override + public void alter_catalog(AlterCatalogRequest rqst) throws TException { + startFunction("alter_catalog " + rqst.getName()); + boolean success = false; + Exception ex = null; + RawStore ms = getMS(); + Map<String, String> transactionalListenersResponses = Collections.emptyMap(); + GetCatalogResponse oldCat = null; + + try { + oldCat = get_catalog(new GetCatalogRequest(rqst.getName())); + // Above should have thrown NoSuchObjectException if there is no such catalog + assert oldCat != null && oldCat.getCatalog() != null; + firePreEvent(new PreAlterCatalogEvent(oldCat.getCatalog(), rqst.getNewCat(), this)); + + ms.openTransaction(); + ms.alterCatalog(rqst.getName(), rqst.getNewCat()); + + if (!transactionalListeners.isEmpty()) { + transactionalListenersResponses = + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, + EventType.ALTER_CATALOG, + new AlterCatalogEvent(oldCat.getCatalog(), rqst.getNewCat(), true, this)); + } + + success = ms.commitTransaction(); + } catch (MetaException|NoSuchObjectException e) { + ex = e; + throw e; + } finally { + if (!success) { + ms.rollbackTransaction(); + } + + if ((null != oldCat) && (!listeners.isEmpty())) { + MetaStoreListenerNotifier.notifyEvent(listeners, + EventType.ALTER_CATALOG, + new AlterCatalogEvent(oldCat.getCatalog(), rqst.getNewCat(), success, this), + null, transactionalListenersResponses, ms); + } + endFunction("alter_catalog", success, ex); + } + + } + + @Override + public GetCatalogResponse get_catalog(GetCatalogRequest rqst) + throws NoSuchObjectException, TException { + String catName = rqst.getName(); + startFunction("get_catalog", ": " + catName); + Catalog cat = null; + Exception ex = null; + try { + cat = getMS().getCatalog(catName); + firePreEvent(new PreReadCatalogEvent(this, cat)); + return new GetCatalogResponse(cat); + } catch (MetaException|NoSuchObjectException e) { + ex = e; + throw e; + } finally { + endFunction("get_database", cat != null, ex); + } + } + + @Override + public GetCatalogsResponse get_catalogs() throws MetaException { + startFunction("get_catalogs"); + + List<String> ret = null; + Exception ex = null; + try { + ret = getMS().getCatalogs(); + } catch (MetaException e) { + ex = e; + throw e; + } finally { + endFunction("get_catalog", ret != null, ex); + } + return new GetCatalogsResponse(ret == null ? Collections.emptyList() : ret); + + } + + @Override + public void drop_catalog(DropCatalogRequest rqst) + throws NoSuchObjectException, InvalidOperationException, MetaException { + String catName = rqst.getName(); + startFunction("drop_catalog", ": " + catName); + if (DEFAULT_CATALOG_NAME.equalsIgnoreCase(catName)) { + endFunction("drop_catalog", false, null); + throw new MetaException("Can not drop " + DEFAULT_CATALOG_NAME + " catalog"); + } + + boolean success = false; + Exception ex = null; + try { + dropCatalogCore(catName); + success = true; + } catch (NoSuchObjectException|InvalidOperationException|MetaException e) { + ex = e; + throw e; + } catch (Exception e) { + ex = e; + throw newMetaException(e); + } finally { + endFunction("drop_catalog", success, ex); + } + + } + + private void dropCatalogCore(String catName) + throws MetaException, NoSuchObjectException, InvalidOperationException { + boolean success = false; + Catalog cat = null; + Map<String, String> transactionalListenerResponses = Collections.emptyMap(); + RawStore ms = getMS(); + try { + ms.openTransaction(); + cat = ms.getCatalog(catName); + + firePreEvent(new PreDropCatalogEvent(this, cat)); + + List<String> allDbs = get_databases(prependNotNullCatToDbName(catName, null)); + if (allDbs != null && !allDbs.isEmpty()) { + // It might just be the default, in which case we can drop that one if it's empty + if (allDbs.size() == 1 && allDbs.get(0).equals(DEFAULT_DATABASE_NAME)) { + try { + drop_database_core(ms, catName, DEFAULT_DATABASE_NAME, true, false); + } catch (InvalidOperationException e) { + // This means there are tables of something in the database + throw new InvalidOperationException("There are still objects in the default " + + "database for catalog " + catName); + } catch (InvalidObjectException|IOException|InvalidInputException e) { + MetaException me = new MetaException("Error attempt to drop default database for " + + "catalog " + catName); + me.initCause(e); + throw me; + } + } else { + throw new InvalidOperationException("There are non-default databases in the catalog " + + catName + " so it cannot be dropped."); + } + } + + ms.dropCatalog(catName) ; + if (!transactionalListeners.isEmpty()) { + transactionalListenerResponses = + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, + EventType.DROP_CATALOG, + new DropCatalogEvent(true, this, cat)); + } + + success = ms.commitTransaction(); + } finally { + if (success) { + wh.deleteDir(wh.getDnsPath(new Path(cat.getLocationUri())), false, false, false); + } else { + ms.rollbackTransaction(); + } + + if (!listeners.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(listeners, + EventType.DROP_CATALOG, + new DropCatalogEvent(success, this, cat), + null, + transactionalListenerResponses, ms); + } + } + } + + + // Assumes that the catalog has already been set. + private void create_database_core(RawStore ms, final Database db) + throws AlreadyExistsException, InvalidObjectException, MetaException { + if (!MetaStoreUtils.validateName(db.getName(), null)) { + throw new InvalidObjectException(db.getName() + " is not a valid database name"); + } + + Catalog cat = null; + try { + cat = getMS().getCatalog(db.getCatalogName()); + } catch (NoSuchObjectException e) { + LOG.error("No such catalog " + db.getCatalogName()); + throw new InvalidObjectException("No such catalog " + db.getCatalogName()); + } + Path dbPath = wh.determineDatabasePath(cat, db); + db.setLocationUri(dbPath.toString()); + + boolean success = false; + boolean madeDir = false; + Map<String, String> transactionalListenersResponses = Collections.emptyMap(); + try { + firePreEvent(new PreCreateDatabaseEvent(db, this)); + if (!wh.isDir(dbPath)) { + LOG.debug("Creating database path " + dbPath); + if (!wh.mkdirs(dbPath)) { + throw new MetaException("Unable to create database path " + dbPath + + ", failed to create database " + db.getName()); + } + madeDir = true; + } + + ms.openTransaction(); + ms.createDatabase(db); + + if (!transactionalListeners.isEmpty()) { + transactionalListenersResponses = + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, + EventType.CREATE_DATABASE, + new CreateDatabaseEvent(db, true, this)); + } + + success = ms.commitTransaction(); + } finally { + if (!success) { + ms.rollbackTransaction(); + if (madeDir) { + wh.deleteDir(dbPath, true, db); + } + } + + if (!listeners.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(listeners, + EventType.CREATE_DATABASE, + new CreateDatabaseEvent(db, success, this), + null, + transactionalListenersResponses, ms); + } + } + } + + @Override + public void create_database(final Database db) + throws AlreadyExistsException, InvalidObjectException, MetaException { + startFunction("create_database", ": " + db.toString()); + boolean success = false; + Exception ex = null; + if (!db.isSetCatalogName()) { + db.setCatalogName(getDefaultCatalog(conf)); + } + try { + try { + if (null != get_database_core(db.getCatalogName(), db.getName())) { + throw new AlreadyExistsException("Database " + db.getName() + " already exists"); + } + } catch (NoSuchObjectException e) { + // expected + } + + if (TEST_TIMEOUT_ENABLED) { + try { + Thread.sleep(TEST_TIMEOUT_VALUE); + } catch (InterruptedException e) { + // do nothing + } + Deadline.checkTimeout(); + } + create_database_core(getMS(), db); + success = true; + } catch (MetaException | InvalidObjectException | AlreadyExistsException e) { + ex = e; + throw e; + } catch (Exception e) { + ex = e; + throw newMetaException(e); + } finally { + endFunction("create_database", success, ex); + } + } + + @Override + public Database get_database(final String name) throws NoSuchObjectException, MetaException { + startFunction("get_database", ": " + name); + Database db = null; + Exception ex = null; + try { + String[] parsedDbName = parseDbName(name, conf); + db = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]); + firePreEvent(new PreReadDatabaseEvent(db, this)); + } catch (MetaException|NoSuchObjectException e) { + ex = e; + throw e; + } finally { + endFunction("get_database", db != null, ex); + } + return db; + } + + @Override + public Database get_database_core(String catName, final String name) throws NoSuchObjectException, MetaException { + Database db = null; + if (name == null) { + throw new MetaException("Database name cannot be null."); + } + try { + db = getMS().getDatabase(catName, name); + } catch (MetaException | NoSuchObjectException e) { + throw e; + } catch (Exception e) { + assert (e instanceof RuntimeException); + throw (RuntimeException) e; + } + return db; + } + + @Override + public void alter_database(final String dbName, final Database newDB) throws TException { + startFunction("alter_database " + dbName); + boolean success = false; + Exception ex = null; + RawStore ms = getMS(); + Database oldDB = null; + Map<String, String> transactionalListenersResponses = Collections.emptyMap(); + + // Perform the same URI normalization as create_database_core. + if (newDB.getLocationUri() != null) { + newDB.setLocationUri(wh.getDnsPath(new Path(newDB.getLocationUri())).toString()); + } + + String[] parsedDbName = parseDbName(dbName, conf); + + try { + oldDB = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]); + if (oldDB == null) { + throw new MetaException("Could not alter database \"" + parsedDbName[DB_NAME] + + "\". Could not retrieve old definition."); + } + firePreEvent(new PreAlterDatabaseEvent(oldDB, newDB, this)); + + ms.openTransaction(); + ms.alterDatabase(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], newDB); + + if (!transactionalListeners.isEmpty()) { + transactionalListenersResponses = + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, + EventType.ALTER_DATABASE, + new AlterDatabaseEvent(oldDB, newDB, true, this)); + } + + success = ms.commitTransaction(); + } catch (MetaException|NoSuchObjectException e) { + ex = e; + throw e; + } finally { + if (!success) { + ms.rollbackTransaction(); + } + + if ((null != oldDB) && (!listeners.isEmpty())) { + MetaStoreListenerNotifier.notifyEvent(listeners, + EventType.ALTER_DATABASE, + new AlterDatabaseEvent(oldDB, newDB, success, this), + null, + transactionalListenersResponses, ms); + } + endFunction("alter_database", success, ex); + } + } + + private void drop_database_core(RawStore ms, String catName, + final String name, final boolean deleteData, final boolean cascade) + throws NoSuchObjectException, InvalidOperationException, MetaException, + IOException, InvalidObjectException, InvalidInputException { + boolean success = false; + Database db = null; + List<Path> tablePaths = new ArrayList<>(); + List<Path> partitionPaths = new ArrayList<>(); + Map<String, String> transactionalListenerResponses = Collections.emptyMap(); + if (name == null) { + throw new MetaException("Database name cannot be null."); + } + try { + ms.openTransaction(); + db = ms.getDatabase(catName, name); + + if (!isInTest && ReplChangeManager.isSourceOfReplication(db)) { + throw new InvalidOperationException("can not drop a database which is a source of replication"); + } + + firePreEvent(new PreDropDatabaseEvent(db, this)); + String catPrependedName = MetaStoreUtils.prependCatalogToDbName(catName, name, conf); + + Set<String> uniqueTableNames = new HashSet<>(get_all_tables(catPrependedName)); + List<String> allFunctions = get_functions(catPrependedName, "*"); + + if (!cascade) { + if (!uniqueTableNames.isEmpty()) { + throw new InvalidOperationException( + "Database " + db.getName() + " is not empty. One or more tables exist."); + } + if (!allFunctions.isEmpty()) { + throw new InvalidOperationException( + "Database " + db.getName() + " is not empty. One or more functions exist."); + } + } + Path path = new Path(db.getLocationUri()).getParent(); + if (!wh.isWritable(path)) { + throw new MetaException("Database not dropped since " + + path + " is not writable by " + + SecurityUtils.getUser()); + } + + Path databasePath = wh.getDnsPath(wh.getDatabasePath(db)); + + // drop any functions before dropping db + for (String funcName : allFunctions) { + drop_function(catPrependedName, funcName); + } + + final int tableBatchSize = MetastoreConf.getIntVar(conf, + ConfVars.BATCH_RETRIEVE_MAX); + + // First pass will drop the materialized views + List<String> materializedViewNames = get_tables_by_type(name, ".*", TableType.MATERIALIZED_VIEW.toString()); + int startIndex = 0; + // retrieve the tables from the metastore in batches to alleviate memory constraints + while (startIndex < materializedViewNames.size()) { + int endIndex = Math.min(startIndex + tableBatchSize, materializedViewNames.size()); + + List<Table> materializedViews; + try { + materializedViews = ms.getTableObjectsByName(catName, name, materializedViewNames.subList(startIndex, endIndex)); + } catch (UnknownDBException e) { + throw new MetaException(e.getMessage()); + } + + if (materializedViews != null && !materializedViews.isEmpty()) { + for (Table materializedView : materializedViews) { + if (materializedView.getSd().getLocation() != null) { + Path materializedViewPath = wh.getDnsPath(new Path(materializedView.getSd().getLocation())); + if (!wh.isWritable(materializedViewPath.getParent())) { + throw new MetaException("Database metadata not deleted since table: " + + materializedView.getTableName() + " has a parent location " + materializedViewPath.getParent() + + " which is not writable by " + SecurityUtils.getUser()); + } + + if (!FileUtils.isSubdirectory(databasePath.toString(), + materializedViewPath.toString())) { + tablePaths.add(materializedViewPath); + } + } + // Drop the materialized view but not its data + drop_table(name, materializedView.getTableName(), false); + // Remove from all tables + uniqueTableNames.remove(materializedView.getTableName()); + } + } + startIndex = endIndex; + } + + // drop tables before dropping db + List<String> allTables = new ArrayList<>(uniqueTableNames); + startIndex = 0; + // retrieve the tables from the metastore in batches to alleviate memory constraints + while (startIndex < allTables.size()) { + int endIndex = Math.min(startIndex + tableBatchSize, allTables.size()); + + List<Table> tables; + try { + tables = ms.getTableObjectsByName(catName, name, allTables.subList(startIndex, endIndex)); + } catch (UnknownDBException e) { + throw new MetaException(e.getMessage()); + } + + if (tables != null && !tables.isEmpty()) { + for (Table table : tables) { + + // If the table is not external and it might not be in a subdirectory of the database + // add it's locations to the list of paths to delete + Path tablePath = null; + boolean tableDataShouldBeDeleted = checkTableDataShouldBeDeleted(table, deleteData); + if (table.getSd().getLocation() != null && tableDataShouldBeDeleted) { + tablePath = wh.getDnsPath(new Path(table.getSd().getLocation())); + if (!wh.isWritable(tablePath.getParent())) { + throw new MetaException("Database metadata not deleted since table: " + + table.getTableName() + " has a parent location " + tablePath.getParent() + + " which is not writable by " + SecurityUtils.getUser()); + } + + if (!FileUtils.isSubdirectory(databasePath.toString(), tablePath.toString())) { + tablePaths.add(tablePath); + } + } + + // For each partition in each table, drop the partitions and get a list of + // partitions' locations which might need to be deleted + partitionPaths = dropPartitionsAndGetLocations(ms, catName, name, table.getTableName(), + tablePath, tableDataShouldBeDeleted); + + // Drop the table but not its data + drop_table(MetaStoreUtils.prependCatalogToDbName(table.getCatName(), table.getDbName(), conf), + table.getTableName(), false); + } + + startIndex = endIndex; + } + } + + if (ms.dropDatabase(catName, name)) { + if (!transactionalListeners.isEmpty()) { + transactionalListenerResponses = + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, + EventType.DROP_DATABASE, + new DropDatabaseEvent(db, true, this)); + } + + success = ms.commitTransaction(); + } + } finally { + if (!success) { + ms.rollbackTransaction(); + } else if (deleteData) { + // Delete the data in the partitions which have other locations + deletePartitionData(partitionPaths, false, db); + // Delete the data in the tables which have other locations + for (Path tablePath : tablePaths) { + deleteTableData(tablePath, false, db); + } + // Delete the data in the database + try { + wh.deleteDir(new Path(db.getLocationUri()), true, db); + } catch (Exception e) { + LOG.error("Failed to delete database directory: " + db.getLocationUri() + + " " + e.getMessage()); + } + // it is not a terrible thing even if the data is not deleted + } + + if (!listeners.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(listeners, + EventType.DROP_DATABASE, + new DropDatabaseEvent(db, success, this), + null, + transactionalListenerResponses, ms); + } + } + } + + @Override + public void drop_database(final String dbName, final boolean deleteData, final boolean cascade) + throws NoSuchObjectException, InvalidOperationException, MetaException { + startFunction("drop_database", ": " + dbName); + String[] parsedDbName = parseDbName(dbName, conf); + if (DEFAULT_CATALOG_NAME.equalsIgnoreCase(parsedDbName[CAT_NAME]) && + DEFAULT_DATABASE_NAME.equalsIgnoreCase(parsedDbName[DB_NAME])) { + endFunction("drop_database", false, null); + throw new MetaException("Can not drop " + DEFAULT_DATABASE_NAME + " database in catalog " + + DEFAULT_CATALOG_NAME); + } + + boolean success = false; + Exception ex = null; + try { + drop_database_core(getMS(), parsedDbName[CAT_NAME], parsedDbName[DB_NAME], deleteData, + cascade); + success = true; + } catch (NoSuchObjectException|InvalidOperationException|MetaException e) { + ex = e; + throw e; + } catch (Exception e) { + ex = e; + throw newMetaException(e); + } finally { + endFunction("drop_database", success, ex); + } + } + + + @Override + public List<String> get_databases(final String pattern) throws MetaException { + startFunction("get_databases", ": " + pattern); + + String[] parsedDbNamed = parseDbName(pattern, conf); + List<String> ret = null; + Exception ex = null; + try { + if (parsedDbNamed[DB_NAME] == null) { + ret = getMS().getAllDatabases(parsedDbNamed[CAT_NAME]); + } else { + ret = getMS().getDatabases(parsedDbNamed[CAT_NAME], parsedDbNamed[DB_NAME]); + } + } catch (MetaException e) { + ex = e; + throw e; + } catch (Exception e) { + ex = e; + throw newMetaException(e); + } finally { + endFunction("get_databases", ret != null, ex); + } + return ret; + } + + @Override + public List<String> get_all_databases() throws MetaException { + return get_databases(MetaStoreUtils.prependCatalogToDbName(null, null, conf)); + } + + private void create_type_core(final RawStore ms, final Type type) + throws AlreadyExistsException, MetaException, InvalidObjectException { + if (!MetaStoreUtils.validateName(type.getName(), null)) { + throw new InvalidObjectException("Invalid type name"); + } + + boolean success = false; + try { + ms.openTransaction(); + if (is_type_exists(ms, type.getName())) { + throw new AlreadyExistsException("Type " + type.getName() + " already exists"); + } + ms.createType(type); + success = ms.commitTransaction(); + } finally { + if (!success) { + ms.rollbackTransaction(); + } + } + } + + @Override + public boolean create_type(final Type type) throws AlreadyExistsException, + MetaException, InvalidObjectException { + startFunction("create_type", ": " + type.toString()); + boolean success = false; + Exception ex = null; + try { + create_type_core(getMS(), type); + success = true; + } catch (MetaException | InvalidObjectException | AlreadyExistsException e) { + ex = e; + throw e; + } catch (Exception e) { + ex = e; + throw newMetaException(e); + } finally { + endFunction("create_type", success, ex); + } + + return success; + } + + @Override + public Type get_type(final String name) throws MetaException, NoSuchObjectException { + startFunction("get_type", ": " + name); + + Type ret = null; + Exception ex = null; + try { + ret = getMS().getType(name); + if (null == ret) { + throw new NoSuchObjectException("Type \"" + name + "\" not found."); + } + } catch (Exception e) { + ex = e; + throwMetaException(e); + } finally { + endFunction("get_type", ret != null, ex); + } + return ret; + } + + private boolean is_type_exists(RawStore ms, String typeName) + throws MetaException { + return (ms.getType(typeName) != null); + } + + @Override + public boolean drop_type(final String name) throws MetaException, NoSuchObjectException { + startFunction("drop_type", ": " + name); + + boolean success = false; + Exception ex = null; + try { + // TODO:pc validate that there are no types that refer to this + success = getMS().dropType(name); + } catch (Exception e) { + ex = e; + throwMetaException(e); + } finally { + endFunction("drop_type", success, ex); + } + return success; + } + + @Override + public Map<String, Type> get_type_all(String name) throws MetaException { + // TODO Auto-generated method stub + startFunction("get_type_all", ": " + name); + endFunction("get_type_all", false, null); + throw new MetaException("Not yet implemented"); + } + + private void create_table_core(final RawStore ms, final Table tbl, + final EnvironmentContext envContext) + throws AlreadyExistsException, MetaException, + InvalidObjectException, NoSuchObjectException { + create_table_core(ms, tbl, envContext, null, null, null, null, null, null); + } + + private void create_table_core(final RawStore ms, final Table tbl, + final EnvironmentContext envContext, List<SQLPrimaryKey> primaryKeys, + List<SQLForeignKey> foreignKeys, List<SQLUniqueConstraint> uniqueConstraints, + List<SQLNotNullConstraint> notNullConstraints, List<SQLDefaultConstraint> defaultConstraints, + List<SQLCheckConstraint> checkConstraints) + throws AlreadyExistsException, MetaException, + InvalidObjectException, NoSuchObjectException { + // To preserve backward compatibility throw MetaException in case of null database + if (tbl.getDbName() == null) { + throw new MetaException("Null database name is not allowed"); + } + + if (!MetaStoreUtils.validateName(tbl.getTableName(), conf)) { + throw new InvalidObjectException(tbl.getTableName() + + " is not a valid object name"); + } + String validate = MetaStoreUtils.validateTblColumns(tbl.getSd().getCols()); + if (validate != null) { + throw new InvalidObjectException("Invalid column " + validate); + } + if (tbl.getPartitionKeys() != null) { + validate = MetaStoreUtils.validateTblColumns(tbl.getPartitionKeys()); + if (validate != null) { + throw new InvalidObjectException("Invalid partition column " + validate); + } + } + SkewedInfo skew = tbl.getSd().getSkewedInfo(); + if (skew != null) { + validate = MetaStoreUtils.validateSkewedColNames(skew.getSkewedColNames()); + if (validate != null) { + throw new InvalidObjectException("Invalid skew column " + validate); + } + validate = MetaStoreUtils.validateSkewedColNamesSubsetCol( + skew.getSkewedColNames(), tbl.getSd().getCols()); + if (validate != null) { + throw new InvalidObjectException("Invalid skew column " + validate); + } + } + + Map<String, String> transactionalListenerResponses = Collections.emptyMap(); + Path tblPath = null; + boolean success = false, madeDir = false; + Database db = null; + try { + if (!tbl.isSetCatName()) { + tbl.setCatName(getDefaultCatalog(conf)); + } + firePreEvent(new PreCreateTableEvent(tbl, this)); + + ms.openTransaction(); + + db = ms.getDatabase(tbl.getCatName(), tbl.getDbName()); + + // get_table checks whether database exists, it should be moved here + if (is_table_exists(ms, tbl.getCatName(), tbl.getDbName(), tbl.getTableName())) { + throw new AlreadyExistsException("Table " + getCatalogQualifiedTableName(tbl) + + " already exists"); + } + + if (!TableType.VIRTUAL_VIEW.toString().equals(tbl.getTableType())) { + if (tbl.getSd().getLocation() == null + || tbl.getSd().getLocation().isEmpty()) { + tblPath = wh.getDefaultTablePath(db, tbl); + } else { + if (!isExternal(tbl) && !MetaStoreUtils.isNonNativeTable(tbl)) { + LOG.warn("Location: " + tbl.getSd().getLocation() + + " specified for non-external table:" + tbl.getTableName()); + } + tblPath = wh.getDnsPath(new Path(tbl.getSd().getLocation())); + } + tbl.getSd().setLocation(tblPath.toString()); + } + + if (tblPath != null) { + if (!wh.isDir(tblPath)) { + if (!wh.mkdirs(tblPath)) { + throw new MetaException(tblPath + + " is not a directory or unable to create one"); + } + madeDir = true; + } + } + if (MetastoreConf.getBoolVar(conf, ConfVars.STATS_AUTO_GATHER) && + !MetaStoreUtils.isView(tbl)) { + MetaStoreUtils.updateTableStatsSlow(db, tbl, wh, madeDir, false, envContext); + } + + // set create time + long time = System.currentTimeMillis() / 1000; + tbl.setCreateTime((int) time); + if (tbl.getParameters() == null || + tbl.getParameters().get(hive_metastoreConstants.DDL_TIME) == null) { + tbl.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(time)); + } + + if (primaryKeys == null && foreignKeys == null + && uniqueConstraints == null && notNullConstraints == null && defaultConstraints == null + && checkConstraints == null) { + ms.createTable(tbl); + } else { + // Check that constraints have catalog name properly set first + if (primaryKeys != null && !primaryKeys.isEmpty() && !primaryKeys.get(0).isSetCatName()) { + for (SQLPrimaryKey pkcol : primaryKeys) pkcol.setCatName(tbl.getCatName()); + } + if (foreignKeys != null && !foreignKeys.isEmpty() && !foreignKeys.get(0).isSetCatName()) { + for (SQLForeignKey fkcol : foreignKeys) fkcol.setCatName(tbl.getCatName()); + } + if (uniqueConstraints != null && !uniqueConstraints.isEmpty() && !uniqueConstraints.get(0).isSetCatName()) { + for (SQLUniqueConstraint uccol : uniqueConstraints) uccol.setCatName(tbl.getCatName()); + } + if (notNullConstraints != null && !notNullConstraints.isEmpty() && !notNullConstraints.get(0).isSetCatName()) { + for (SQLNotNullConstraint nncol : notNullConstraints) nncol.setCatName(tbl.getCatName()); + } + if (defaultConstraints != null && !defaultConstraints.isEmpty() && !defaultConstraints.get(0).isSetCatName()) { + for (SQLDefaultConstraint dccol : defaultConstraints) dccol.setCatName(tbl.getCatName()); + } + if (checkConstraints != null && !checkConstraints.isEmpty() && !checkConstraints.get(0).isSetCatName()) { + for (SQLCheckConstraint cccol : checkConstraints) cccol.setCatName(tbl.getCatName()); + } + // Set constraint name if null before sending to listener + List<String> constraintNames = ms.createTableWithConstraints(tbl, primaryKeys, foreignKeys, + uniqueConstraints, notNullConstraints, defaultConstraints, checkConstraints); + int primaryKeySize = 0; + if (primaryKeys != null) { + primaryKeySize = primaryKeys.size(); + for (int i = 0; i < primaryKeys.size(); i++) { + if (primaryKeys.get(i).getPk_name() == null) { + primaryKeys.get(i).setPk_name(constraintNames.get(i)); + } + if (!primaryKeys.get(i).isSetCatName()) primaryKeys.get(i).setCatName(tbl.getCatName()); + } + } + int foreignKeySize = 0; + if (foreignKeys != null) { + foreignKeySize = foreignKeys.size(); + for (int i = 0; i < foreignKeySize; i++) { + if (foreignKeys.get(i).getFk_name() == null) { + foreignKeys.get(i).setFk_name(constraintNames.get(primaryKeySize + i)); + } + if (!foreignKeys.get(i).isSetCatName()) foreignKeys.get(i).setCatName(tbl.getCatName()); + } + } + int uniqueConstraintSize = 0; + if (uniqueConstraints != null) { + uniqueConstraintSize = uniqueConstraints.size(); + for (int i = 0; i < uniqueConstraintSize; i++) { + if (uniqueConstraints.get(i).getUk_name() == null) { + uniqueConstraints.get(i).setUk_name(constraintNames.get(primaryKeySize + foreignKeySize + i)); + } + if (!uniqueConstraints.get(i).isSetCatName()) uniqueConstraints.get(i).setCatName(tbl.getCatName()); + } + } + int notNullConstraintSize = 0; + if (notNullConstraints != null) { + for (int i = 0; i < notNullConstraints.size(); i++) { + if (notNullConstraints.get(i).getNn_name() == null) { + notNullConstraints.get(i).setNn_name(constraintNames.get(primaryKeySize + foreignKeySize + uniqueConstraintSize + i)); + } + if (!notNullConstraints.get(i).isSetCatName()) notNullConstraints.get(i).setCatName(tbl.getCatName()); + } + } + int defaultConstraintSize = 0; + if (defaultConstraints!= null) { + for (int i = 0; i < defaultConstraints.size(); i++) { + if (defaultConstraints.get(i).getDc_name() == null) { + defaultConstraints.get(i).setDc_name(constraintNames.get(primaryKeySize + foreignKeySize + + uniqueConstraintSize + notNullConstraintSize + i)); + } + if (!defaultConstraints.get(i).isSetCatName()) defaultConstraints.get(i).setCatName(tbl.getCatName()); + } + } + if (checkConstraints!= null) { + for (int i = 0; i < checkConstraints.size(); i++) { + if (checkConstraints.get(i).getDc_name() == null) { + checkConstraints.get(i).setDc_name(constraintNames.get(primaryKeySize + foreignKeySize + + uniqueConstraintSize + + defaultConstraintSize + + notNullConstraintSize + i)); + } + if (!checkConstraints.get(i).isSetCatName()) checkConstraints.get(i).setCatName(tbl.getCatName()); + } + } + } + + if (!transactionalListeners.isEmpty()) { + transactionalListenerResponses = MetaStoreListenerNotifier.notifyEvent(transactionalListeners, + EventType.CREATE_TABLE, new CreateTableEvent(tbl, true, this), envContext); + if (primaryKeys != null && !primaryKeys.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.ADD_PRIMARYKEY, + new AddPrimaryKeyEvent(primaryKeys, true, this), envContext); + } + if (foreignKeys != null && !foreignKeys.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.ADD_FOREIGNKEY, + new AddForeignKeyEvent(foreignKeys, true, this), envContext); + } + if (uniqueConstraints != null && !uniqueConstraints.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.ADD_UNIQUECONSTRAINT, + new AddUniqueConstraintEvent(uniqueConstraints, true, this), envContext); + } + if (notNullConstraints != null && !notNullConstraints.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.ADD_NOTNULLCONSTRAINT, + new AddNotNullConstraintEvent(notNullConstraints, true, this), envContext); + } + } + + success = ms.commitTransaction(); + } finally { + if (!success) { + ms.rollbackTransaction(); + if (madeDir) { + wh.deleteDir(tblPath, true, db); + } + } + + if (!listeners.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(listeners, EventType.CREATE_TABLE, + new CreateTableEvent(tbl, success, this), envContext, transactionalListenerResponses, ms); + if (primaryKeys != null && !primaryKeys.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_PRIMARYKEY, + new AddPrimaryKeyEvent(primaryKeys, success, this), envContext); + } + if (foreignKeys != null && !foreignKeys.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_FOREIGNKEY, + new AddForeignKeyEvent(foreignKeys, success, this), envContext); + } + if (uniqueConstraints != null && !uniqueConstraints.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_UNIQUECONSTRAINT, + new AddUniqueConstraintEvent(uniqueConstraints, success, this), envContext); + } + if (notNullConstraints != null && !notNullConstraints.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_NOTNULLCONSTRAINT, + new AddNotNullConstraintEvent(notNullConstraints, success, this), envContext); + } + } + } + } + + @Override + public void create_table(final Table tbl) throws AlreadyExistsException, + MetaException, InvalidObjectException { + create_table_with_environment_context(tbl, null); + } + + @Override + public void create_table_with_environment_context(final Table tbl, + final EnvironmentContext envContext) + throws AlreadyExistsException, MetaException, InvalidObjectException { + startFunction("create_table", ": " + tbl.toString()); + boolean success = false; + Exception ex = null; + try { + create_table_core(getMS(), tbl, envContext); + success = true; + } catch (NoSuchObjectException e) { + LOG.warn("create_table_with_environment_context got ", e); + ex = e; + throw new InvalidObjectException(e.getMessage()); + } catch (MetaException | InvalidObjectException | AlreadyExistsException e) { + ex = e; + throw e; + } catch (Exception e) { + ex = e; + throw newMetaException(e); + } finally { + endFunction("create_table", success, ex, tbl.getTableName()); + } + } + + @Override + public void create_table_with_constraints(final Table tbl, + final List<SQLPrimaryKey> primaryKeys, final List<SQLForeignKey> foreignKeys, + List<SQLUniqueConstraint> uniqueConstraints, + List<SQLNotNullConstraint> notNullConstraints, + List<SQLDefaultConstraint> defaultConstraints, + List<SQLCheckConstraint> checkConstraints) + throws AlreadyExistsException, MetaException, InvalidObjectException { + startFunction("create_table", ": " + tbl.toString()); + boolean success = false; + Exception ex = null; + try { + create_table_core(getMS(), tbl, null, primaryKeys, foreignKeys, + uniqueConstraints, notNullConstraints, defaultConstraints, checkConstraints); + success = true; + } catch (NoSuchObjectException e) { + ex = e; + throw new InvalidObjectException(e.getMessage()); + } catch (MetaException | InvalidObjectException | AlreadyExistsException e) { + ex = e; + throw e; + } catch (Exception e) { + ex = e; + throw newMetaException(e); + } finally { + endFunction("create_table", success, ex, tbl.getTableName()); + } + } + + @Override + public void drop_constraint(DropConstraintRequest req) + throws MetaException, InvalidObjectException { + String catName = req.isSetCatName() ? req.getCatName() : getDefaultCatalog(conf); + String dbName = req.getDbname(); + String tableName = req.getTablename(); + String constraintName = req.getConstraintname(); + startFunction("drop_constraint", ": " + constraintName); + boolean success = false; + Exception ex = null; + RawStore ms = getMS(); + try { + ms.openTransaction(); + ms.dropConstraint(catName, dbName, tableName, constraintName); + if (transactionalListeners.size() > 0) { + DropConstraintEvent dropConstraintEvent = new DropConstraintEvent(catName, dbName, + tableName, constraintName, true, this); + for (MetaStoreEventListener transactionalListener : transactionalListeners) { + transactionalListener.onDropConstraint(dropConstraintEvent); + } + } + success = ms.commitTransaction(); + } catch (NoSuchObjectException e) { + ex = e; + throw new InvalidObjectException(e.getMessage()); + } catch (MetaException e)
<TRUNCATED>
