HIVE-20905 : querying streaming table fails with out of memory exception (mahesh kumar behera via Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/148e7acb Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/148e7acb Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/148e7acb Branch: refs/heads/master-tez092 Commit: 148e7acba46da997a023b57794c7f1f209097320 Parents: 52f94b8 Author: Mahesh Kumar Behera <mbeh...@hortonworks.com> Authored: Tue Nov 13 14:03:23 2018 -0700 Committer: Thejas M Nair <the...@hortonworks.com> Committed: Tue Nov 13 14:03:23 2018 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/txn/compactor/Cleaner.java | 12 +++++++----- .../org/apache/hadoop/hive/metastore/HiveMetaStore.java | 3 +++ 2 files changed, 10 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/148e7acb/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java index 3565616..3bc1f8a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java @@ -20,8 +20,7 @@ package org.apache.hadoop.hive.ql.txn.compactor; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.metastore.ReplChangeManager; import org.apache.hadoop.hive.metastore.txn.TxnStore; -import org.apache.hadoop.hive.ql.metadata.Hive; -import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.FileStatus; @@ -58,6 +57,8 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; + /** * A class to clean directories after compactions. This will run in a separate thread. */ @@ -323,7 +324,7 @@ public class Cleaner extends CompactorThread { return " id=" + ci.id; } private void removeFiles(String location, ValidWriteIdList writeIdList, CompactionInfo ci) - throws IOException, HiveException { + throws IOException, NoSuchObjectException { Path locPath = new Path(location); AcidUtils.Directory dir = AcidUtils.getAcidState(locPath, conf, writeIdList); List<FileStatus> obsoleteDirs = dir.getObsolete(); @@ -349,11 +350,12 @@ public class Cleaner extends CompactorThread { } FileSystem fs = filesToDelete.get(0).getFileSystem(conf); - Database db = Hive.get().getDatabase(ci.dbname); + Database db = rs.getDatabase(getDefaultCatalog(conf), ci.dbname); + Boolean isSourceOfRepl = ReplChangeManager.isSourceOfReplication(db); for (Path dead : filesToDelete) { LOG.debug("Going to delete path " + dead.toString()); - if (ReplChangeManager.isSourceOfReplication(db)) { + if (isSourceOfRepl) { replChangeManager.recycle(dead, ReplChangeManager.RecycleType.MOVE, true); } fs.delete(dead, true); http://git-wip-us.apache.org/repos/asf/hive/blob/148e7acb/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 0485184..23a78ca 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -9085,6 +9085,9 @@ public class HiveMetaStore extends ThriftHiveMetastore { conf.set((String) item.getKey(), (String) item.getValue()); } + //for metastore process, all metastore call should be embedded metastore call. + conf.set(ConfVars.THRIFT_URIS.getHiveName(), ""); + // Add shutdown hook. shutdownHookMgr.addShutdownHook(() -> { String shutdownMsg = "Shutting down hive metastore.";
