KYLIN-3092 fix cyclic lock
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f37fbce1 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f37fbce1 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f37fbce1 Branch: refs/heads/master Commit: f37fbce17f6e25775877290ba1f0de076c62f32f Parents: 9f603d3 Author: Li Yang <liy...@apache.org> Authored: Wed Dec 20 21:50:54 2017 +0800 Committer: Hongbin Ma <m...@kyligence.io> Committed: Thu Dec 21 11:06:34 2017 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/common/util/HiveCmdBuilder.java | 4 ++-- .../main/java/org/apache/kylin/cube/CubeDescManager.java | 9 --------- .../kylin/job/impl/threadpool/DefaultScheduler.java | 5 +++-- .../org/apache/kylin/metadata/TableMetadataManager.java | 10 +++++++--- 4 files changed, 12 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f37fbce1/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java index 10d6798..77dc579 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java @@ -188,8 +188,8 @@ public class HiveCmdBuilder { hiveConfFile = new File(path + File.separator + "conf", hiveConfFileName); } - if (hiveConfFile == null || !hiveConfFile.exists()) { - throw new RuntimeException("Failed to read " + HIVE_CONF_FILENAME + ".xml"); + if (!hiveConfFile.exists()) { + throw new RuntimeException("Missing config file: " + hiveConfFile.getAbsolutePath()); } String fileUrl = OptionsHelper.convertToFileURL(hiveConfFile.getAbsolutePath()); http://git-wip-us.apache.org/repos/asf/kylin/blob/f37fbce1/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java index a58ba40..a416499 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java @@ -46,7 +46,6 @@ import org.apache.kylin.metadata.model.ParameterDesc; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.realization.IRealization; -import org.apache.kylin.metadata.realization.RealizationStatusEnum; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -166,14 +165,6 @@ public class CubeDescManager { if (ndesc.isBroken()) throw new IllegalStateException("CubeDesc " + name + " is broken"); - // if related cube is in DESCBROKEN state before, change it back to DISABLED - CubeManager cubeManager = CubeManager.getInstance(config); - for (CubeInstance cube : cubeManager.getCubesByDesc(name)) { - if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) { - cube.init(config); - } - } - return ndesc; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/f37fbce1/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java index 78f10bf..c8ab7ea 100644 --- a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java +++ b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java @@ -163,8 +163,6 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti try (SetThreadName ignored = new SetThreadName("Scheduler %s Job %s", System.identityHashCode(DefaultScheduler.this), executable.getId())) { executable.execute(context); - // trigger the next step asap - fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS); } catch (ExecuteException e) { logger.error("ExecuteException job:" + executable.getId(), e); } catch (Exception e) { @@ -172,6 +170,9 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti } finally { context.removeRunningJob(executable); } + + // trigger the next step asap + fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/f37fbce1/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java index 844159d..efbc33e 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java @@ -156,6 +156,9 @@ public class TableMetadataManager { } public Map<String, TableDesc> getAllTablesMap(String prj) { + // avoid cyclic locks + ProjectInstance project = (prj == null) ? null : ProjectManager.getInstance(config).getProject(prj); + try (AutoLock lock = srcTableMapLock.lockForWrite()) { //TODO prj == null case is now only used by test case and CubeMetaIngester //should refactor these test case and tool ASAP and stop supporting null case @@ -168,7 +171,6 @@ public class TableMetadataManager { return globalTables; } - ProjectInstance project = ProjectManager.getInstance(config).getProject(prj); Set<String> prjTableNames = project.getTables(); Map<String, TableDesc> ret = new LinkedHashMap<>(); @@ -248,8 +250,10 @@ public class TableMetadataManager { * again */ public void resetProjectSpecificTableDesc(String prj) throws IOException { + // avoid cyclic locks + ProjectInstance project = ProjectManager.getInstance(config).getProject(prj); + try (AutoLock lock = srcTableMapLock.lockForWrite()) { - ProjectInstance project = ProjectManager.getInstance(config).getProject(prj); for (String tableName : project.getTables()) { String tableIdentity = getTableIdentity(tableName); String key = mapKey(tableIdentity, prj); @@ -338,7 +342,7 @@ public class TableMetadataManager { result.setUuid(UUID.randomUUID().toString()); result.setLastModified(0); result.init(t.getProject()); - srcExtMap.put(mapKey(t.getIdentity(), t.getProject()), result); + srcExtMap.putLocal(mapKey(t.getIdentity(), t.getProject()), result); } return result; }