http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java new file mode 100644 index 0000000..5c2ca17 --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.metadata.cachesync; + +import java.io.IOException; +import java.util.List; + +import org.apache.kylin.common.persistence.JsonSerializer; +import org.apache.kylin.common.persistence.ResourceStore; +import org.apache.kylin.common.persistence.RootPersistentEntity; +import org.apache.kylin.common.persistence.Serializer; +import org.apache.kylin.metadata.MetadataConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Preconditions; + +abstract public class CachedCrudAssist<T extends RootPersistentEntity> { + + private static final Logger logger = LoggerFactory.getLogger(CachedCrudAssist.class); + + final private ResourceStore store; + final private Class<T> entityType; + final private String resRootPath; + final private Serializer<T> serializer; + final private SingleValueCache<String, T> cache; + + private boolean checkCopyOnWrite; + + public CachedCrudAssist(ResourceStore store, String resourceRootPath, Class<T> entityType, + SingleValueCache<String, T> cache) { + this.store = store; + this.entityType = entityType; + this.resRootPath = resourceRootPath; + this.serializer = new JsonSerializer<T>(entityType); + this.cache = cache; + + this.checkCopyOnWrite = store.getConfig().isCheckCopyOnWrite(); + + Preconditions.checkArgument(resRootPath.startsWith("/")); + Preconditions.checkArgument(resRootPath.endsWith("/") == false); + } + + public void setCheckOnWrite(boolean check) { + this.checkCopyOnWrite = check; + } + + private String resourcePath(String resourceName) { + return resRootPath + "/" + resourceName + MetadataConstants.FILE_SURFIX; + } + + public void reloadAll() throws IOException { + logger.debug("Reloading " + entityType.getName() + " from " + store.getReadableResourcePath(resRootPath)); + + cache.clear(); + + List<String> paths = store.collectResourceRecursively(resRootPath, MetadataConstants.FILE_SURFIX); + for (String path : paths) { + reloadQuietlyAt(path); + } + + logger.debug("Loaded " + cache.size() + " " + entityType.getName() + "(s)"); + } + + public T reload(String resourceName) throws IOException { + return reloadAt(resourcePath(resourceName)); + } + + public T reloadQuietly(String resourceName) { + return reloadQuietlyAt(resourcePath(resourceName)); + } + + private T reloadQuietlyAt(String path) { + try { + return reloadAt(path); + } catch (Exception ex) { + logger.error("Error loading " + entityType.getName() + " at " + path, ex); + return null; + } + } + + public T reloadAt(String path) throws IOException { + T entity = store.getResource(path, entityType, serializer); + if (entity == null) { + logger.warn("No " + entityType.getName() + " found at " + path + ", returning null"); + return null; + } + + initEntityAfterReload(entity); + cache.putLocal(entity.resourceName(), entity); + return entity; + } + + abstract protected void initEntityAfterReload(T entity); + + public T save(T entity) throws IOException { + Preconditions.checkArgument(entity != null); + Preconditions.checkArgument(entityType.isInstance(entity)); + + String resName = entity.resourceName(); + if (checkCopyOnWrite) { + if (cache.get(resName) == entity) { + throw new IllegalStateException("Copy-on-write violation! The updating entity " + entity + + " is a shared object in " + entityType.getName() + " cache, which should not be."); + } + } + + store.putResource(resourcePath(resName), entity, serializer); + cache.put(resName, entity); + return entity; + } + + public void delete(T entity) throws IOException { + delete(entity.resourceName()); + } + + public void delete(String resName) throws IOException { + Preconditions.checkArgument(resName != null); + store.deleteResource(resourcePath(resName)); + cache.remove(resName); + } + +}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/core-metadata/src/main/java/org/apache/kylin/metadata/draft/DraftManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/draft/DraftManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/draft/DraftManager.java index e2cdfa5..103da10 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/draft/DraftManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/draft/DraftManager.java @@ -22,8 +22,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.NavigableSet; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.JsonSerializer; @@ -41,35 +39,13 @@ public class DraftManager { public static final Serializer<Draft> DRAFT_SERIALIZER = new JsonSerializer<Draft>(Draft.class); - private static final ConcurrentMap<KylinConfig, DraftManager> CACHE = new ConcurrentHashMap<KylinConfig, DraftManager>(); - public static DraftManager getInstance(KylinConfig config) { - DraftManager r = CACHE.get(config); - if (r != null) { - return r; - } - - synchronized (DraftManager.class) { - r = CACHE.get(config); - if (r != null) { - return r; - } - try { - r = new DraftManager(config); - CACHE.put(config, r); - if (CACHE.size() > 1) { - logger.warn("More than one singleton exist"); - } - - return r; - } catch (IOException e) { - throw new IllegalStateException("Failed to init DraftManager from " + config, e); - } - } + return config.getManager(DraftManager.class); } - public static void clearCache() { - CACHE.clear(); + // called by reflection + static DraftManager newInstance(KylinConfig config) throws IOException { + return new DraftManager(config); } // ============================================================================ http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java index 8fbb8b1..dc1ac44 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java @@ -22,12 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import javax.annotation.Nullable; - -import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.JsonSerializer; import org.apache.kylin.common.persistence.ResourceStore; @@ -44,8 +39,6 @@ import org.apache.kylin.metadata.project.ProjectManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Function; -import com.google.common.collect.Iterators; import com.google.common.collect.Lists; /** @@ -54,38 +47,12 @@ public class DataModelManager { private static final Logger logger = LoggerFactory.getLogger(DataModelManager.class); - // static cached instances - private static final ConcurrentMap<KylinConfig, DataModelManager> CACHE = new ConcurrentHashMap<KylinConfig, DataModelManager>(); - public static DataModelManager getInstance(KylinConfig config) { - DataModelManager r = CACHE.get(config); - if (r != null) { - return r; - } - - synchronized (DataModelManager.class) { - r = CACHE.get(config); - if (r != null) { - return r; - } - r = newInstance(config); - CACHE.put(config, r); - if (CACHE.size() > 1) { - logger.warn("More than one singleton exist, current keys: {}", StringUtils - .join(Iterators.transform(CACHE.keySet().iterator(), new Function<KylinConfig, String>() { - @Nullable - @Override - public String apply(@Nullable KylinConfig input) { - return String.valueOf(System.identityHashCode(input)); - } - }), ",")); - } - - return r; - } + return config.getManager(DataModelManager.class); } - private static DataModelManager newInstance(KylinConfig conf) { + // called by reflection + static DataModelManager newInstance(KylinConfig conf) { try { String cls = StringUtil.noBlank(conf.getDataModelManagerImpl(), DataModelManager.class.getName()); Class<? extends DataModelManager> clz = ClassUtil.forName(cls, DataModelManager.class); @@ -95,10 +62,6 @@ public class DataModelManager { } } - public static void clearCache() { - CACHE.clear(); - } - // ============================================================================ private KylinConfig config; @@ -148,10 +111,6 @@ public class DataModelManager { } private class DataModelSyncListener extends Broadcaster.Listener { - @Override - public void onClearAll(Broadcaster broadcaster) throws IOException { - clearCache(); - } @Override public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException { http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java index f25acc7..7b1f840 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java @@ -126,6 +126,11 @@ public class ProjectInstance extends RootPersistentEntity { public ProjectInstance() { } + @Override + public String resourceName() { + return this.name; + } + public String getDescription() { return description; } http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java index 11d0f62..4622f35 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java @@ -26,20 +26,17 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.JsonSerializer; import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.Serializer; +import org.apache.kylin.common.util.AutoReadWriteLock; +import org.apache.kylin.common.util.AutoReadWriteLock.AutoLock; import org.apache.kylin.metadata.TableMetadataManager; import org.apache.kylin.metadata.badquery.BadQueryHistoryManager; import org.apache.kylin.metadata.cachesync.Broadcaster; import org.apache.kylin.metadata.cachesync.Broadcaster.Event; +import org.apache.kylin.metadata.cachesync.CachedCrudAssist; import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.ExternalFilterDesc; @@ -56,36 +53,14 @@ import com.google.common.collect.Sets; public class ProjectManager { private static final Logger logger = LoggerFactory.getLogger(ProjectManager.class); - private static final ConcurrentMap<KylinConfig, ProjectManager> CACHE = new ConcurrentHashMap<KylinConfig, ProjectManager>(); - public static final Serializer<ProjectInstance> PROJECT_SERIALIZER = new JsonSerializer<ProjectInstance>( - ProjectInstance.class); public static ProjectManager getInstance(KylinConfig config) { - ProjectManager r = CACHE.get(config); - if (r != null) { - return r; - } - - synchronized (ProjectManager.class) { - r = CACHE.get(config); - if (r != null) { - return r; - } - try { - r = new ProjectManager(config); - CACHE.put(config, r); - if (CACHE.size() > 1) { - logger.warn("More than one singleton exist"); - } - return r; - } catch (IOException e) { - throw new IllegalStateException("Failed to init ProjectManager from " + config, e); - } - } + return config.getManager(ProjectManager.class); } - public static void clearCache() { - CACHE.clear(); + // called by reflection + static ProjectManager newInstance(KylinConfig config) throws IOException { + return new ProjectManager(config); } // ============================================================================ @@ -98,27 +73,31 @@ public class ProjectManager { // protects concurrent operations around the projectMap, to avoid for example // writing a project in the middle of reloading it (dirty read) - private ReadWriteLock prjMapLock = new ReentrantReadWriteLock(); + private AutoReadWriteLock prjMapLock = new AutoReadWriteLock(); + + private CachedCrudAssist<ProjectInstance> crud; private ProjectManager(KylinConfig config) throws IOException { logger.info("Initializing ProjectManager with metadata url " + config); this.config = config; this.projectMap = new CaseInsensitiveStringCache<ProjectInstance>(config, "project"); this.l2Cache = new ProjectL2Cache(this); + this.crud = new CachedCrudAssist<ProjectInstance>(getStore(), ResourceStore.PROJECT_RESOURCE_ROOT, + ProjectInstance.class, projectMap) { + @Override + protected void initEntityAfterReload(ProjectInstance prj) { + prj.init(); + } + }; - // touch lower level metadata before registering my listener - reloadAllProjects(); + crud.reloadAll(); + Broadcaster.getInstance(config).registerListener(new ProjectSyncListener(), "project"); } private class ProjectSyncListener extends Broadcaster.Listener { @Override - public void onClearAll(Broadcaster broadcaster) throws IOException { - clearCache(); - } - - @Override public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) throws IOException { String project = cacheKey; @@ -128,7 +107,7 @@ public class ProjectManager { return; } - reloadProjectLocal(project); + reloadProjectQuietly(project); broadcaster.notifyProjectSchemaUpdate(project); broadcaster.notifyProjectDataUpdate(project); } @@ -138,85 +117,39 @@ public class ProjectManager { l2Cache.clear(); } - private void reloadAllProjects() throws IOException { - prjMapLock.writeLock().lock(); - try { - ResourceStore store = getStore(); - List<String> paths = store.collectResourceRecursively(ResourceStore.PROJECT_RESOURCE_ROOT, ".json"); - - logger.debug("Loading Project from folder " - + store.getReadableResourcePath(ResourceStore.PROJECT_RESOURCE_ROOT)); - - for (String path : paths) { - reloadProjectLocalAt(path); - } - logger.debug("Loaded " + projectMap.size() + " Project(s)"); - } finally { - prjMapLock.writeLock().unlock(); - } - } - - public ProjectInstance reloadProjectLocal(String project) throws IOException { - prjMapLock.writeLock().lock(); - try { - return reloadProjectLocalAt(ProjectInstance.concatResourcePath(project)); - } finally { - prjMapLock.writeLock().unlock(); - } - } - - private ProjectInstance reloadProjectLocalAt(String path) throws IOException { - ProjectInstance projectInstance = getStore().getResource(path, ProjectInstance.class, PROJECT_SERIALIZER); - if (projectInstance == null) { - logger.warn("reload project at path:" + path + " not found, this:" + this.toString()); - return null; + public ProjectInstance reloadProjectQuietly(String project) throws IOException { + try (AutoLock lock = prjMapLock.lockForWrite()) { + ProjectInstance prj = crud.reloadQuietly(project); + clearL2Cache(); + return prj; } - - projectInstance.init(); - - projectMap.putLocal(projectInstance.getName(), projectInstance); - clearL2Cache(); - - return projectInstance; } public List<ProjectInstance> listAllProjects() { - prjMapLock.readLock().lock(); - try { + try (AutoLock lock = prjMapLock.lockForRead()) { return new ArrayList<ProjectInstance>(projectMap.values()); - } finally { - prjMapLock.readLock().unlock(); } } public ProjectInstance getProject(String projectName) { - prjMapLock.readLock().lock(); - try { - projectName = norm(projectName); + try (AutoLock lock = prjMapLock.lockForRead()) { return projectMap.get(projectName); - } finally { - prjMapLock.readLock().unlock(); } } public ProjectInstance getPrjByUuid(String uuid) { - prjMapLock.readLock().lock(); - try { - Collection<ProjectInstance> copy = new ArrayList<ProjectInstance>(projectMap.values()); - for (ProjectInstance prj : copy) { + try (AutoLock lock = prjMapLock.lockForRead()) { + for (ProjectInstance prj : projectMap.values()) { if (uuid.equals(prj.getUuid())) return prj; } return null; - } finally { - prjMapLock.readLock().unlock(); } } public ProjectInstance createProject(String projectName, String owner, String description, LinkedHashMap<String, String> overrideProps) throws IOException { - prjMapLock.writeLock().lock(); - try { + try (AutoLock lock = prjMapLock.lockForWrite()) { logger.info("Creating project " + projectName); ProjectInstance currentProject = getProject(projectName); @@ -225,18 +158,30 @@ public class ProjectManager { } else { throw new IllegalStateException("The project named " + projectName + "already exists"); } + checkOverrideProps(currentProject); - updateProject(currentProject); + return save(currentProject); + } + } - return currentProject; - } finally { - prjMapLock.writeLock().unlock(); + private void checkOverrideProps(ProjectInstance prj) throws IOException { + LinkedHashMap<String, String> overrideProps = prj.getOverrideKylinProps(); + + if (overrideProps != null) { + Iterator<Map.Entry<String, String>> iterator = overrideProps.entrySet().iterator(); + + while (iterator.hasNext()) { + Map.Entry<String, String> entry = iterator.next(); + + if (StringUtils.isAnyBlank(entry.getKey(), entry.getValue())) { + throw new IllegalStateException("Property key/value must not be blank"); + } + } } } public ProjectInstance dropProject(String projectName) throws IOException { - prjMapLock.writeLock().lock(); - try { + try (AutoLock lock = prjMapLock.lockForWrite()) { if (projectName == null) throw new IllegalArgumentException("Project name not given"); @@ -253,45 +198,18 @@ public class ProjectManager { logger.info("Dropping project '" + projectInstance.getName() + "'"); - removeProject(projectInstance); + crud.delete(projectInstance); BadQueryHistoryManager.getInstance(config).removeBadQueryHistory(projectName); + clearL2Cache(); return projectInstance; - } finally { - prjMapLock.writeLock().unlock(); - } - } - - // rename project - public ProjectInstance renameProject(ProjectInstance project, String newName, String newDesc, - LinkedHashMap<String, String> overrideProps) throws IOException { - prjMapLock.writeLock().lock(); - try { - Preconditions.checkArgument(!project.getName().equals(newName)); - ProjectInstance newProject = this.createProject(newName, project.getOwner(), newDesc, overrideProps); - - newProject.setUuid(project.getUuid()); - newProject.setCreateTimeUTC(project.getCreateTimeUTC()); - newProject.recordUpdateTime(System.currentTimeMillis()); - newProject.setRealizationEntries(project.getRealizationEntries()); - newProject.setTables(project.getTables()); - newProject.setModels(project.getModels()); - newProject.setExtFilters(project.getExtFilters()); - - removeProject(project); - updateProject(newProject); - - return newProject; - } finally { - prjMapLock.writeLock().unlock(); } } // update project itself public ProjectInstance updateProject(ProjectInstance project, String newName, String newDesc, LinkedHashMap<String, String> overrideProps) throws IOException { - prjMapLock.writeLock().lock(); - try { + try (AutoLock lock = prjMapLock.lockForWrite()) { Preconditions.checkArgument(project.getName().equals(newName)); project.setName(newName); project.setDescription(newDesc); @@ -300,48 +218,19 @@ public class ProjectManager { if (project.getUuid() == null) project.updateRandomUuid(); - updateProject(project); - - return project; - } finally { - prjMapLock.writeLock().unlock(); + return save(project); } } - private void updateProject(ProjectInstance prj) throws IOException { - LinkedHashMap<String, String> overrideProps = prj.getOverrideKylinProps(); - - if (overrideProps != null) { - Iterator<Map.Entry<String, String>> iterator = overrideProps.entrySet().iterator(); - - while (iterator.hasNext()) { - Map.Entry<String, String> entry = iterator.next(); - - if (StringUtils.isAnyBlank(entry.getKey(), entry.getValue())) { - throw new IllegalStateException("Property key/value must not be blank"); - } - } + public void removeProjectLocal(String proj) { + try (AutoLock lock = prjMapLock.lockForWrite()) { + projectMap.removeLocal(proj); + clearL2Cache(); } - - getStore().putResource(prj.getResourcePath(), prj, PROJECT_SERIALIZER); - projectMap.put(norm(prj.getName()), prj); // triggers update broadcast - clearL2Cache(); - } - - private void removeProject(ProjectInstance proj) throws IOException { - getStore().deleteResource(proj.getResourcePath()); - projectMap.remove(norm(proj.getName())); - clearL2Cache(); - } - - private void removeProjectLocal(String proj) { - projectMap.remove(norm(proj)); - clearL2Cache(); } public ProjectInstance addModelToProject(String modelName, String newProjectName) throws IOException { - prjMapLock.writeLock().lock(); - try { + try (AutoLock lock = prjMapLock.lockForWrite()) { removeModelFromProjects(modelName); ProjectInstance prj = getProject(newProjectName); @@ -349,71 +238,57 @@ public class ProjectManager { throw new IllegalArgumentException("Project " + newProjectName + " does not exist."); } prj.addModel(modelName); - updateProject(prj); - - return prj; - } finally { - prjMapLock.writeLock().unlock(); + + return save(prj); } } public void removeModelFromProjects(String modelName) throws IOException { - prjMapLock.writeLock().lock(); - try { + try (AutoLock lock = prjMapLock.lockForWrite()) { for (ProjectInstance projectInstance : findProjectsByModel(modelName)) { projectInstance.removeModel(modelName); - updateProject(projectInstance); + save(projectInstance); } - } finally { - prjMapLock.writeLock().unlock(); } } public ProjectInstance moveRealizationToProject(RealizationType type, String realizationName, String newProjectName, String owner) throws IOException { - prjMapLock.writeLock().lock(); - try { + try (AutoLock lock = prjMapLock.lockForWrite()) { removeRealizationsFromProjects(type, realizationName); return addRealizationToProject(type, realizationName, newProjectName, owner); - } finally { - prjMapLock.writeLock().unlock(); } } private ProjectInstance addRealizationToProject(RealizationType type, String realizationName, String project, String user) throws IOException { - String newProjectName = norm(project); - if (StringUtils.isEmpty(newProjectName)) { + if (StringUtils.isEmpty(project)) { throw new IllegalArgumentException("Project name should not be empty."); } - ProjectInstance newProject = getProject(newProjectName); + ProjectInstance newProject = getProject(project); if (newProject == null) { - newProject = this.createProject(newProjectName, user, + newProject = this.createProject(project, user, "This is a project automatically added when adding realization " + realizationName + "(" + type + ")", null); } newProject.addRealizationEntry(type, realizationName); - updateProject(newProject); + save(newProject); return newProject; } public void removeRealizationsFromProjects(RealizationType type, String realizationName) throws IOException { - prjMapLock.writeLock().lock(); - try { + try (AutoLock lock = prjMapLock.lockForWrite()) { for (ProjectInstance projectInstance : findProjects(type, realizationName)) { projectInstance.removeRealization(type, realizationName); - updateProject(projectInstance); + save(projectInstance); } - } finally { - prjMapLock.writeLock().unlock(); } } public ProjectInstance addTableDescToProject(String[] tableIdentities, String projectName) throws IOException { - prjMapLock.writeLock().lock(); - try { + try (AutoLock lock = prjMapLock.lockForWrite()) { TableMetadataManager metaMgr = getTableManager(); ProjectInstance projectInstance = getProject(projectName); for (String tableId : tableIdentities) { @@ -424,16 +299,12 @@ public class ProjectManager { projectInstance.addTable(table.getIdentity()); } - updateProject(projectInstance); - return projectInstance; - } finally { - prjMapLock.writeLock().unlock(); + return save(projectInstance); } } public void removeTableDescFromProject(String tableIdentities, String projectName) throws IOException { - prjMapLock.writeLock().lock(); - try { + try (AutoLock lock = prjMapLock.lockForWrite()) { TableMetadataManager metaMgr = getTableManager(); ProjectInstance projectInstance = getProject(projectName); TableDesc table = metaMgr.getTableDesc(tableIdentities, projectName); @@ -442,15 +313,12 @@ public class ProjectManager { } projectInstance.removeTable(table.getIdentity()); - updateProject(projectInstance); - } finally { - prjMapLock.writeLock().unlock(); + save(projectInstance); } } public ProjectInstance addExtFilterToProject(String[] filters, String projectName) throws IOException { - prjMapLock.writeLock().lock(); - try { + try (AutoLock lock = prjMapLock.lockForWrite()) { TableMetadataManager metaMgr = getTableManager(); ProjectInstance projectInstance = getProject(projectName); for (String filterName : filters) { @@ -462,16 +330,12 @@ public class ProjectManager { projectInstance.addExtFilter(filterName); } - updateProject(projectInstance); - return projectInstance; - } finally { - prjMapLock.writeLock().unlock(); + return save(projectInstance); } } public void removeExtFilterFromProject(String filterName, String projectName) throws IOException { - prjMapLock.writeLock().lock(); - try { + try (AutoLock lock = prjMapLock.lockForWrite()) { TableMetadataManager metaMgr = getTableManager(); ProjectInstance projectInstance = getProject(projectName); ExternalFilterDesc filter = metaMgr.getExtFilterDesc(filterName); @@ -480,28 +344,28 @@ public class ProjectManager { } projectInstance.removeExtFilter(filterName); - updateProject(projectInstance); - } finally { - prjMapLock.writeLock().unlock(); + save(projectInstance); } } + + private ProjectInstance save(ProjectInstance prj) throws IOException { + crud.save(prj); + clearL2Cache(); + return prj; + } public ProjectInstance getProjectOfModel(String model) { - prjMapLock.readLock().lock(); - try { + try (AutoLock lock = prjMapLock.lockForRead()) { for (ProjectInstance prj : projectMap.values()) { if (prj.getModels().contains(model)) return prj; } throw new IllegalStateException("No project found for model " + model); - } finally { - prjMapLock.readLock().unlock(); } } public List<ProjectInstance> findProjects(RealizationType type, String realizationName) { - prjMapLock.readLock().lock(); - try { + try (AutoLock lock = prjMapLock.lockForWrite()) { List<ProjectInstance> result = Lists.newArrayList(); for (ProjectInstance prj : projectMap.values()) { for (RealizationEntry entry : prj.getRealizationEntries()) { @@ -512,14 +376,11 @@ public class ProjectManager { } } return result; - } finally { - prjMapLock.readLock().unlock(); } } public List<ProjectInstance> findProjectsByModel(String modelName) { - prjMapLock.readLock().lock(); - try { + try (AutoLock lock = prjMapLock.lockForWrite()) { List<ProjectInstance> projects = new ArrayList<ProjectInstance>(); for (ProjectInstance projectInstance : projectMap.values()) { if (projectInstance.containsModel(modelName)) { @@ -527,14 +388,11 @@ public class ProjectManager { } } return projects; - } finally { - prjMapLock.readLock().unlock(); } } public List<ProjectInstance> findProjectsByTable(String tableIdentity) { - prjMapLock.readLock().lock(); - try { + try (AutoLock lock = prjMapLock.lockForWrite()) { List<ProjectInstance> projects = new ArrayList<ProjectInstance>(); for (ProjectInstance projectInstance : projectMap.values()) { if (projectInstance.containsTable(tableIdentity)) { @@ -542,8 +400,6 @@ public class ProjectManager { } } return projects; - } finally { - prjMapLock.readLock().unlock(); } } @@ -552,11 +408,11 @@ public class ProjectManager { } public List<TableDesc> listDefinedTables(String project) { - return l2Cache.listDefinedTables(norm(project)); + return l2Cache.listDefinedTables(project); } private Collection<TableDesc> listExposedTablesByRealizations(String project) { - return l2Cache.listExposedTables(norm(project)); + return l2Cache.listExposedTables(project); } public Collection<TableDesc> listExposedTables(String project, boolean exposeMore) { @@ -568,7 +424,7 @@ public class ProjectManager { } public List<ColumnDesc> listExposedColumns(String project, TableDesc tableDesc, boolean exposeMore) { - Set<ColumnDesc> exposedColumns = l2Cache.listExposedColumns(norm(project), tableDesc.getIdentity()); + Set<ColumnDesc> exposedColumns = l2Cache.listExposedColumns(project, tableDesc.getIdentity()); if (exposeMore) { Set<ColumnDesc> dedup = Sets.newHashSet(tableDesc.getColumns()); @@ -580,19 +436,19 @@ public class ProjectManager { } public Set<IRealization> listAllRealizations(String project) { - return l2Cache.listAllRealizations(norm(project)); + return l2Cache.listAllRealizations(project); } public Set<IRealization> getRealizationsByTable(String project, String tableName) { - return l2Cache.getRealizationsByTable(norm(project), tableName.toUpperCase()); + return l2Cache.getRealizationsByTable(project, tableName.toUpperCase()); } public List<MeasureDesc> listEffectiveRewriteMeasures(String project, String factTable) { - return l2Cache.listEffectiveRewriteMeasures(norm(project), factTable.toUpperCase(), true); + return l2Cache.listEffectiveRewriteMeasures(project, factTable.toUpperCase(), true); } public List<MeasureDesc> listEffectiveMeasures(String project, String factTable) { - return l2Cache.listEffectiveRewriteMeasures(norm(project), factTable.toUpperCase(), false); + return l2Cache.listEffectiveRewriteMeasures(project, factTable.toUpperCase(), false); } KylinConfig getConfig() { @@ -607,8 +463,4 @@ public class ProjectManager { return TableMetadataManager.getInstance(config); } - private String norm(String project) { - return project; - } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java index 2d1a4a5..279fe44 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java @@ -22,12 +22,9 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.ClassUtil; -import org.apache.kylin.metadata.cachesync.Broadcaster; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,34 +35,14 @@ import com.google.common.collect.Maps; public class RealizationRegistry { private static final Logger logger = LoggerFactory.getLogger(RealizationRegistry.class); - private static final ConcurrentMap<KylinConfig, RealizationRegistry> CACHE = new ConcurrentHashMap<KylinConfig, RealizationRegistry>(); public static RealizationRegistry getInstance(KylinConfig config) { - RealizationRegistry r = CACHE.get(config); - if (r != null) { - return r; - } - - synchronized (RealizationRegistry.class) { - r = CACHE.get(config); - if (r != null) { - return r; - } - try { - r = new RealizationRegistry(config); - CACHE.put(config, r); - if (CACHE.size() > 1) { - logger.warn("More than one singleton exist"); - } - return r; - } catch (IOException e) { - throw new IllegalStateException("Failed to init CubeManager from " + config, e); - } - } + return config.getManager(RealizationRegistry.class); } - public static void clearCache() { - CACHE.clear(); + // called by reflection + static RealizationRegistry newInstance(KylinConfig config) throws IOException { + return new RealizationRegistry(config); } // ============================================================================ @@ -77,13 +54,6 @@ public class RealizationRegistry { logger.info("Initializing RealizationRegistry with metadata url " + config); this.config = config; init(); - - Broadcaster.getInstance(config).registerListener(new Broadcaster.Listener() { - @Override - public void onClearAll(Broadcaster broadcaster) throws IOException { - clearCache(); - } - }, ""); } private void init() { http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java index 48febeb..b5d7e37 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java @@ -21,8 +21,6 @@ package org.apache.kylin.metadata.streaming; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; @@ -42,20 +40,24 @@ public class StreamingManager { private static final Logger logger = LoggerFactory.getLogger(StreamingManager.class); - // static cached instances - private static final ConcurrentMap<KylinConfig, StreamingManager> CACHE = new ConcurrentHashMap<KylinConfig, StreamingManager>(); - public static final Serializer<StreamingConfig> STREAMING_SERIALIZER = new JsonSerializer<StreamingConfig>(StreamingConfig.class); + public static StreamingManager getInstance(KylinConfig config) { + return config.getManager(StreamingManager.class); + } + + // called by reflection + static StreamingManager newInstance(KylinConfig config) throws IOException { + return new StreamingManager(config); + } + + // ============================================================================ + private KylinConfig config; // name ==> StreamingConfig private CaseInsensitiveStringCache<StreamingConfig> streamingMap; - public static void clearCache() { - CACHE.clear(); - } - private StreamingManager(KylinConfig config) throws IOException { this.config = config; this.streamingMap = new CaseInsensitiveStringCache<StreamingConfig>(config, "streaming"); @@ -66,10 +68,6 @@ public class StreamingManager { } private class StreamingSyncListener extends Broadcaster.Listener { - @Override - public void onClearAll(Broadcaster broadcaster) throws IOException { - clearCache(); - } @Override public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) throws IOException { @@ -84,42 +82,6 @@ public class StreamingManager { return ResourceStore.getStore(this.config); } - public static StreamingManager getInstance(KylinConfig config) { - StreamingManager r = CACHE.get(config); - if (r != null) { - return r; - } - - synchronized (StreamingManager.class) { - r = CACHE.get(config); - if (r != null) { - return r; - } - try { - r = new StreamingManager(config); - CACHE.put(config, r); - if (CACHE.size() > 1) { - logger.warn("More than one singleton exist"); - } - return r; - } catch (IOException e) { - throw new IllegalStateException("Failed to init StreamingManager from " + config, e); - } - } - } - - private static String formatStreamingConfigPath(String name) { - return ResourceStore.STREAMING_RESOURCE_ROOT + "/" + name + ".json"; - } - - private static String formatStreamingOutputPath(String streaming, int partition) { - return ResourceStore.STREAMING_OUTPUT_RESOURCE_ROOT + "/" + streaming + "_" + partition + ".json"; - } - - private static String formatStreamingOutputPath(String streaming, List<Integer> partitions) { - return ResourceStore.STREAMING_OUTPUT_RESOURCE_ROOT + "/" + streaming + "_" + StringUtils.join(partitions, "_") + ".json"; - } - public StreamingConfig getStreamingConfig(String name) { return streamingMap.get(name); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java index 095b46f..b910ffe 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java @@ -175,7 +175,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { protected abstract String getGTStorage(); protected Cuboid findCuboid(CubeInstance cubeInstance, Set<TblColRef> dimensionsD, Set<FunctionDesc> metrics) { - return Cuboid.identifyCuboid(cubeInstance, dimensionsD, metrics); + return Cuboid.findCuboid(cubeInstance.getCuboidScheduler(), dimensionsD, metrics); } protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java index 2b1c2cc..37f4aff 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java @@ -20,8 +20,6 @@ package org.apache.kylin.storage.hybrid; import java.io.IOException; import java.util.Collection; import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; @@ -49,35 +47,13 @@ public class HybridManager implements IRealizationProvider { private static final Logger logger = LoggerFactory.getLogger(HybridManager.class); - // static cached instances - private static final ConcurrentMap<KylinConfig, HybridManager> CACHE = new ConcurrentHashMap<KylinConfig, HybridManager>(); - public static HybridManager getInstance(KylinConfig config) { - HybridManager r = CACHE.get(config); - if (r != null) { - return r; - } - - synchronized (HybridManager.class) { - r = CACHE.get(config); - if (r != null) { - return r; - } - try { - r = new HybridManager(config); - CACHE.put(config, r); - if (CACHE.size() > 1) { - logger.warn("More than one singleton exist"); - } - return r; - } catch (IOException e) { - throw new IllegalStateException("Failed to init Hybrid Manager from " + config, e); - } - } + return config.getManager(HybridManager.class); } - public static void clearCache() { - CACHE.clear(); + // called by reflection + static HybridManager newInstance(KylinConfig config) throws IOException { + return new HybridManager(config); } // ============================================================================ @@ -99,11 +75,6 @@ public class HybridManager implements IRealizationProvider { private class HybridSyncListener extends Broadcaster.Listener { @Override - public void onClearAll(Broadcaster broadcaster) throws IOException { - clearCache(); - } - - @Override public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException { for (IRealization real : ProjectManager.getInstance(config).listAllRealizations(project)) { if (real instanceof HybridInstance) { http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java index 4d92f8e..5ddb024 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java @@ -41,9 +41,7 @@ import org.apache.kylin.dict.DictionaryManager; import org.apache.kylin.dict.IterableDictionaryValueEnumerator; import org.apache.kylin.metadata.datatype.DataType; import org.apache.kylin.metadata.model.SegmentRange.TSRange; -import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.source.IReadableTable.TableSignature; import org.junit.After; import org.junit.Before; @@ -89,11 +87,7 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase { createTestMetadata(); logger.info("The metadataUrl is : " + getTestConfig()); - - DataModelManager.clearCache(); - CubeManager.clearCache(); - ProjectManager.clearCache(); - DictionaryManager.clearCache(); + getTestConfig().clearManagers(); // hack for distributed cache // CubeManager.removeInstance(KylinConfig.createInstanceFromUri("../job/meta"));//to http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java index 4bb8e82..c892ff4 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java @@ -156,7 +156,8 @@ public class ProjectController extends BasicController { if (projectDesc.getName().equals(currentProject.getName())) { updatedProj = projectService.updateProject(projectDesc, currentProject); } else { - updatedProj = projectService.renameProject(projectDesc, currentProject); + throw new IllegalStateException("Rename project is not supported yet, from " + formerProjectName + + " to " + projectDesc.getName()); } } catch (Exception e) { logger.error("Failed to deal with the request.", e); http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java b/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java index 146a0a5..2ab4c24 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java +++ b/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java @@ -249,6 +249,10 @@ public class Message { return "Cannot modify non-empty project"; } + public String getPROJECT_RENAME() { + return "Project renaming is not allowed."; + } + // Table public String getHIVE_TABLE_NOT_FOUND() { return "Cannot find Hive table '%s'."; http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java index d9627b9..417fb10 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java @@ -27,7 +27,6 @@ import java.util.List; import javax.annotation.Nullable; import org.apache.directory.api.util.Strings; -import org.apache.kylin.metadata.draft.Draft; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.realization.RealizationType; import org.apache.kylin.rest.constant.Constant; @@ -95,9 +94,6 @@ public class ProjectService extends BasicService { @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#currentProject, 'ADMINISTRATION')") public ProjectInstance updateProject(ProjectInstance newProject, ProjectInstance currentProject) throws IOException { - if (!newProject.getName().equals(currentProject.getName())) { - return renameProject(newProject, currentProject); - } String newProjectName = newProject.getName(); String newDescription = newProject.getDescription(); @@ -110,27 +106,6 @@ public class ProjectService extends BasicService { return updatedProject; } - @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#currentProject, 'ADMINISTRATION')") - public ProjectInstance renameProject(ProjectInstance newProject, ProjectInstance currentProject) - throws IOException { - String newProjectName = newProject.getName(); - String newDescription = newProject.getDescription(); - LinkedHashMap<String, String> overrideProps = newProject.getOverrideKylinProps(); - - // rename project but keep UUID, acl keeps the same - ProjectInstance renamedProject = getProjectManager().renameProject(currentProject, newProjectName, - newDescription, overrideProps); - - // rebind draft and project - for (Draft draft : getDraftManager().list(currentProject.getName())) { - draft.setProject(newProjectName); - getDraftManager().save(draft); - } - - logger.debug("Project rename."); - return renamedProject; - } - @PostFilter(Constant.ACCESS_POST_FILTER_READ) public List<ProjectInstance> listProjects(final Integer limit, final Integer offset) { List<ProjectInstance> projects = listAllProjects(limit, offset); http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/server-base/src/test/java/org/apache/kylin/rest/service/DiagnosisServiceTest.java ---------------------------------------------------------------------- diff --git a/server-base/src/test/java/org/apache/kylin/rest/service/DiagnosisServiceTest.java b/server-base/src/test/java/org/apache/kylin/rest/service/DiagnosisServiceTest.java index e51fd31..b200980 100644 --- a/server-base/src/test/java/org/apache/kylin/rest/service/DiagnosisServiceTest.java +++ b/server-base/src/test/java/org/apache/kylin/rest/service/DiagnosisServiceTest.java @@ -26,7 +26,6 @@ import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.metadata.badquery.BadQueryEntry; import org.apache.kylin.metadata.badquery.BadQueryHistory; import org.apache.kylin.metadata.badquery.BadQueryHistoryManager; -import org.apache.kylin.metadata.model.DataModelManager; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -37,7 +36,6 @@ public class DiagnosisServiceTest extends LocalFileMetadataTestCase { @Before public void setUp() throws Exception { this.createTestMetadata(); - DataModelManager.clearCache(); } @After http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/server/src/test/java/org/apache/kylin/rest/controller/ProjectControllerTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/controller/ProjectControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/ProjectControllerTest.java index f805095..8d08d61 100644 --- a/server/src/test/java/org/apache/kylin/rest/controller/ProjectControllerTest.java +++ b/server/src/test/java/org/apache/kylin/rest/controller/ProjectControllerTest.java @@ -31,9 +31,9 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import com.fasterxml.jackson.databind.ObjectMapper; -import org.springframework.beans.factory.annotation.Qualifier; /** */ @@ -76,25 +76,15 @@ public class ProjectControllerTest extends ServiceTestBase { Assert.assertEquals(ret.getOwner(), "ADMIN"); Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1); - //test update project - ProjectInstance newProject = new ProjectInstance(); - newProject.setName("new_project_2"); - projectController.updateProject(getProjectRequest(newProject, "new_project")); - - Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1); - Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project"), null); - Assert.assertNotEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project_2"), null); - //test update project description only ProjectInstance newProject2 = new ProjectInstance(); - newProject2.setName("new_project_2"); + newProject2.setName("new_project"); newProject2.setDescription("hello world"); - projectController.updateProject(getProjectRequest(newProject2, "new_project_2")); + projectController.updateProject(getProjectRequest(newProject2, "new_project")); Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1); - Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project"), null); - Assert.assertNotEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project_2"), null); - Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project_2").getDescription(), "hello world"); + Assert.assertNotEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project"), null); + Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project").getDescription(), "hello world"); } @Test(expected = InternalErrorException.class) http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java ---------------------------------------------------------------------- diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java index 50295c3..5e451b4 100644 --- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java +++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java @@ -21,8 +21,6 @@ package org.apache.kylin.source.kafka; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; @@ -43,20 +41,24 @@ public class KafkaConfigManager { private static final Logger logger = LoggerFactory.getLogger(KafkaConfigManager.class); - // static cached instances - private static final ConcurrentMap<KylinConfig, KafkaConfigManager> CACHE = new ConcurrentHashMap<KylinConfig, KafkaConfigManager>(); - - private KylinConfig config; - - // name ==> StreamingConfig - private CaseInsensitiveStringCache<KafkaConfig> kafkaMap; - public static final Serializer<KafkaConfig> KAFKA_SERIALIZER = new JsonSerializer<KafkaConfig>(KafkaConfig.class); + + public static KafkaConfigManager getInstance(KylinConfig config) { + return config.getManager(KafkaConfigManager.class); + } - public static void clearCache() { - CACHE.clear(); + // called by reflection + static KafkaConfigManager newInstance(KylinConfig config) throws IOException { + return new KafkaConfigManager(config); } + // ============================================================================ + + private KylinConfig config; + + // name ==> StreamingConfig + private CaseInsensitiveStringCache<KafkaConfig> kafkaMap; + private KafkaConfigManager(KylinConfig config) throws IOException { this.config = config; this.kafkaMap = new CaseInsensitiveStringCache<KafkaConfig>(config, "kafka"); @@ -67,10 +69,6 @@ public class KafkaConfigManager { } private class KafkaSyncListener extends Broadcaster.Listener { - @Override - public void onClearAll(Broadcaster broadcaster) throws IOException { - clearCache(); - } @Override public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) throws IOException { @@ -85,30 +83,6 @@ public class KafkaConfigManager { return ResourceStore.getStore(this.config); } - public static KafkaConfigManager getInstance(KylinConfig config) { - KafkaConfigManager r = CACHE.get(config); - if (r != null) { - return r; - } - - synchronized (KafkaConfigManager.class) { - r = CACHE.get(config); - if (r != null) { - return r; - } - try { - r = new KafkaConfigManager(config); - CACHE.put(config, r); - if (CACHE.size() > 1) { - logger.warn("More than one singleton exist"); - } - return r; - } catch (IOException e) { - throw new IllegalStateException("Failed to init KafkaConfigManager from " + config, e); - } - } - } - public List<KafkaConfig> listAllKafkaConfigs() { return new ArrayList(kafkaMap.values()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java index 7f55895..092023e 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java @@ -42,9 +42,9 @@ import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.model.IEngineAware; import org.apache.kylin.metadata.model.IStorageAware; -import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.model.Segments; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.project.ProjectManager; @@ -210,16 +210,11 @@ public class ExtendCubeToHybridCLI { } private void verify() { - CubeDescManager.clearCache(); + kylinConfig.clearManagers(); + CubeDescManager.getInstance(kylinConfig); - - CubeManager.clearCache(); CubeManager.getInstance(kylinConfig); - - ProjectManager.clearCache(); ProjectManager.getInstance(kylinConfig); - - HybridManager.clearCache(); HybridManager.getInstance(kylinConfig); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java index befe8a1..61f8386 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java @@ -37,7 +37,6 @@ import org.apache.kylin.measure.MeasureIngester; import org.apache.kylin.measure.MeasureType; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.MeasureDesc; -import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.model.TblColRef; import org.junit.After; import org.junit.Before; @@ -65,7 +64,6 @@ public class MeasureTypeOnlyAggrInBaseTest extends LocalFileMetadataTestCase { @Before public void setUp() throws Exception { this.createTestMetadata(); - DataModelManager.clearCache(); cube = getTestKylinCubeWithSeller(); cubeDesc = cube.getDescriptor(); @@ -91,10 +89,10 @@ public class MeasureTypeOnlyAggrInBaseTest extends LocalFileMetadataTestCase { @Test public void testIdentifyCuboidV2() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException, NoSuchFieldException { CubeDesc cubeDesc = cube.getDescriptor(); - Cuboid ret = Cuboid.identifyCuboid(cube, Sets.<TblColRef> newHashSet(), Lists.<FunctionDesc> newArrayList()); + Cuboid ret = Cuboid.findCuboid(cube.getCuboidScheduler(), Sets.<TblColRef> newHashSet(), Lists.<FunctionDesc> newArrayList()); long baseCuboidId = cubeDesc.getRowkey().getFullMask(); assertNotEquals(baseCuboidId, ret.getId()); - ret = Cuboid.identifyCuboid(cube, dimensions, metrics); + ret = Cuboid.findCuboid(cube.getCuboidScheduler(), dimensions, metrics); assertEquals(baseCuboidId, ret.getId()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java index 5bd5499..199179b 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java @@ -32,7 +32,6 @@ import org.apache.kylin.cube.model.HBaseColumnDesc; import org.apache.kylin.measure.BufferedMeasureCodec; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.MeasureDesc; -import org.apache.kylin.metadata.model.DataModelManager; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -42,7 +41,6 @@ public class RowValueDecoderTest extends LocalFileMetadataTestCase { @Before public void setUp() throws Exception { this.createTestMetadata(); - DataModelManager.clearCache(); } @After http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java index 7517c07..9c6cba6 100644 --- a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java +++ b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java @@ -42,9 +42,9 @@ import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.model.IEngineAware; import org.apache.kylin.metadata.model.IStorageAware; -import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.model.Segments; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.project.ProjectManager; @@ -206,16 +206,11 @@ public class ExtendCubeToHybridCLI { } private void verify() { - CubeDescManager.clearCache(); + kylinConfig.clearManagers(); + CubeDescManager.getInstance(kylinConfig); - - CubeManager.clearCache(); CubeManager.getInstance(kylinConfig); - - ProjectManager.clearCache(); ProjectManager.getInstance(kylinConfig); - - HybridManager.clearCache(); HybridManager.getInstance(kylinConfig); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java b/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java index 1471832..cda3fec 100644 --- a/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java +++ b/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java @@ -59,14 +59,12 @@ public class KylinLogExtractor extends AbstractInfoExtractor { private void beforeExtract() { // reload metadata before extract diagnosis info logger.info("Start to reload metadata from diagnosis."); + + config.clearManagers(); - CubeManager.clearCache(); CubeManager.getInstance(config); - CubeDescManager.clearCache(); CubeDescManager.getInstance(config); - DataModelManager.clearCache(); DataModelManager.getInstance(config); - ProjectManager.clearCache(); ProjectManager.getInstance(config); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d79870/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java ---------------------------------------------------------------------- diff --git a/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java b/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java index d974009..eca82be 100644 --- a/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java +++ b/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java @@ -22,10 +22,8 @@ import java.util.Collections; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.cube.CubeDescManager; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.project.RealizationEntry; @@ -66,9 +64,7 @@ public class CubeMetaIngesterTest extends LocalFileMetadataTestCase { Assert.assertTrue(project.getModels().contains("cloned_model")); Assert.assertTrue(project.getRealizationEntries().contains(RealizationEntry.create(RealizationType.CUBE, "cloned_cube"))); - DataModelManager.clearCache(); - CubeDescManager.clearCache(); - CubeManager.clearCache(); + getTestConfig().clearManagers(); CubeInstance instance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("cloned_cube"); Assert.assertTrue(instance != null); } @@ -83,9 +79,7 @@ public class CubeMetaIngesterTest extends LocalFileMetadataTestCase { Assert.assertTrue(project.getModels().contains("benchmark_model")); Assert.assertTrue(project.getRealizationEntries().contains(RealizationEntry.create(RealizationType.CUBE, "benchmark_cube"))); - DataModelManager.clearCache(); - CubeDescManager.clearCache(); - CubeManager.clearCache(); + getTestConfig().clearManagers(); CubeInstance instance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("benchmark_cube"); Assert.assertTrue(instance != null); }
