http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f487fe/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index f39432b..efb9c59 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -47,6 +47,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.KylinConfigExt; import org.apache.kylin.common.KylinVersion; +import org.apache.kylin.common.persistence.JsonSerializer; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.RootPersistentEntity; import org.apache.kylin.common.util.Array; @@ -58,13 +59,13 @@ import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType; import org.apache.kylin.metadata.MetadataConstants; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.IEngineAware; import org.apache.kylin.metadata.model.IStorageAware; import org.apache.kylin.metadata.model.JoinDesc; import org.apache.kylin.metadata.model.JoinTableDesc; import org.apache.kylin.metadata.model.MeasureDesc; -import org.apache.kylin.metadata.model.DataModelManager; import org.apache.kylin.metadata.model.TableRef; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.project.ProjectInstance; @@ -90,6 +91,11 @@ import com.google.common.collect.Sets; public class CubeDesc extends RootPersistentEntity implements IEngineAware { private static final Logger logger = LoggerFactory.getLogger(CubeDesc.class); + // Use with care! Normally you should go to CubeDescManager and don't need this. + public static JsonSerializer<CubeDesc> newSerializerForLowLevelAccess() { + return new JsonSerializer<>(CubeDesc.class); + } + public static class CannotFilterExtendedColumnException extends RuntimeException { public CannotFilterExtendedColumnException(TblColRef tblColRef) { super(tblColRef == null ? "null" : tblColRef.getCanonicalName()); @@ -122,6 +128,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { } } + + // ============================================================================ private KylinConfigExt config; private DataModelDesc model; @@ -183,6 +191,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { @JsonInclude(JsonInclude.Include.NON_NULL) private int parentForward = 3; + // Error messages during resolving json metadata + private List<String> errors = new ArrayList<String>(); + private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<>(); private LinkedHashSet<ColumnDesc> allColumnDescs = new LinkedHashSet<>(); private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<>(); @@ -194,6 +205,11 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { transient volatile private CuboidScheduler cuboidScheduler = null; + @Override + public String resourceName() { + return name; + } + public boolean isEnableSharding() { //in the future may extend to other storage that is shard-able return storageType != IStorageAware.ID_HBASE && storageType != IStorageAware.ID_HYBRID; @@ -204,11 +220,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { } /** - * Error messages during resolving json metadata - */ - private List<String> errors = new ArrayList<String>(); - - /** * @return all columns this cube can support, including derived */ public Set<TblColRef> listAllColumns() { @@ -311,7 +322,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { } public String getResourcePath() { - return concatResourcePath(name); + return concatResourcePath(resourceName()); } public static String concatResourcePath(String descName) { @@ -1116,6 +1127,10 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { public void setAutoMergeTimeRanges(long[] autoMergeTimeRanges) { this.autoMergeTimeRanges = autoMergeTimeRanges; } + + public boolean isBroken() { + return !errors.isEmpty(); + } public void addError(String message) { this.errors.add(message);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f487fe/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java index af275a5..844159d 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java @@ -32,10 +32,13 @@ import org.apache.kylin.common.persistence.JsonSerializer; import org.apache.kylin.common.persistence.RawResource; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.Serializer; +import org.apache.kylin.common.util.AutoReadWriteLock; +import org.apache.kylin.common.util.AutoReadWriteLock.AutoLock; import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.common.util.Pair; import org.apache.kylin.metadata.cachesync.Broadcaster; import org.apache.kylin.metadata.cachesync.Broadcaster.Event; +import org.apache.kylin.metadata.cachesync.CachedCrudAssist; import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache; import org.apache.kylin.metadata.model.ExternalFilterDesc; import org.apache.kylin.metadata.model.TableDesc; @@ -52,13 +55,11 @@ import com.google.common.collect.Maps; */ public class TableMetadataManager { + @SuppressWarnings("unused") private static final Logger logger = LoggerFactory.getLogger(TableMetadataManager.class); - public static final Serializer<TableDesc> TABLE_SERIALIZER = new JsonSerializer<TableDesc>(TableDesc.class); - public static final Serializer<TableExtDesc> TABLE_EXT_SERIALIZER = new JsonSerializer<TableExtDesc>( + private static final Serializer<TableExtDesc> TABLE_EXT_SERIALIZER = new JsonSerializer<TableExtDesc>( TableExtDesc.class); - public static final Serializer<ExternalFilterDesc> EXTERNAL_FILTER_DESC_SERIALIZER = new JsonSerializer<ExternalFilterDesc>( - ExternalFilterDesc.class); public static TableMetadataManager getInstance(KylinConfig config) { return config.getManager(TableMetadataManager.class); @@ -72,15 +73,28 @@ public class TableMetadataManager { // ============================================================================ private KylinConfig config; + // table name ==> SourceTable private CaseInsensitiveStringCache<TableDesc> srcTableMap; + private CachedCrudAssist<TableDesc> srcTableCrud; + private AutoReadWriteLock srcTableMapLock = new AutoReadWriteLock(); + // name => SourceTableExt - private CaseInsensitiveStringCache<TableExtDesc> srcTableExtMap; - // name => External Filter Desc + private CaseInsensitiveStringCache<TableExtDesc> srcExtMap; + private CachedCrudAssist<TableExtDesc> srcExtCrud; + private AutoReadWriteLock srcExtMapLock = new AutoReadWriteLock(); + + // name => ExternalFilterDesc private CaseInsensitiveStringCache<ExternalFilterDesc> extFilterMap; + private CachedCrudAssist<ExternalFilterDesc> extFilterCrud; + private AutoReadWriteLock extFilterMapLock = new AutoReadWriteLock(); + + private TableMetadataManager(KylinConfig cfg) throws IOException { + this.config = cfg; - private TableMetadataManager(KylinConfig config) throws IOException { - init(config); + initSrcTable(); + initSrcExt(); + initExtFilter(); } public KylinConfig getConfig() { @@ -91,42 +105,114 @@ public class TableMetadataManager { return ResourceStore.getStore(this.config); } - public List<TableDesc> listAllTables(String prj) { - return Lists.newArrayList(getAllTablesMap(prj).values()); + // ============================================================================ + // TableDesc methods + // ============================================================================ + + private void initSrcTable() throws IOException { + this.srcTableMap = new CaseInsensitiveStringCache<>(config, "table"); + this.srcTableCrud = new CachedCrudAssist<TableDesc>(getStore(), ResourceStore.TABLE_RESOURCE_ROOT, + TableDesc.class, srcTableMap) { + @Override + protected TableDesc initEntityAfterReload(TableDesc t, String resourceName) { + String prj = TableDesc.parseResourcePath(resourceName).getSecond(); + t.init(prj); + return t; + } + }; + srcTableCrud.reloadAll(); + Broadcaster.getInstance(config).registerListener(new SrcTableSyncListener(), "table"); } - public List<ExternalFilterDesc> listAllExternalFilters() { - return Lists.newArrayList(extFilterMap.values()); + private class SrcTableSyncListener extends Broadcaster.Listener { + @Override + public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) + throws IOException { + try (AutoLock lock = srcTableMapLock.lockForWrite()) { + if (event == Event.DROP) + srcTableMap.removeLocal(cacheKey); + else + srcTableCrud.reloadQuietly(cacheKey); + } + + Pair<String, String> pair = TableDesc.parseResourcePath(cacheKey); + String table = pair.getFirst(); + String prj = pair.getSecond(); + + if (prj == null) { + for (ProjectInstance p : ProjectManager.getInstance(config).findProjectsByTable(table)) { + broadcaster.notifyProjectSchemaUpdate(p.getName()); + } + } else { + broadcaster.notifyProjectSchemaUpdate(prj); + } + } + } + + public List<TableDesc> listAllTables(String prj) { + try (AutoLock lock = srcTableMapLock.lockForWrite()) { + return Lists.newArrayList(getAllTablesMap(prj).values()); + } } public Map<String, TableDesc> getAllTablesMap(String prj) { - //TODO prj == null case is now only used by test case and CubeMetaIngester - //should refactor these test case and tool ASAP and stop supporting null case - if (prj == null) { - Map<String, TableDesc> globalTables = new LinkedHashMap<>(); + try (AutoLock lock = srcTableMapLock.lockForWrite()) { + //TODO prj == null case is now only used by test case and CubeMetaIngester + //should refactor these test case and tool ASAP and stop supporting null case + if (prj == null) { + Map<String, TableDesc> globalTables = new LinkedHashMap<>(); - for (TableDesc t : srcTableMap.values()) { - globalTables.put(t.getIdentity(), t); + for (TableDesc t : srcTableMap.values()) { + globalTables.put(t.getIdentity(), t); + } + return globalTables; } - return globalTables; - } - - ProjectInstance project = ProjectManager.getInstance(config).getProject(prj); - Set<String> prjTableNames = project.getTables(); - - Map<String, TableDesc> ret = new LinkedHashMap<>(); - for (String tableName : prjTableNames) { - String tableIdentity = getTableIdentity(tableName); - ret.put(tableIdentity, getProjectSpecificTableDesc(tableIdentity, prj)); + + ProjectInstance project = ProjectManager.getInstance(config).getProject(prj); + Set<String> prjTableNames = project.getTables(); + + Map<String, TableDesc> ret = new LinkedHashMap<>(); + for (String tableName : prjTableNames) { + String tableIdentity = getTableIdentity(tableName); + ret.put(tableIdentity, getProjectSpecificTableDesc(tableIdentity, prj)); + } + return ret; } - return ret; } /** * Get TableDesc by name */ public TableDesc getTableDesc(String tableName, String prj) { - return getProjectSpecificTableDesc(getTableIdentity(tableName), prj); + try (AutoLock lock = srcTableMapLock.lockForWrite()) { + return getProjectSpecificTableDesc(getTableIdentity(tableName), prj); + } + } + + /** + * Make sure the returned table desc is project-specific. + * + * All locks on srcTableMapLock are WRITE LOCKS because of this method!! + */ + private TableDesc getProjectSpecificTableDesc(String fullTableName, String prj) { + String key = mapKey(fullTableName, prj); + TableDesc result = srcTableMap.get(key); + + if (result == null) { + try (AutoLock lock = srcTableMapLock.lockForWrite()) { + result = srcTableMap.get(mapKey(fullTableName, null)); + if (result != null) { + result = new TableDesc(result);// deep copy of global tabledesc + + result.setLastModified(0); + result.setProject(prj); + result.setBorrowedFromGlobal(true); + + srcTableMap.putLocal(key, result); + } + } + } + return result; } /** @@ -139,55 +225,87 @@ public class TableMetadataManager { return tableName.toUpperCase(); } + public void saveSourceTable(TableDesc srcTable, String prj) throws IOException { + try (AutoLock lock = srcTableMapLock.lockForWrite()) { + srcTable.init(prj); + srcTableCrud.save(srcTable); + } + } + + public void removeSourceTable(String tableIdentity, String prj) throws IOException { + try (AutoLock lock = srcTableMapLock.lockForWrite()) { + TableDesc t = getTableDesc(tableIdentity, prj); + if (t == null) + return; + + srcTableCrud.delete(t); + } + } + /** * the project-specific table desc will be expand by computed columns from the projects' models * when the projects' model list changed, project-specific table should be reset and get expanded * again */ public void resetProjectSpecificTableDesc(String prj) throws IOException { - ProjectInstance project = ProjectManager.getInstance(config).getProject(prj); - for (String tableName : project.getTables()) { - String tableIdentity = getTableIdentity(tableName); - String key = mapKey(tableIdentity, prj); - TableDesc originTableDesc = srcTableMap.get(key); - if (originTableDesc == null) { - continue; - } + try (AutoLock lock = srcTableMapLock.lockForWrite()) { + ProjectInstance project = ProjectManager.getInstance(config).getProject(prj); + for (String tableName : project.getTables()) { + String tableIdentity = getTableIdentity(tableName); + String key = mapKey(tableIdentity, prj); + TableDesc originTableDesc = srcTableMap.get(key); + if (originTableDesc == null) { + continue; + } - if (originTableDesc.isBorrowedFromGlobal()) { - srcTableMap.removeLocal(key);//delete it so that getProjectSpecificTableDesc will create again - } else { - String s = originTableDesc.getResourcePath(); - TableDesc tableDesc = reloadSourceTableAt(s); - srcTableMap.putLocal(key, tableDesc); + if (originTableDesc.isBorrowedFromGlobal()) { + srcTableMap.removeLocal(key);//delete it so that getProjectSpecificTableDesc will create again + } else { + srcTableCrud.reload(key); + } } } } - /** - * make sure the returned table desc is project-specific - */ - private TableDesc getProjectSpecificTableDesc(String fullTableName, String prj) { - String key = mapKey(fullTableName, prj); - TableDesc result = srcTableMap.get(key); + private String mapKey(String identity, String prj) { + return TableDesc.makeResourceName(identity, prj); + } - if (result == null) { - result = srcTableMap.get(mapKey(fullTableName, null)); - if (result != null) { - result = new TableDesc(result);// deep copy of global tabledesc + // ============================================================================ + // TableExtDesc methods + // ============================================================================ - result.setProject(prj); - result.setBorrowedFromGlobal(true); + private void initSrcExt() throws IOException { + this.srcExtMap = new CaseInsensitiveStringCache<>(config, "table_ext"); + this.srcExtCrud = new CachedCrudAssist<TableExtDesc>(getStore(), ResourceStore.TABLE_EXD_RESOURCE_ROOT, + TableExtDesc.class, srcExtMap) { + @Override + protected TableExtDesc initEntityAfterReload(TableExtDesc t, String resourceName) { + // convert old tableExt json to new one + if (t.getIdentity() == null) { + t = convertOldTableExtToNewer(resourceName); + } - srcTableMap.putLocal(key, result); + String prj = TableDesc.parseResourcePath(resourceName).getSecond(); + t.init(prj); + return t; } - } - return result; + }; + srcExtCrud.reloadAll(); + Broadcaster.getInstance(config).registerListener(new SrcTableExtSyncListener(), "table_ext"); } - public ExternalFilterDesc getExtFilterDesc(String filterTableName) { - ExternalFilterDesc result = extFilterMap.get(filterTableName); - return result; + private class SrcTableExtSyncListener extends Broadcaster.Listener { + @Override + public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) + throws IOException { + try (AutoLock lock = srcExtMapLock.lockForWrite()) { + if (event == Event.DROP) + srcExtMap.removeLocal(cacheKey); + else + srcExtCrud.reloadQuietly(cacheKey); + } + } } /** @@ -205,233 +323,89 @@ public class TableMetadataManager { } public TableExtDesc getTableExt(TableDesc t) { - TableExtDesc result = srcTableExtMap.get(mapKey(t.getIdentity(), t.getProject())); + try (AutoLock lock = srcExtMapLock.lockForRead()) { + TableExtDesc result = srcExtMap.get(mapKey(t.getIdentity(), t.getProject())); - if (null == result) { - //TODO: notice the table ext is not project-specific, seems not necessary at all - result = srcTableExtMap.get(mapKey(t.getIdentity(), null)); - } + if (null == result) { + //TODO: notice the table ext is not project-specific, seems not necessary at all + result = srcExtMap.get(mapKey(t.getIdentity(), null)); + } - // avoid returning null, since the TableDesc exists - if (null == result) { - result = new TableExtDesc(); - result.setIdentity(t.getIdentity()); - result.setUuid(UUID.randomUUID().toString()); - result.setLastModified(0); - result.init(t.getProject()); - srcTableExtMap.put(mapKey(t.getIdentity(), t.getProject()), result); + // avoid returning null, since the TableDesc exists + if (null == result) { + result = new TableExtDesc(); + result.setIdentity(t.getIdentity()); + result.setUuid(UUID.randomUUID().toString()); + result.setLastModified(0); + result.init(t.getProject()); + srcExtMap.put(mapKey(t.getIdentity(), t.getProject()), result); + } + return result; } - return result; } public void saveTableExt(TableExtDesc tableExt, String prj) throws IOException { - if (tableExt.getUuid() == null || tableExt.getIdentity() == null) { - throw new IllegalArgumentException(); - } - - // updating a legacy global table - if (tableExt.getProject() == null) { - if (getTableExt(tableExt.getIdentity(), prj).getProject() != null) - throw new IllegalStateException( - "Updating a legacy global TableExtDesc while a project level version exists: " - + tableExt.getIdentity() + ", " + prj); - prj = tableExt.getProject(); - } - - tableExt.init(prj); - - String path = TableExtDesc.concatResourcePath(tableExt.getIdentity(), prj); - - ResourceStore store = getStore(); - - TableExtDesc t = store.getResource(path, TableExtDesc.class, TABLE_EXT_SERIALIZER); - if (t != null && t.getIdentity() == null) - store.deleteResource(path); - - store.putResource(path, tableExt, TABLE_EXT_SERIALIZER); - srcTableExtMap.put(mapKey(tableExt.getIdentity(), tableExt.getProject()), tableExt); - } - - public void removeTableExt(String tableName, String prj) throws IOException { - // note, here assume always delete TableExtDesc first, then TableDesc - TableExtDesc t = getTableExt(tableName, prj); - if (t == null) - return; - - String path = TableExtDesc.concatResourcePath(t.getIdentity(), t.getProject()); - getStore().deleteResource(path); - srcTableExtMap.remove(mapKey(t.getIdentity(), t.getProject())); - } - - public void saveSourceTable(TableDesc srcTable, String prj) throws IOException { - if (srcTable.getUuid() == null || srcTable.getIdentity() == null) { - throw new IllegalArgumentException(); - } - - srcTable.init(prj); - - String path = srcTable.getResourcePath(); - getStore().putResource(path, srcTable, TABLE_SERIALIZER); - - srcTableMap.put(mapKey(srcTable.getIdentity(), prj), srcTable); - } - - public void removeSourceTable(String tableIdentity, String prj) throws IOException { - TableDesc t = getTableDesc(tableIdentity, prj); - if (t == null) - return; - - String path = t.getResourcePath(); - getStore().deleteResource(path); - srcTableMap.remove(mapKey(t.getIdentity(), t.getProject())); - } - - public void saveExternalFilter(ExternalFilterDesc desc) throws IOException { - if (desc.getUuid() == null) { - throw new IllegalArgumentException("UUID not set."); - } - String path = desc.getResourcePath(); - getStore().putResource(path, desc, EXTERNAL_FILTER_DESC_SERIALIZER); - desc = reloadExternalFilterAt(path); - extFilterMap.put(desc.getName(), desc); - - } - - public void removeExternalFilter(String name) throws IOException { - String path = ExternalFilterDesc.concatResourcePath(name); - getStore().deleteResource(path); - extFilterMap.remove(name); - - } - - private void init(KylinConfig config) throws IOException { - this.config = config; - this.srcTableMap = new CaseInsensitiveStringCache<>(config, "table"); - this.srcTableExtMap = new CaseInsensitiveStringCache<>(config, "table_ext"); - this.extFilterMap = new CaseInsensitiveStringCache<>(config, "external_filter"); - - reloadAllSourceTable(); - reloadAllTableExt(); - reloadAllExternalFilter(); - - // touch lower level metadata before registering my listener - Broadcaster.getInstance(config).registerListener(new SrcTableSyncListener(), "table"); - Broadcaster.getInstance(config).registerListener(new SrcTableExtSyncListener(), "table_ext"); - Broadcaster.getInstance(config).registerListener(new ExtFilterSyncListener(), "external_filter"); - } - - private class SrcTableSyncListener extends Broadcaster.Listener { - - @Override - public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) - throws IOException { - if (event == Event.DROP) - srcTableMap.removeLocal(cacheKey); - else - reloadSourceTableAt(TableDesc.concatRawResourcePath(cacheKey)); - - Pair<String, String> pair = TableDesc.parseResourcePath(cacheKey); - String table = pair.getFirst(); - String prj = pair.getSecond(); - - if (prj == null) { - for (ProjectInstance p : ProjectManager.getInstance(config).findProjectsByTable(table)) { - broadcaster.notifyProjectSchemaUpdate(p.getName()); - } - } else { - broadcaster.notifyProjectSchemaUpdate(prj); + try (AutoLock lock = srcExtMapLock.lockForWrite()) { + if (tableExt.getUuid() == null || tableExt.getIdentity() == null) { + throw new IllegalArgumentException(); } - } - } - private class SrcTableExtSyncListener extends Broadcaster.Listener { + // updating a legacy global table + if (tableExt.getProject() == null) { + if (getTableExt(tableExt.getIdentity(), prj).getProject() != null) + throw new IllegalStateException( + "Updating a legacy global TableExtDesc while a project level version exists: " + + tableExt.getIdentity() + ", " + prj); + prj = tableExt.getProject(); + } - @Override - public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) - throws IOException { - if (event == Event.DROP) - srcTableExtMap.removeLocal(cacheKey); - else - reloadTableExtAt(TableExtDesc.concatRawResourcePath(cacheKey)); - } - } + tableExt.init(prj); - private class ExtFilterSyncListener extends Broadcaster.Listener { + // what is this doing?? + String path = TableExtDesc.concatResourcePath(tableExt.getIdentity(), prj); + ResourceStore store = getStore(); + TableExtDesc t = store.getResource(path, TableExtDesc.class, TABLE_EXT_SERIALIZER); + if (t != null && t.getIdentity() == null) + store.deleteResource(path); - @Override - public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) - throws IOException { - if (event == Event.DROP) - extFilterMap.removeLocal(cacheKey); - else - reloadExtFilter(cacheKey); + srcExtCrud.save(tableExt); } } - private void reloadAllTableExt() throws IOException { - ResourceStore store = getStore(); - logger.debug("Reloading Table_exd info from folder " - + store.getReadableResourcePath(ResourceStore.TABLE_EXD_RESOURCE_ROOT)); - - srcTableExtMap.clear(); + public void removeTableExt(String tableName, String prj) throws IOException { + try (AutoLock lock = srcExtMapLock.lockForWrite()) { + // note, here assume always delete TableExtDesc first, then TableDesc + TableExtDesc t = getTableExt(tableName, prj); + if (t == null) + return; - List<String> paths = store.collectResourceRecursively(ResourceStore.TABLE_EXD_RESOURCE_ROOT, - MetadataConstants.FILE_SURFIX); - for (String path : paths) { - reloadTableExtAt(path); + srcExtCrud.delete(t); } - - logger.debug("Loaded " + srcTableExtMap.size() + " SourceTable EXD(s)"); } - private TableExtDesc reloadTableExtAt(String path) throws IOException { + private TableExtDesc convertOldTableExtToNewer(String resourceName) { ResourceStore store = getStore(); - String prj = TableExtDesc.parseResourcePath(path).getSecond(); - - TableExtDesc t = store.getResource(path, TableExtDesc.class, TABLE_EXT_SERIALIZER); - - if (t == null) { - return null; - } - - // convert old tableExt json to new one - if (t.getIdentity() == null) { - t = convertOldTableExtToNewer(path); - } - - t.init(prj); - - srcTableExtMap.putLocal(mapKey(t.getIdentity(), prj), t); - return t; - } - - private String mapKey(String identity, String prj) { - return prj == null ? identity : identity + "--" + prj; - } - - private TableExtDesc convertOldTableExtToNewer(String path) throws IOException { Map<String, String> attrs = Maps.newHashMap(); - ResourceStore store = getStore(); - RawResource res = store.getResource(path); - - InputStream is = res.inputStream; - try { - attrs.putAll(JsonUtil.readValue(is, HashMap.class)); - } finally { - if (is != null) - is.close(); + RawResource res = store.getResource( + ResourceStore.TABLE_EXD_RESOURCE_ROOT + "/" + resourceName + MetadataConstants.FILE_SURFIX); + + InputStream is = res.inputStream; + try { + attrs.putAll(JsonUtil.readValue(is, HashMap.class)); + } finally { + if (is != null) + is.close(); + } + } catch (IOException ex) { + throw new RuntimeException(ex); } String cardinality = attrs.get(MetadataConstants.TABLE_EXD_CARDINALITY); // parse table identity from file name - String file = path; - if (file.indexOf("/") > -1) { - file = file.substring(file.lastIndexOf("/") + 1); - } - String tableIdentity = file.substring(0, file.length() - MetadataConstants.FILE_SURFIX.length()).toUpperCase(); + String tableIdentity = TableDesc.parseResourcePath(resourceName).getFirst(); TableExtDesc result = new TableExtDesc(); result.setIdentity(tableIdentity); result.setUuid(UUID.randomUUID().toString()); @@ -440,66 +414,59 @@ public class TableMetadataManager { return result; } - private void reloadAllExternalFilter() throws IOException { - ResourceStore store = getStore(); - logger.debug("Reloading ExternalFilter from folder " - + store.getReadableResourcePath(ResourceStore.EXTERNAL_FILTER_RESOURCE_ROOT)); + // ============================================================================ + // ExternalFilterDesc methods + // ============================================================================ - extFilterMap.clear(); + private void initExtFilter() throws IOException { + this.extFilterMap = new CaseInsensitiveStringCache<>(config, "external_filter"); + this.extFilterCrud = new CachedCrudAssist<ExternalFilterDesc>(getStore(), + ResourceStore.EXTERNAL_FILTER_RESOURCE_ROOT, ExternalFilterDesc.class, extFilterMap) { + @Override + protected ExternalFilterDesc initEntityAfterReload(ExternalFilterDesc t, String resourceName) { + return t; // noop + } + }; + extFilterCrud.reloadAll(); + Broadcaster.getInstance(config).registerListener(new ExtFilterSyncListener(), "external_filter"); + } - List<String> paths = store.collectResourceRecursively(ResourceStore.EXTERNAL_FILTER_RESOURCE_ROOT, - MetadataConstants.FILE_SURFIX); - for (String path : paths) { - reloadExternalFilterAt(path); + private class ExtFilterSyncListener extends Broadcaster.Listener { + @Override + public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) + throws IOException { + try (AutoLock lock = extFilterMapLock.lockForWrite()) { + if (event == Event.DROP) + extFilterMap.removeLocal(cacheKey); + else + extFilterCrud.reloadQuietly(cacheKey); + } } - - logger.debug("Loaded " + extFilterMap.size() + " ExternalFilter(s)"); } - private void reloadAllSourceTable() throws IOException { - ResourceStore store = getStore(); - logger.debug("Reloading SourceTable from folder " - + store.getReadableResourcePath(ResourceStore.TABLE_RESOURCE_ROOT)); - - srcTableMap.clear(); - - List<String> paths = store.collectResourceRecursively(ResourceStore.TABLE_RESOURCE_ROOT, - MetadataConstants.FILE_SURFIX); - for (String path : paths) { - reloadSourceTableAt(path); + public List<ExternalFilterDesc> listAllExternalFilters() { + try (AutoLock lock = extFilterMapLock.lockForRead()) { + return Lists.newArrayList(extFilterMap.values()); } - - logger.debug("Loaded " + srcTableMap.size() + " SourceTable(s)"); } - private TableDesc reloadSourceTableAt(String path) throws IOException { - ResourceStore store = getStore(); - String prj = TableDesc.parseResourcePath(path).getSecond(); - - TableDesc t = store.getResource(path, TableDesc.class, TABLE_SERIALIZER); - if (t == null) { - return null; + public ExternalFilterDesc getExtFilterDesc(String filterTableName) { + try (AutoLock lock = extFilterMapLock.lockForRead()) { + ExternalFilterDesc result = extFilterMap.get(filterTableName); + return result; } - t.init(prj); - - srcTableMap.putLocal(mapKey(t.getIdentity(), prj), t); - - return t; } - private ExternalFilterDesc reloadExternalFilterAt(String path) throws IOException { - ResourceStore store = getStore(); - ExternalFilterDesc t = store.getResource(path, ExternalFilterDesc.class, EXTERNAL_FILTER_DESC_SERIALIZER); - if (t == null) { - return null; + public void saveExternalFilter(ExternalFilterDesc desc) throws IOException { + try (AutoLock lock = extFilterMapLock.lockForWrite()) { + extFilterCrud.save(desc); } - extFilterMap.putLocal(t.getName(), t); - - return t; } - public void reloadExtFilter(String extFilterName) throws IOException { - reloadExternalFilterAt(ExternalFilterDesc.concatResourcePath(extFilterName)); + public void removeExternalFilter(String name) throws IOException { + try (AutoLock lock = extFilterMapLock.lockForWrite()) { + extFilterCrud.delete(name); + } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f487fe/core-metadata/src/main/java/org/apache/kylin/metadata/TempStatementManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/TempStatementManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/TempStatementManager.java index 30ff934..970df0c 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/TempStatementManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/TempStatementManager.java @@ -23,11 +23,12 @@ import java.util.ArrayList; import java.util.List; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.JsonSerializer; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.RootPersistentEntity; -import org.apache.kylin.common.persistence.Serializer; +import org.apache.kylin.common.util.AutoReadWriteLock; +import org.apache.kylin.common.util.AutoReadWriteLock.AutoLock; import org.apache.kylin.metadata.cachesync.Broadcaster; +import org.apache.kylin.metadata.cachesync.CachedCrudAssist; import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,8 +38,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; public class TempStatementManager { private static final Logger logger = LoggerFactory.getLogger(TempStatementManager.class); - public static final Serializer<TempStatementEntity> TEMP_STATEMENT_SERIALIZER = new JsonSerializer<>( - TempStatementEntity.class); public static TempStatementManager getInstance(KylinConfig config) { return config.getManager(TempStatementManager.class); @@ -52,73 +51,62 @@ public class TempStatementManager { // ============================================================================ private KylinConfig config; - private CaseInsensitiveStringCache<String> tempStatementMap; - - private TempStatementManager(KylinConfig config) throws IOException { - init(config); - } - - private void init(KylinConfig config) throws IOException { - this.config = config; - this.tempStatementMap = new CaseInsensitiveStringCache<>(config, "temp_statement"); + private CaseInsensitiveStringCache<TempStatementEntity> tmpStatMap; + private CachedCrudAssist<TempStatementEntity> crud; + private AutoReadWriteLock lock = new AutoReadWriteLock(); + + private TempStatementManager(KylinConfig cfg) throws IOException { + this.config = cfg; + this.tmpStatMap = new CaseInsensitiveStringCache<>(config, "temp_statement"); + this.crud = new CachedCrudAssist<TempStatementEntity>(getStore(), ResourceStore.TEMP_STATMENT_RESOURCE_ROOT, + TempStatementEntity.class, tmpStatMap) { + @Override + protected TempStatementEntity initEntityAfterReload(TempStatementEntity t, String resourceName) { + return t; // noop + } + }; - reloadAllTempStatement(); + crud.reloadAll(); // touch lower level metadata before registering my listener Broadcaster.getInstance(config).registerListener(new TempStatementSyncListener(), "temp_statement"); } - private void reloadAllTempStatement() throws IOException { - ResourceStore store = getStore(); - logger.debug("Reloading temp statement from folder " - + store.getReadableResourcePath(ResourceStore.TEMP_STATMENT_RESOURCE_ROOT)); - - tempStatementMap.clear(); + private class TempStatementSyncListener extends Broadcaster.Listener { - List<String> paths = store.collectResourceRecursively(ResourceStore.TEMP_STATMENT_RESOURCE_ROOT, - MetadataConstants.FILE_SURFIX); - for (String path : paths) { - reloadTempStatementAt(path); + @Override + public void onEntityChange(Broadcaster broadcaster, String entity, Broadcaster.Event event, String cacheKey) + throws IOException { + try (AutoLock l = lock.lockForWrite()) { + if (event == Broadcaster.Event.DROP) + tmpStatMap.removeLocal(cacheKey); + else + crud.reloadQuietly(cacheKey); + } } - - logger.debug("Loaded " + tempStatementMap.size() + " Temp Statement(s)"); } - private TempStatementEntity reloadTempStatement(String statementId) throws IOException { - return reloadTempStatement(TempStatementEntity.DEFAULT_SESSION_ID, statementId); + public String getTempStatement(String statementId) { + return getTempStatement(TempStatementEntity.DEFAULT_SESSION_ID, statementId); } - private TempStatementEntity reloadTempStatement(String sessionId, String statementId) throws IOException { - return reloadTempStatementAt(TempStatementEntity.concatResourcePath(sessionId, statementId)); + public String getTempStatement(String sessionId, String statementId) { + TempStatementEntity entity = getTempStatEntity(sessionId, statementId); + return entity == null ? null : entity.statement; } - private TempStatementEntity reloadTempStatementAt(String path) throws IOException { - ResourceStore store = getStore(); - - TempStatementEntity s = store.getResource(path, TempStatementEntity.class, TEMP_STATEMENT_SERIALIZER); - if (s == null) { - return null; + public TempStatementEntity getTempStatEntity(String sessionId, String statementId) { + try (AutoLock l = lock.lockForRead()) { + return tmpStatMap.get(TempStatementEntity.resourceName(sessionId, statementId)); } - - tempStatementMap.putLocal(s.getMapKey(), s.getStatement()); - return s; - } - - public ResourceStore getStore() { - return ResourceStore.getStore(this.config); - } - - public String getTempStatement(String statementId) { - return getTempStatement(TempStatementEntity.DEFAULT_SESSION_ID, statementId); } - public String getTempStatement(String sessionId, String statementId) { - return tempStatementMap.get(TempStatementEntity.getMapKey(sessionId, statementId)); - } // for test - List<String> listAllTempStatement() throws IOException { - reloadAllTempStatement(); - return new ArrayList<>(tempStatementMap.keySet()); + List<String> reloadAllTempStatement() throws IOException { + try (AutoLock l = lock.lockForWrite()) { + crud.reloadAll(); + return new ArrayList<>(tmpStatMap.keySet()); + } } public void updateTempStatement(String statementId, String statement) throws IOException { @@ -126,15 +114,28 @@ public class TempStatementManager { } public void updateTempStatement(String sessionId, String statementId, String statement) throws IOException { - TempStatementEntity entity = new TempStatementEntity(sessionId, statementId, statement); - updateTempStatementWithRetry(entity, 0); - tempStatementMap.put(entity.getMapKey(), statement); + try (AutoLock l = lock.lockForWrite()) { + TempStatementEntity entity = new TempStatementEntity(sessionId, statementId, statement); + entity = prepareToOverwrite(entity, getTempStatEntity(sessionId, statementId)); + updateTempStatementWithRetry(entity, 0); + } } - public void updateTempStatementWithRetry(TempStatementEntity entity, int retry) throws IOException { - ResourceStore store = getStore(); + private TempStatementEntity prepareToOverwrite(TempStatementEntity entity, TempStatementEntity origin) { + if (origin == null) { + // create + entity.updateRandomUuid(); + } else { + // update + entity.setUuid(origin.getUuid()); + entity.setLastModified(origin.getLastModified()); + } + return entity; + } + + private void updateTempStatementWithRetry(TempStatementEntity entity, int retry) throws IOException { try { - store.putResource(entity.concatResourcePath(), entity, TEMP_STATEMENT_SERIALIZER); + crud.save(entity); } catch (IllegalStateException ise) { logger.warn("Write conflict to update temp statement" + entity.statementId + " at try " + retry + ", will retry..."); @@ -143,35 +144,27 @@ public class TempStatementManager { throw ise; } - TempStatementEntity reload = reloadTempStatement(entity.statementId); - reload.setStatement(entity.statement); - retry++; - updateTempStatementWithRetry(reload, retry); + TempStatementEntity reload = crud.reload(entity.resourceName()); + entity = prepareToOverwrite(entity, reload); + updateTempStatementWithRetry(entity, ++retry); } } + public void removeTempStatement(String statementId) throws IOException { removeTempStatement(TempStatementEntity.DEFAULT_SESSION_ID, statementId); } public void removeTempStatement(String session, String statementId) throws IOException { - ResourceStore store = getStore(); - store.deleteResource(TempStatementEntity.concatResourcePath(session, statementId)); - tempStatementMap.remove(TempStatementEntity.concatResourcePath(session, statementId)); + try (AutoLock l = lock.lockForWrite()) { + crud.delete(TempStatementEntity.resourceName(session, statementId)); + } } - private class TempStatementSyncListener extends Broadcaster.Listener { - - @Override - public void onEntityChange(Broadcaster broadcaster, String entity, Broadcaster.Event event, String cacheKey) - throws IOException { - if (event == Broadcaster.Event.DROP) - tempStatementMap.removeLocal(cacheKey); - else - reloadTempStatementAt(TempStatementEntity.concatResourcePath(cacheKey)); - } + private ResourceStore getStore() { + return ResourceStore.getStore(this.config); } - @SuppressWarnings("serial") + @SuppressWarnings({ "serial", "unused" }) @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) private static class TempStatementEntity extends RootPersistentEntity { private static final String DEFAULT_SESSION_ID = "DEFAULT_SESSION"; @@ -217,10 +210,15 @@ public class TempStatementManager { * @return */ public String getMapKey() { + return resourceName(); + } + + @Override + public String resourceName() { return sessionId + "/" + statementId; } - public static String getMapKey(String sessionId, String statementId) { + public static String resourceName(String sessionId, String statementId) { return sessionId + "/" + statementId; } @@ -233,7 +231,8 @@ public class TempStatementManager { } public static String concatResourcePath(String sessionId, String statementId) { - return ResourceStore.TEMP_STATMENT_RESOURCE_ROOT + "/" + sessionId + "/" + statementId + MetadataConstants.FILE_SURFIX; + return ResourceStore.TEMP_STATMENT_RESOURCE_ROOT + "/" + sessionId + "/" + statementId + + MetadataConstants.FILE_SURFIX; } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f487fe/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java b/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java index 22e55cc..57ebb61 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java @@ -48,6 +48,17 @@ public class TableACL extends RootPersistentEntity { @JsonProperty() private TableACLEntry groupTableBlackList = new TableACLEntry(); + private String project; + + void init(String project) { + this.project = project; + } + + @Override + public String resourceName() { + return project; + } + public Set<String> getTableBlackList(String username, Set<String> groups) { Set<String> tableBlackList = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); tableBlackList.addAll(userTableBlackList.getTableBlackList(username)); http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f487fe/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java index 905fa16..163d340 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java @@ -19,13 +19,14 @@ package org.apache.kylin.metadata.acl; import java.io.IOException; -import java.util.List; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.JsonSerializer; import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.Serializer; +import org.apache.kylin.common.util.AutoReadWriteLock; +import org.apache.kylin.common.util.AutoReadWriteLock.AutoLock; import org.apache.kylin.metadata.cachesync.Broadcaster; +import org.apache.kylin.metadata.cachesync.Broadcaster.Event; +import org.apache.kylin.metadata.cachesync.CachedCrudAssist; import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,9 +37,6 @@ public class TableACLManager { private static final Logger logger = LoggerFactory.getLogger(TableACLManager.class); - private static final Serializer<TableACL> TABLE_ACL_SERIALIZER = new JsonSerializer<>(TableACL.class); - private static final String DIR_PREFIX = "/table_acl/"; - public static TableACLManager getInstance(KylinConfig config) { return config.getManager(TableACLManager.class); } @@ -53,20 +51,36 @@ public class TableACLManager { private KylinConfig config; // user ==> TableACL private CaseInsensitiveStringCache<TableACL> tableACLMap; + private CachedCrudAssist<TableACL> crud; + private AutoReadWriteLock lock = new AutoReadWriteLock(); public TableACLManager(KylinConfig config) throws IOException { logger.info("Initializing TableACLManager with config " + config); this.config = config; this.tableACLMap = new CaseInsensitiveStringCache<>(config, "table_acl"); - loadAllTableACL(); + this.crud = new CachedCrudAssist<TableACL>(getStore(), "/table_acl", "", TableACL.class, tableACLMap) { + @Override + protected TableACL initEntityAfterReload(TableACL acl, String resourceName) { + acl.init(resourceName); + return acl; + } + }; + + crud.reloadAll(); Broadcaster.getInstance(config).registerListener(new TableACLSyncListener(), "table_acl"); } private class TableACLSyncListener extends Broadcaster.Listener { @Override - public void onEntityChange(Broadcaster broadcaster, String entity, Broadcaster.Event event, String cacheKey) throws IOException { - reloadTableACL(cacheKey); + public void onEntityChange(Broadcaster broadcaster, String entity, Broadcaster.Event event, String cacheKey) + throws IOException { + try (AutoLock l = lock.lockForWrite()) { + if (event == Event.DROP) + tableACLMap.removeLocal(cacheKey); + else + crud.reloadQuietly(cacheKey); + } broadcaster.notifyProjectACLUpdate(cacheKey); } } @@ -79,64 +93,57 @@ public class TableACLManager { return ResourceStore.getStore(this.config); } - public TableACL getTableACLByCache(String project){ - TableACL tableACL = tableACLMap.get(project); - if (tableACL == null) { - return new TableACL(); + public TableACL getTableACLByCache(String project) { + try (AutoLock l = lock.lockForRead()) { + TableACL tableACL = tableACLMap.get(project); + if (tableACL == null) { + return newTableACL(project); + } + return tableACL; } - return tableACL; } - private void loadAllTableACL() throws IOException { - ResourceStore store = getStore(); - List<String> paths = store.collectResourceRecursively("/table_acl", ""); - final int prefixLen = DIR_PREFIX.length(); - for (String path : paths) { - String project = path.substring(prefixLen, path.length()); - reloadTableACL(project); + public void addTableACL(String project, String name, String table, String type) throws IOException { + try (AutoLock l = lock.lockForWrite()) { + TableACL tableACL = loadTableACL(project).add(name, table, type); + crud.save(tableACL); } - logger.info("Loading table ACL from folder " + store.getReadableResourcePath("/table_acl")); } - private void reloadTableACL(String project) throws IOException { - TableACL tableACLRecord = getTableACL(project); - tableACLMap.putLocal(project, tableACLRecord); + public void deleteTableACL(String project, String name, String table, String type) throws IOException { + try (AutoLock l = lock.lockForWrite()) { + TableACL tableACL = loadTableACL(project).delete(name, table, type); + crud.save(tableACL); + } } - private TableACL getTableACL(String project) throws IOException { - String path = DIR_PREFIX + project; - TableACL tableACLRecord = getStore().getResource(path, TableACL.class, TABLE_ACL_SERIALIZER); - if (tableACLRecord == null) { - return new TableACL(); + public void deleteTableACL(String project, String name, String type) throws IOException { + try (AutoLock l = lock.lockForWrite()) { + TableACL tableACL = loadTableACL(project).delete(name, type); + crud.save(tableACL); } - return tableACLRecord; } - public void addTableACL(String project, String name, String table, String type) throws IOException { - String path = DIR_PREFIX + project; - TableACL tableACL = getTableACL(project).add(name, table, type); - getStore().putResource(path, tableACL, System.currentTimeMillis(), TABLE_ACL_SERIALIZER); - tableACLMap.put(project, tableACL); + public void deleteTableACLByTbl(String project, String table) throws IOException { + try (AutoLock l = lock.lockForWrite()) { + TableACL tableACL = loadTableACL(project).deleteByTbl(table); + crud.save(tableACL); + } } - public void deleteTableACL(String project, String name, String table, String type) throws IOException { - String path = DIR_PREFIX + project; - TableACL tableACL = getTableACL(project).delete(name, table, type); - getStore().putResource(path, tableACL, System.currentTimeMillis(), TABLE_ACL_SERIALIZER); - tableACLMap.put(project, tableACL); + private TableACL loadTableACL(String project) throws IOException { + TableACL acl = crud.reload(project); + if (acl == null) { + acl = newTableACL(project); + } + return acl; } - public void deleteTableACL(String project, String name, String type) throws IOException { - String path = DIR_PREFIX + project; - TableACL tableACL = getTableACL(project).delete(name, type); - getStore().putResource(path, tableACL, System.currentTimeMillis(), TABLE_ACL_SERIALIZER); - tableACLMap.put(project, tableACL); + private TableACL newTableACL(String project) { + TableACL acl = new TableACL(); + acl.updateRandomUuid(); + acl.init(project); + return acl; } - public void deleteTableACLByTbl(String project, String table) throws IOException { - String path = DIR_PREFIX + project; - TableACL tableACL = getTableACL(project).deleteByTbl(table); - getStore().putResource(path, tableACL, System.currentTimeMillis(), TABLE_ACL_SERIALIZER); - tableACLMap.put(project, tableACL); - } } http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f487fe/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java index 5c2ca17..ccc0a4a 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java @@ -26,6 +26,7 @@ import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.RootPersistentEntity; import org.apache.kylin.common.persistence.Serializer; import org.apache.kylin.metadata.MetadataConstants; +import org.apache.kylin.metadata.model.DataModelDesc; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,6 +39,7 @@ abstract public class CachedCrudAssist<T extends RootPersistentEntity> { final private ResourceStore store; final private Class<T> entityType; final private String resRootPath; + final private String resPathSuffix; final private Serializer<T> serializer; final private SingleValueCache<String, T> cache; @@ -45,9 +47,15 @@ abstract public class CachedCrudAssist<T extends RootPersistentEntity> { public CachedCrudAssist(ResourceStore store, String resourceRootPath, Class<T> entityType, SingleValueCache<String, T> cache) { + this(store, resourceRootPath, MetadataConstants.FILE_SURFIX, entityType, cache); + } + + public CachedCrudAssist(ResourceStore store, String resourceRootPath, String resourcePathSuffix, + Class<T> entityType, SingleValueCache<String, T> cache) { this.store = store; this.entityType = entityType; this.resRootPath = resourceRootPath; + this.resPathSuffix = resourcePathSuffix; this.serializer = new JsonSerializer<T>(entityType); this.cache = cache; @@ -57,12 +65,23 @@ abstract public class CachedCrudAssist<T extends RootPersistentEntity> { Preconditions.checkArgument(resRootPath.endsWith("/") == false); } + public Serializer<DataModelDesc> getSerializer() { + return (Serializer<DataModelDesc>) serializer; + } + public void setCheckOnWrite(boolean check) { this.checkCopyOnWrite = check; } private String resourcePath(String resourceName) { - return resRootPath + "/" + resourceName + MetadataConstants.FILE_SURFIX; + return resRootPath + "/" + resourceName + resPathSuffix; + } + + private String resourceName(String resourcePath) { + Preconditions.checkArgument(resourcePath.startsWith(resRootPath)); + Preconditions.checkArgument(resourcePath.endsWith(resPathSuffix)); + return resourcePath.substring(resRootPath.length() + 1, + resourcePath.length() - resPathSuffix.length()); } public void reloadAll() throws IOException { @@ -70,15 +89,16 @@ abstract public class CachedCrudAssist<T extends RootPersistentEntity> { cache.clear(); - List<String> paths = store.collectResourceRecursively(resRootPath, MetadataConstants.FILE_SURFIX); + List<String> paths = store.collectResourceRecursively(resRootPath, resPathSuffix); for (String path : paths) { reloadQuietlyAt(path); } - logger.debug("Loaded " + cache.size() + " " + entityType.getName() + "(s)"); + logger.debug( + "Loaded " + cache.size() + " " + entityType.getName() + "(s) out of " + paths.size() + " resource"); } - public T reload(String resourceName) throws IOException { + public T reload(String resourceName) { return reloadAt(resourcePath(resourceName)); } @@ -95,25 +115,38 @@ abstract public class CachedCrudAssist<T extends RootPersistentEntity> { } } - public T reloadAt(String path) throws IOException { - T entity = store.getResource(path, entityType, serializer); - if (entity == null) { - logger.warn("No " + entityType.getName() + " found at " + path + ", returning null"); - return null; + public T reloadAt(String path) { + try { + T entity = store.getResource(path, entityType, serializer); + if (entity == null) { + logger.warn("No " + entityType.getName() + " found at " + path + ", returning null"); + cache.removeLocal(resourceName(path)); + return null; + } + + entity = initEntityAfterReload(entity, resourceName(path)); + + if (path.equals(resourcePath(entity.resourceName())) == false) + throw new IllegalStateException("The entity " + entity + " read from " + path + + " will save to a different path " + resourcePath(entity.resourceName())); + + cache.putLocal(entity.resourceName(), entity); + return entity; + } catch (Exception e) { + throw new IllegalStateException("Error loading " + entityType.getName() + " at " + path, e); } - - initEntityAfterReload(entity); - cache.putLocal(entity.resourceName(), entity); - return entity; } - abstract protected void initEntityAfterReload(T entity); + abstract protected T initEntityAfterReload(T entity, String resourceName); public T save(T entity) throws IOException { Preconditions.checkArgument(entity != null); + Preconditions.checkArgument(entity.getUuid() != null); Preconditions.checkArgument(entityType.isInstance(entity)); String resName = entity.resourceName(); + Preconditions.checkArgument(resName != null && resName.length() > 0); + if (checkCopyOnWrite) { if (cache.get(resName) == entity) { throw new IllegalStateException("Copy-on-write violation! The updating entity " + entity @@ -121,7 +154,10 @@ abstract public class CachedCrudAssist<T extends RootPersistentEntity> { } } - store.putResource(resourcePath(resName), entity, serializer); + String path = resourcePath(resName); + logger.debug("Saving {} at {}", entityType.getName(), path); + + store.putResource(path, entity, serializer); cache.put(resName, entity); return entity; } @@ -132,7 +168,11 @@ abstract public class CachedCrudAssist<T extends RootPersistentEntity> { public void delete(String resName) throws IOException { Preconditions.checkArgument(resName != null); - store.deleteResource(resourcePath(resName)); + + String path = resourcePath(resName); + logger.debug("Deleting {} at {}", entityType.getName(), path); + + store.deleteResource(path); cache.remove(resName); } http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f487fe/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java index 6e30ffb..bf0db73 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java @@ -124,6 +124,11 @@ public class DataModelDesc extends RootPersistentEntity { public KylinConfig getConfig() { return config; } + + @Override + public String resourceName() { + return name; + } public String getName() { return name; @@ -721,7 +726,7 @@ public class DataModelDesc extends RootPersistentEntity { } public String getResourcePath() { - return concatResourcePath(name); + return concatResourcePath(resourceName()); } public static String concatResourcePath(String descName) { http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f487fe/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java index dc1ac44..c06a31b 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java @@ -24,15 +24,16 @@ import java.util.List; import java.util.Map; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.JsonSerializer; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.Serializer; +import org.apache.kylin.common.util.AutoReadWriteLock; +import org.apache.kylin.common.util.AutoReadWriteLock.AutoLock; import org.apache.kylin.common.util.ClassUtil; import org.apache.kylin.common.util.StringUtil; -import org.apache.kylin.metadata.MetadataConstants; import org.apache.kylin.metadata.TableMetadataManager; import org.apache.kylin.metadata.cachesync.Broadcaster; import org.apache.kylin.metadata.cachesync.Broadcaster.Event; +import org.apache.kylin.metadata.cachesync.CachedCrudAssist; import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.project.ProjectManager; @@ -65,48 +66,37 @@ public class DataModelManager { // ============================================================================ private KylinConfig config; - private Serializer<DataModelDesc> serializer; // name => DataModelDesc private CaseInsensitiveStringCache<DataModelDesc> dataModelDescMap; + private CachedCrudAssist<DataModelDesc> crud; + + // protects concurrent operations around the cached map, to avoid for example + // writing an entity in the middle of reloading it (dirty read) + private AutoReadWriteLock modelMapLock = new AutoReadWriteLock(); public DataModelManager(KylinConfig config) throws IOException { init(config); } - public KylinConfig getConfig() { - return config; - } - - public ResourceStore getStore() { - return ResourceStore.getStore(this.config); - } - - public Serializer<DataModelDesc> getDataModelSerializer() { - if (serializer == null) { - try { - String cls = StringUtil.noBlank(config.getDataModelImpl(), DataModelDesc.class.getName()); - Class<? extends DataModelDesc> clz = ClassUtil.forName(cls, DataModelDesc.class); - serializer = new JsonSerializer(clz); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } - } - return serializer; - } - - public List<DataModelDesc> listDataModels() { - return Lists.newArrayList(dataModelDescMap.values()); - } - - protected void init(KylinConfig config) throws IOException { - this.config = config; + protected void init(KylinConfig cfg) throws IOException { + this.config = cfg; this.dataModelDescMap = new CaseInsensitiveStringCache<>(config, "data_model"); + this.crud = new CachedCrudAssist<DataModelDesc>(getStore(), ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT, + getDataModelImplClass(), dataModelDescMap) { + @Override + protected DataModelDesc initEntityAfterReload(DataModelDesc model, String resourceName) { + String prj = ProjectManager.getInstance(config).getProjectOfModel(model.getName()).getName(); + if (!model.isDraft()) { + model.init(config, getAllTablesMap(prj), listDataModels(), true); + } + return model; + } + }; // touch lower level metadata before registering model listener TableMetadataManager.getInstance(config); - - reloadAllDataModel(); + crud.reloadAll(); Broadcaster.getInstance(config).registerListener(new DataModelSyncListener(), "data_model"); } @@ -117,18 +107,22 @@ public class DataModelManager { //clean up the current project's table desc TableMetadataManager.getInstance(config).resetProjectSpecificTableDesc(project); - for (String model : ProjectManager.getInstance(config).getProject(project).getModels()) { - reloadDataModelDescAt(DataModelDesc.concatResourcePath(model)); + try (AutoLock lock = modelMapLock.lockForWrite()) { + for (String model : ProjectManager.getInstance(config).getProject(project).getModels()) { + crud.reloadQuietly(model); + } } } @Override public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) throws IOException { - if (event == Event.DROP) - dataModelDescMap.removeLocal(cacheKey); - else - reloadDataModelDescAt(DataModelDesc.concatResourcePath(cacheKey)); + try (AutoLock lock = modelMapLock.lockForWrite()) { + if (event == Event.DROP) + dataModelDescMap.removeLocal(cacheKey); + else + crud.reloadQuietly(cacheKey); + } for (ProjectInstance prj : ProjectManager.getInstance(config).findProjectsByModel(cacheKey)) { broadcaster.notifyProjectSchemaUpdate(prj.getName()); @@ -136,141 +130,143 @@ public class DataModelManager { } } - public DataModelDesc getDataModelDesc(String name) { - return dataModelDescMap.get(name); + private Class<DataModelDesc> getDataModelImplClass() { + try { + String cls = StringUtil.noBlank(config.getDataModelImpl(), DataModelDesc.class.getName()); + Class<? extends DataModelDesc> clz = ClassUtil.forName(cls, DataModelDesc.class); + return (Class<DataModelDesc>) clz; + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } } - public List<DataModelDesc> getModels() { - return new ArrayList<>(dataModelDescMap.values()); + public KylinConfig getConfig() { + return config; } - public List<DataModelDesc> getModels(String projectName) { - ProjectInstance projectInstance = ProjectManager.getInstance(config).getProject(projectName); - ArrayList<DataModelDesc> ret = new ArrayList<>(); - - if (projectInstance != null && projectInstance.getModels() != null) { - for (String modelName : projectInstance.getModels()) { - DataModelDesc model = getDataModelDesc(modelName); - if (null != model) { - ret.add(model); - } else { - logger.error("Failed to load model " + modelName); - } - } - } + public ResourceStore getStore() { + return ResourceStore.getStore(this.config); + } - return ret; + // for test mostly + public Serializer<DataModelDesc> getDataModelSerializer() { + return crud.getSerializer(); } - // within a project, find models that use the specified table - public List<String> getModelsUsingTable(TableDesc table, String project) throws IOException { - List<String> models = new ArrayList<>(); - for (DataModelDesc modelDesc : getModels(project)) { - if (modelDesc.containsTable(table)) - models.add(modelDesc.getName()); + public List<DataModelDesc> listDataModels() { + try (AutoLock lock = modelMapLock.lockForRead()) { + return Lists.newArrayList(dataModelDescMap.values()); } - return models; } - public boolean isTableInAnyModel(TableDesc table) { - for (DataModelDesc modelDesc : getModels()) { - if (modelDesc.containsTable(table)) - return true; + public DataModelDesc getDataModelDesc(String name) { + try (AutoLock lock = modelMapLock.lockForRead()) { + return dataModelDescMap.get(name); } - return false; } - private void reloadAllDataModel() throws IOException { - ResourceStore store = getStore(); - logger.debug("Reloading DataModel from folder " - + store.getReadableResourcePath(ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT)); + public List<DataModelDesc> getModels() { + try (AutoLock lock = modelMapLock.lockForRead()) { + return new ArrayList<>(dataModelDescMap.values()); + } + } - dataModelDescMap.clear(); + public List<DataModelDesc> getModels(String projectName) { + try (AutoLock lock = modelMapLock.lockForRead()) { + ProjectInstance projectInstance = ProjectManager.getInstance(config).getProject(projectName); + ArrayList<DataModelDesc> ret = new ArrayList<>(); + + if (projectInstance != null && projectInstance.getModels() != null) { + for (String modelName : projectInstance.getModels()) { + DataModelDesc model = getDataModelDesc(modelName); + if (null != model) { + ret.add(model); + } else { + logger.error("Failed to load model " + modelName); + } + } + } - List<String> paths = store.collectResourceRecursively(ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT, - MetadataConstants.FILE_SURFIX); - for (String path : paths) { + return ret; + } + } - try { - logger.info("Reloading data model at " + path); - reloadDataModelDescAt(path); - } catch (IllegalStateException e) { - logger.error("Error to load DataModel at " + path, e); - continue; + // within a project, find models that use the specified table + public List<String> getModelsUsingTable(TableDesc table, String project) throws IOException { + try (AutoLock lock = modelMapLock.lockForRead()) { + List<String> models = new ArrayList<>(); + for (DataModelDesc modelDesc : getModels(project)) { + if (modelDesc.containsTable(table)) + models.add(modelDesc.getName()); } + return models; } - - logger.debug("Loaded " + dataModelDescMap.size() + " DataModel(s)"); } - public DataModelDesc reloadDataModelDescAt(String path) { - ResourceStore store = getStore(); - try { - DataModelDesc dataModelDesc = store.getResource(path, DataModelDesc.class, getDataModelSerializer()); - String prj = ProjectManager.getInstance(config).getProjectOfModel(dataModelDesc.getName()).getName(); - - if (!dataModelDesc.isDraft()) { - dataModelDesc.init(config, this.getAllTablesMap(prj), listDataModels(), true); + public boolean isTableInAnyModel(TableDesc table) { + try (AutoLock lock = modelMapLock.lockForRead()) { + for (DataModelDesc modelDesc : getModels()) { + if (modelDesc.containsTable(table)) + return true; } - - dataModelDescMap.putLocal(dataModelDesc.getName(), dataModelDesc); - return dataModelDesc; - } catch (Exception e) { - throw new IllegalStateException("Error to load " + path, e); + return false; } } - // sync on update - public DataModelDesc dropModel(DataModelDesc desc) throws IOException { - logger.info("Dropping model '" + desc.getName() + "'"); - ResourceStore store = getStore(); - store.deleteResource(desc.getResourcePath()); - // delete model from project - ProjectManager.getInstance(config).removeModelFromProjects(desc.getName()); - // clean model cache - this.afterModelDropped(desc); - return desc; + public DataModelDesc reloadDataModel(String modelName) { + try (AutoLock lock = modelMapLock.lockForWrite()) { + return crud.reload(modelName); + } } - private void afterModelDropped(DataModelDesc desc) { - dataModelDescMap.remove(desc.getName()); + public DataModelDesc dropModel(DataModelDesc desc) throws IOException { + try (AutoLock lock = modelMapLock.lockForWrite()) { + crud.delete(desc); + // delete model from project + ProjectManager.getInstance(config).removeModelFromProjects(desc.getName()); + return desc; + } } public DataModelDesc createDataModelDesc(DataModelDesc desc, String projectName, String owner) throws IOException { - String name = desc.getName(); - if (dataModelDescMap.containsKey(name)) - throw new IllegalArgumentException("DataModelDesc '" + name + "' already exists"); + try (AutoLock lock = modelMapLock.lockForWrite()) { + String name = desc.getName(); + if (dataModelDescMap.containsKey(name)) + throw new IllegalArgumentException("DataModelDesc '" + name + "' already exists"); - ProjectManager prjMgr = ProjectManager.getInstance(config); - ProjectInstance prj = prjMgr.getProject(projectName); - if (prj.containsModel(name)) - throw new IllegalStateException("project " + projectName + " already contains model " + name); + ProjectManager prjMgr = ProjectManager.getInstance(config); + ProjectInstance prj = prjMgr.getProject(projectName); + if (prj.containsModel(name)) + throw new IllegalStateException("project " + projectName + " already contains model " + name); - try { - // Temporarily register model under project, because we want to - // update project formally after model is saved. - prj.getModels().add(name); + try { + // Temporarily register model under project, because we want to + // update project formally after model is saved. + prj.getModels().add(name); - desc.setOwner(owner); - desc = saveDataModelDesc(desc); + desc.setOwner(owner); + desc = saveDataModelDesc(desc); - } finally { - prj.getModels().remove(name); - } + } finally { + prj.getModels().remove(name); + } - // now that model is saved, update project formally - prjMgr.addModelToProject(name, projectName); + // now that model is saved, update project formally + prjMgr.addModelToProject(name, projectName); - return desc; + return desc; + } } public DataModelDesc updateDataModelDesc(DataModelDesc desc) throws IOException { - String name = desc.getName(); - if (!dataModelDescMap.containsKey(name)) { - throw new IllegalArgumentException("DataModelDesc '" + name + "' does not exist."); - } + try (AutoLock lock = modelMapLock.lockForWrite()) { + String name = desc.getName(); + if (!dataModelDescMap.containsKey(name)) { + throw new IllegalArgumentException("DataModelDesc '" + name + "' does not exist."); + } - return saveDataModelDesc(desc); + return saveDataModelDesc(desc); + } } private DataModelDesc saveDataModelDesc(DataModelDesc dataModelDesc) throws IOException { @@ -280,9 +276,7 @@ public class DataModelManager { if (!dataModelDesc.isDraft()) dataModelDesc.init(config, this.getAllTablesMap(prj), listDataModels(), true); - String path = dataModelDesc.getResourcePath(); - getStore().putResource(path, dataModelDesc, getDataModelSerializer()); - dataModelDescMap.put(dataModelDesc.getName(), dataModelDesc); + crud.save(dataModelDesc); return dataModelDesc; } @@ -290,4 +284,5 @@ public class DataModelManager { private Map<String, TableDesc> getAllTablesMap(String prj) { return TableMetadataManager.getInstance(config).getAllTablesMap(prj); } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f487fe/core-metadata/src/main/java/org/apache/kylin/metadata/model/ExternalFilterDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ExternalFilterDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ExternalFilterDesc.java index 115b154..35018c7 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ExternalFilterDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ExternalFilterDesc.java @@ -58,6 +58,11 @@ public class ExternalFilterDesc extends RootPersistentEntity implements ISourceA // ============================================================================ + @Override + public String resourceName() { + return name; + } + public String getFilterResourceIdentifier() { return filterResourceIdentifier; } http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f487fe/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java index 3b779db..e456d9a 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java @@ -41,11 +41,49 @@ import com.google.common.collect.Lists; @SuppressWarnings("serial") @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) public class TableDesc extends RootPersistentEntity implements ISourceAware { + @SuppressWarnings("unused") private static final Logger logger = LoggerFactory.getLogger(TableDesc.class); private static final String TABLE_TYPE_VIRTUAL_VIEW = "VIRTUAL_VIEW"; private static final String materializedTableNamePrefix = "kylin_intermediate_"; + public static String concatRawResourcePath(String nameOnPath) { + return ResourceStore.TABLE_RESOURCE_ROOT + "/" + nameOnPath + ".json"; + } + + public static String makeResourceName(String tableIdentity, String prj) { + return prj == null ? tableIdentity : tableIdentity + "--" + prj; + } + + // this method should only used for getting dest path when copying from src to dest. + // if you want to get table's src path, use getResourcePath() instead. + private static String concatResourcePath(String tableIdentity, String prj) { + return concatRawResourcePath(makeResourceName(tableIdentity, prj)); + } + + // returns <table, project> + public static Pair<String, String> parseResourcePath(String path) { + if (path.endsWith(".json")) + path = path.substring(0, path.length() - ".json".length()); + + int cut = path.lastIndexOf("/"); + if (cut >= 0) + path = path.substring(cut + 1); + + String table, prj; + int dash = path.indexOf("--"); + if (dash >= 0) { + table = path.substring(0, dash); + prj = path.substring(dash + 2); + } else { + table = path; + prj = null; + } + return Pair.newPair(table, prj); + } + + // ============================================================================ + @JsonProperty("name") private String name; @JsonProperty("columns") @@ -87,6 +125,11 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware { this.identity = other.identity; } + @Override + public String resourceName() { + return makeResourceName(getIdentity(), getProject()); + } + public TableDesc appendColumns(ColumnDesc[] computedColumns, boolean makeCopy) { if (computedColumns == null || computedColumns.length == 0) { return this; @@ -170,42 +213,6 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware { return TABLE_TYPE_VIRTUAL_VIEW.equals(tableType); } - public static String concatRawResourcePath(String nameOnPath) { - return ResourceStore.TABLE_RESOURCE_ROOT + "/" + nameOnPath + ".json"; - } - - // this method should only used for getting dest path when copying from src to dest. - // if you want to get table's src path, use getResourcePath() instead. - private static String concatResourcePath(String tableIdentity, String prj) { - if (prj == null || prj.isEmpty()) - return ResourceStore.TABLE_RESOURCE_ROOT + "/" + tableIdentity + ".json"; - else - return ResourceStore.TABLE_RESOURCE_ROOT + "/" + tableIdentity + "--" + prj + ".json"; - } - - // returns <table, project> - public static Pair<String, String> parseResourcePath(String path) { - if (path.endsWith(".json")) - path = path.substring(0, path.length() - ".json".length()); - - int cut = path.lastIndexOf("/"); - if (cut >= 0) - path = path.substring(cut + 1); - - String table, prj; - int dash = path.indexOf("--"); - if (dash >= 0) { - table = path.substring(0, dash); - prj = path.substring(dash + 2); - } else { - table = path; - prj = null; - } - return Pair.newPair(table, prj); - } - - // ============================================================================ - public boolean isBorrowedFromGlobal() { return isBorrowedFromGlobal; }
