ATLAS-856 Lazy-load type cache provider (dkantor via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/9b00a9dd Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/9b00a9dd Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/9b00a9dd Branch: refs/heads/master Commit: 9b00a9ddce40150fd66d87ddaae65250247bf196 Parents: a7869be Author: Shwetha GS <[email protected]> Authored: Wed Jun 22 10:56:54 2016 +0530 Committer: Shwetha GS <[email protected]> Committed: Wed Jun 22 10:56:54 2016 +0530 ---------------------------------------------------------------------- .../org/apache/atlas/ApplicationProperties.java | 5 +- release-log.txt | 1 + .../apache/atlas/RepositoryMetadataModule.java | 49 ++- .../typestore/GraphBackedTypeStore.java | 14 + .../atlas/repository/typestore/ITypeStore.java | 9 + .../typestore/StoreBackedTypeCache.java | 250 ++++++++++++ .../atlas/services/DefaultMetadataService.java | 20 +- .../typestore/GraphBackedTypeStoreTest.java | 44 ++- .../StoreBackedTypeCacheConfigurationTest.java | 43 +++ .../typestore/StoreBackedTypeCacheTest.java | 212 ++++++++++ .../StoreBackedTypeCacheTestModule.java | 42 ++ ...StoreBackedTypeCacheMetadataServiceTest.java | 81 ++++ .../DefaultMetadataServiceMockTest.java | 16 +- .../atlas/typesystem/types/TypeSystem.java | 42 +- .../types/cache/DefaultTypeCache.java | 229 +++++++++++ .../types/cache/DefaultTypeCacheProvider.java | 229 ----------- .../types/cache/ITypeCacheProvider.java | 137 ------- .../atlas/typesystem/types/cache/TypeCache.java | 137 +++++++ .../apache/atlas/ApplicationPropertiesTest.java | 11 +- .../cache/DefaultTypeCacheProviderTest.java | 383 ------------------- .../types/cache/DefaultTypeCacheTest.java | 383 +++++++++++++++++++ .../apache/atlas/web/listeners/TestModule.java | 4 +- 22 files changed, 1513 insertions(+), 828 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/common/src/main/java/org/apache/atlas/ApplicationProperties.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/atlas/ApplicationProperties.java b/common/src/main/java/org/apache/atlas/ApplicationProperties.java index 741e1a6..9b1e9cd 100644 --- a/common/src/main/java/org/apache/atlas/ApplicationProperties.java +++ b/common/src/main/java/org/apache/atlas/ApplicationProperties.java @@ -108,10 +108,9 @@ public final class ApplicationProperties extends PropertiesConfiguration { return inConf.subset(prefix); } - public static Class getClass(String propertyName, String defaultValue, Class assignableClass) - throws AtlasException { + public static Class getClass(Configuration configuration, String propertyName, String defaultValue, + Class assignableClass) throws AtlasException { try { - Configuration configuration = get(); String propertyValue = configuration.getString(propertyName, defaultValue); Class<?> clazz = Class.forName(propertyValue); if (assignableClass == null || assignableClass.isAssignableFrom(clazz)) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 661bf21..235cb97 100644 --- a/release-log.txt +++ b/release-log.txt @@ -40,6 +40,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-856 Lazy-load type cache provider (dkantor via shwethags) ATLAS-931 Delete entities fails when hard delete is configured (dkantor via sumasai) ATLAS-932 UI: 'create tag' button does not work (mneethiraj via sumasai) ATLAS-928 UI is not showing the name column for hive tables in the schema tab (yhemanth via sumasai) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java index c4c7678..3486436 100755 --- a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java +++ b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java @@ -24,6 +24,7 @@ import com.google.inject.matcher.Matchers; import com.google.inject.multibindings.Multibinder; import com.google.inject.throwingproviders.ThrowingProviderBinder; import com.thinkaurelius.titan.core.TitanGraph; + import org.aopalliance.intercept.MethodInterceptor; import org.apache.atlas.discovery.DiscoveryService; import org.apache.atlas.discovery.DataSetLineageService; @@ -50,6 +51,9 @@ import org.apache.atlas.services.MetadataService; import org.apache.atlas.services.ReservedTypesRegistrar; import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeSystemProvider; +import org.apache.atlas.typesystem.types.cache.DefaultTypeCache; +import org.apache.atlas.typesystem.types.cache.TypeCache; +import org.apache.commons.configuration.Configuration; /** * Guice module for Repository module. @@ -85,9 +89,12 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { bind(LineageService.class).to(DataSetLineageService.class).asEagerSingleton(); - bindAuditRepository(binder()); + Configuration configuration = getConfiguration(); + bindAuditRepository(binder(), configuration); + + bind(DeleteHandler.class).to(getDeleteHandlerImpl(configuration)).asEagerSingleton(); - bind(DeleteHandler.class).to(getDeleteHandlerImpl()).asEagerSingleton(); + bind(TypeCache.class).to(getTypeCache(configuration)).asEagerSingleton(); //Add EntityAuditListener as EntityChangeListener Multibinder<EntityChangeListener> entityChangeListenerBinder = @@ -99,9 +106,17 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), interceptor); } - protected void bindAuditRepository(Binder binder) { + protected Configuration getConfiguration() { + try { + return ApplicationProperties.get(); + } catch (AtlasException e) { + throw new RuntimeException(e); + } + } + + protected void bindAuditRepository(Binder binder, Configuration configuration) { - Class<? extends EntityAuditRepository> auditRepoImpl = getAuditRepositoryImpl(); + Class<? extends EntityAuditRepository> auditRepoImpl = getAuditRepositoryImpl(getConfiguration()); //Map EntityAuditRepository interface to configured implementation binder.bind(EntityAuditRepository.class).to(auditRepoImpl).asEagerSingleton(); @@ -117,10 +132,10 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { private static final String AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY = "atlas.EntityAuditRepository.impl"; - private Class<? extends EntityAuditRepository> getAuditRepositoryImpl() { + private Class<? extends EntityAuditRepository> getAuditRepositoryImpl(Configuration configuration) { try { - return ApplicationProperties.getClass(AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, - HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class); + return ApplicationProperties.getClass(configuration, + AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class); } catch (AtlasException e) { throw new RuntimeException(e); } @@ -128,12 +143,26 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { private static final String DELETE_HANDLER_IMPLEMENTATION_PROPERTY = "atlas.DeleteHandler.impl"; - private Class<? extends DeleteHandler> getDeleteHandlerImpl() { + private Class<? extends DeleteHandler> getDeleteHandlerImpl(Configuration configuration) { try { - return ApplicationProperties.getClass(DELETE_HANDLER_IMPLEMENTATION_PROPERTY, - SoftDeleteHandler.class.getName(), DeleteHandler.class); + return ApplicationProperties.getClass(configuration, + DELETE_HANDLER_IMPLEMENTATION_PROPERTY, SoftDeleteHandler.class.getName(), DeleteHandler.class); } catch (AtlasException e) { throw new RuntimeException(e); } } + + public static final String TYPE_CACHE_IMPLEMENTATION_PROPERTY = "atlas.TypeCache.impl"; + + protected Class<? extends TypeCache> getTypeCache(Configuration configuration) { + + // Get the type cache implementation class from Atlas configuration. + try { + return ApplicationProperties.getClass(configuration, TYPE_CACHE_IMPLEMENTATION_PROPERTY, + DefaultTypeCache.class.getName(), TypeCache.class); + } catch (AtlasException e) { + throw new RuntimeException("Error getting TypeCache implementation class", e); + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java index 5ed9e02..4503899 100755 --- a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java @@ -214,6 +214,20 @@ public class GraphBackedTypeStore implements ITypeStore { Iterator vertices = titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).vertices().iterator(); + return getTypesFromVertices(vertices); + } + + @Override + @GraphTransaction + public TypesDef restoreType(String typeName) throws AtlasException { + // Get vertex for the specified type name. + Iterator vertices = + titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).has(Constants.TYPENAME_PROPERTY_KEY, typeName).vertices().iterator(); + + return getTypesFromVertices(vertices); + } + + private TypesDef getTypesFromVertices(Iterator vertices) throws AtlasException { ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder(); ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder(); ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java b/repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java index 790c4b3..25f5f8b 100755 --- a/repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java @@ -40,4 +40,13 @@ public interface ITypeStore { * @throws AtlasException */ TypesDef restore() throws AtlasException; + + /** + * Restore the specified type definition + * + * @param typeName name of requested type + * @return persisted type definition + * @throws AtlasException + */ + TypesDef restoreType(String typeName) throws AtlasException; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java b/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java new file mode 100644 index 0000000..d0c6f6a --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java @@ -0,0 +1,250 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.typestore; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.typesystem.TypesDef; +import org.apache.atlas.typesystem.types.AttributeDefinition; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.EnumTypeDefinition; +import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; +import org.apache.atlas.typesystem.types.IDataType; +import org.apache.atlas.typesystem.types.StructTypeDefinition; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.TypeSystem; +import org.apache.atlas.typesystem.types.TypeSystem.TransientTypeSystem; +import org.apache.atlas.typesystem.types.TypeUtils; +import org.apache.atlas.typesystem.types.cache.DefaultTypeCache; +import org.apache.atlas.typesystem.types.utils.TypesUtil; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; +import com.google.inject.Singleton; + + +/** + * An extension of {@link DefaultTypeCache} which loads + * the requested type from the type store if it is not found in the cache, + * and adds it to the cache if it's found in the store. + * Any attribute and super types that are required by the requested type + * are also loaded from the store if they are not already in the cache. + */ +@Singleton +public class StoreBackedTypeCache extends DefaultTypeCache { + + private ITypeStore typeStore; + + private ImmutableList<String> coreTypes; + private TypeSystem typeSystem; + + @Inject + public StoreBackedTypeCache(final ITypeStore typeStore) { + this.typeStore = typeStore; + typeSystem = TypeSystem.getInstance(); + coreTypes = typeSystem.getCoreTypes(); + } + + private static class Context { + ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder(); + ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder(); + ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder(); + ImmutableList.Builder<HierarchicalTypeDefinition<TraitType>> traits = ImmutableList.builder(); + Set<String> loadedFromStore = new HashSet<>(); + + public void addTypesDefToLists(TypesDef typesDef) { + + List<EnumTypeDefinition> enumTypesAsJavaList = typesDef.enumTypesAsJavaList(); + enums.addAll(enumTypesAsJavaList); + for (EnumTypeDefinition etd : enumTypesAsJavaList) { + loadedFromStore.add(etd.name); + } + List<StructTypeDefinition> structTypesAsJavaList = typesDef.structTypesAsJavaList(); + structs.addAll(structTypesAsJavaList); + for (StructTypeDefinition std : structTypesAsJavaList) { + loadedFromStore.add(std.typeName); + } + List<HierarchicalTypeDefinition<ClassType>> classTypesAsJavaList = typesDef.classTypesAsJavaList(); + classTypes.addAll(classTypesAsJavaList); + for (HierarchicalTypeDefinition<ClassType> classTypeDef : classTypesAsJavaList) { + loadedFromStore.add(classTypeDef.typeName); + } + List<HierarchicalTypeDefinition<TraitType>> traitTypesAsJavaList = typesDef.traitTypesAsJavaList(); + traits.addAll(traitTypesAsJavaList); + for (HierarchicalTypeDefinition<TraitType> traitTypeDef : traitTypesAsJavaList) { + loadedFromStore.add(traitTypeDef.typeName); + } + } + + public boolean isLoadedFromStore(String typeName) { + return loadedFromStore.contains(typeName); + } + + public TypesDef getTypesDef() { + return TypesUtil.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build()); + } + } + + @Override + public boolean has(String typeName) throws AtlasException { + + return (get(typeName) != null); + } + + /** + * Checks whether the specified type is cached in memory and does *not* + * access the type store. Used for testing. + * + * @param typeName + * @return + */ + public boolean isCachedInMemory(String typeName) throws AtlasException { + return super.has(typeName); + } + + /** + * Gets the requested type from the cache. + * This implementation will check the type store if the type is + * not already cached. If found in the type store, the type and + * any required super and attribute types are loaded from the type store, and + * added to the cache. + * + * @see org.apache.atlas.typesystem.types.cache.DefaultTypeCache#get(java.lang.String) + */ + @Override + public IDataType get(String typeName) throws AtlasException { + + IDataType type = super.get(typeName); + if (type != null) { + return type; + } + + // Type is not cached - check the type store. + // Any super and attribute types needed by the requested type + // which are not cached will also be loaded from the store. + Context context = new Context(); + TypesDef typesDef = getTypeFromStore(typeName, context); + if (typesDef.isEmpty()) { + // Type not found in the type store. + return null; + } + + // Add all types that were loaded from the store to the cache. + TransientTypeSystem transientTypeSystem = typeSystem.createTransientTypeSystem(context.getTypesDef(), false); + Map<String, IDataType> typesAdded = transientTypeSystem.getTypesAdded(); + putAll(typesAdded.values()); + return typesAdded.get(typeName); + } + + private void getTypeFromCacheOrStore(String typeName, Context context) + throws AtlasException { + + if (coreTypes.contains(typeName) || super.has(typeName)) { + return; + } + + if (context.isLoadedFromStore(typeName)) { + return; + } + + // Type not cached and hasn't been loaded during this operation, so check the store. + TypesDef typesDef = getTypeFromStore(typeName, context); + if (typesDef.isEmpty()) { + // Attribute type not found in cache or store. + throw new AtlasException(typeName + " not found in type store"); + } + } + + private TypesDef getTypeFromStore(String typeName, Context context) + throws AtlasException { + + TypesDef typesDef = typeStore.restoreType(typeName); + if (!typesDef.isEmpty()) { + // Type found in store, add it to lists. + context.addTypesDefToLists(typesDef); + + // Check the attribute and super types that are + // used by the requested type, and restore them + // as needed. + checkAttributeAndSuperTypes(typesDef, context); + } + return typesDef; + } + + private void checkAttributeAndSuperTypes(TypesDef typesDef, Context context) + throws AtlasException { + + // Check the cache and store for attribute types and super types. + for (HierarchicalTypeDefinition<ClassType> classTypeDef : typesDef.classTypesAsJavaList()) { + checkAttributeTypes(classTypeDef.attributeDefinitions, context); + for (String superTypeName : classTypeDef.superTypes) { + getTypeFromCacheOrStore(superTypeName, context); + } + } + for (HierarchicalTypeDefinition<TraitType> traitTypeDef : typesDef.traitTypesAsJavaList()) { + checkAttributeTypes(traitTypeDef.attributeDefinitions, context); + for (String superTypeName : traitTypeDef.superTypes) { + getTypeFromCacheOrStore(superTypeName, context); + } + } + for (StructTypeDefinition structTypeDef : typesDef.structTypesAsJavaList()) { + checkAttributeTypes(structTypeDef.attributeDefinitions, context); + } + } + + private void checkAttributeTypes(AttributeDefinition[] attributeDefinitions, + Context context) throws AtlasException { + + for (AttributeDefinition attrDef : attributeDefinitions) { + checkAttributeType(attrDef, context); + } + } + + private void checkAttributeType(AttributeDefinition attrDef, Context context) throws AtlasException { + + List<String> typeNamesToLookup = new ArrayList<>(2); + + // Get the attribute type(s). + String elementTypeName = TypeUtils.parseAsArrayType(attrDef.dataTypeName); + if (elementTypeName != null) { + // Array attribute, lookup the element type. + typeNamesToLookup.add(elementTypeName); + } + else { + String[] mapTypeNames = TypeUtils.parseAsMapType(attrDef.dataTypeName); + if (mapTypeNames != null) { + // Map attribute, lookup the key and value types. + typeNamesToLookup.addAll(Arrays.asList(mapTypeNames)); + } + else { + // Not an array or map, lookup the attribute type. + typeNamesToLookup.add(attrDef.dataTypeName); + } + } + + for (String typeName : typeNamesToLookup) { + getTypeFromCacheOrStore(typeName, context); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java index 37e7b66..99f0036 100755 --- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java +++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.inject.Provider; + import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; @@ -60,6 +61,7 @@ import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.ValueConversionException; +import org.apache.atlas.typesystem.types.cache.TypeCache; import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.apache.atlas.utils.ParamChecker; import org.apache.commons.configuration.Configuration; @@ -71,6 +73,7 @@ import org.slf4j.LoggerFactory; import javax.inject.Inject; import javax.inject.Singleton; + import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashSet; @@ -109,10 +112,10 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore, final IBootstrapTypesRegistrar typesRegistrar, final Collection<Provider<TypesChangeListener>> typeListenerProviders, - final Collection<Provider<EntityChangeListener>> entityListenerProviders) + final Collection<Provider<EntityChangeListener>> entityListenerProviders, TypeCache typeCache) throws AtlasException { this(repository, typeStore, typesRegistrar, typeListenerProviders, entityListenerProviders, - TypeSystem.getInstance(), ApplicationProperties.get()); + TypeSystem.getInstance(), ApplicationProperties.get(), typeCache); } DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore, @@ -120,10 +123,21 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang final Collection<Provider<TypesChangeListener>> typeListenerProviders, final Collection<Provider<EntityChangeListener>> entityListenerProviders, final TypeSystem typeSystem, - final Configuration configuration) throws AtlasException { + final Configuration configuration, TypeCache typeCache) throws AtlasException { this.typeStore = typeStore; this.typesRegistrar = typesRegistrar; this.typeSystem = typeSystem; + /** + * Ideally a TypeCache implementation should have been injected in the TypeSystemProvider, + * but a singleton of TypeSystem is constructed privately within the class so that + * clients of TypeSystem would never instantiate a TypeSystem object directly in + * their code. As soon as a client makes a call to TypeSystem.getInstance(), they + * should have the singleton ready for consumption. Manually inject TypeSystem with + * the Guice-instantiated type cache here, before types are restored. + * This allows cache implementations to participate in Guice dependency injection. + */ + this.typeSystem.setTypeCache(typeCache); + this.repository = repository; for (Provider<TypesChangeListener> provider : typeListenerProviders) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java index dd530a7..da47dc1 100755 --- a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java @@ -25,6 +25,7 @@ import com.thinkaurelius.titan.core.util.TitanCleanup; import com.tinkerpop.blueprints.Direction; import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Vertex; + import org.apache.atlas.AtlasException; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.TestUtils; @@ -53,6 +54,7 @@ import org.testng.annotations.Guice; import org.testng.annotations.Test; import javax.inject.Inject; + import java.util.Iterator; import java.util.List; import java.util.Map; @@ -64,6 +66,8 @@ import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructType @Guice(modules = RepositoryMetadataModule.class) public class GraphBackedTypeStoreTest { + private static final String DESCRIPTION = "_description"; + @Inject private GraphProvider<TitanGraph> graphProvider; @@ -97,6 +101,12 @@ public class GraphBackedTypeStoreTest { dumpGraph(); } + @Test(dependsOnMethods = "testStore") + public void testRestoreType() throws Exception { + TypesDef typesDef = ((GraphBackedTypeStore)typeStore).restoreType("Manager"); + verifyRestoredClassType(typesDef, "Manager"); + } + private void dumpGraph() { TitanGraph graph = graphProvider.get(); for (Vertex v : graph.getVertices()) { @@ -109,7 +119,6 @@ public class GraphBackedTypeStoreTest { @Test(dependsOnMethods = "testStore") public void testRestore() throws Exception { - String description = "_description"; TypesDef types = typeStore.restore(); //validate enum @@ -117,7 +126,7 @@ public class GraphBackedTypeStoreTest { Assert.assertEquals(1, enumTypes.size()); EnumTypeDefinition orgLevel = enumTypes.get(0); Assert.assertEquals(orgLevel.name, "OrgLevel"); - Assert.assertEquals(orgLevel.description, "OrgLevel"+description); + Assert.assertEquals(orgLevel.description, "OrgLevel"+DESCRIPTION); Assert.assertEquals(orgLevel.enumValues.length, 2); EnumValue enumValue = orgLevel.enumValues[0]; Assert.assertEquals(enumValue.value, "L1"); @@ -127,25 +136,14 @@ public class GraphBackedTypeStoreTest { List<StructTypeDefinition> structTypes = types.structTypesAsJavaList(); Assert.assertEquals(1, structTypes.size()); - boolean clsTypeFound = false; - List<HierarchicalTypeDefinition<ClassType>> classTypes = types.classTypesAsJavaList(); - for (HierarchicalTypeDefinition<ClassType> classType : classTypes) { - if (classType.typeName.equals("Manager")) { - ClassType expectedType = ts.getDataType(ClassType.class, classType.typeName); - Assert.assertEquals(expectedType.immediateAttrs.size(), classType.attributeDefinitions.length); - Assert.assertEquals(expectedType.superTypes.size(), classType.superTypes.size()); - Assert.assertEquals(classType.typeDescription, classType.typeName+description); - clsTypeFound = true; - } - } - Assert.assertTrue(clsTypeFound, "Manager type not restored"); + verifyRestoredClassType(types, "Manager"); //validate trait List<HierarchicalTypeDefinition<TraitType>> traitTypes = types.traitTypesAsJavaList(); Assert.assertEquals(1, traitTypes.size()); HierarchicalTypeDefinition<TraitType> trait = traitTypes.get(0); Assert.assertEquals("SecurityClearance", trait.typeName); - Assert.assertEquals(trait.typeName+description, trait.typeDescription); + Assert.assertEquals(trait.typeName+DESCRIPTION, trait.typeDescription); Assert.assertEquals(1, trait.attributeDefinitions.length); AttributeDefinition attribute = trait.attributeDefinitions[0]; Assert.assertEquals("level", attribute.name); @@ -229,4 +227,20 @@ public class GraphBackedTypeStoreTest { } return edgeCount; } + + private void verifyRestoredClassType(TypesDef types, String typeName) throws AtlasException { + boolean clsTypeFound = false; + List<HierarchicalTypeDefinition<ClassType>> classTypes = types.classTypesAsJavaList(); + for (HierarchicalTypeDefinition<ClassType> classType : classTypes) { + if (classType.typeName.equals(typeName)) { + ClassType expectedType = ts.getDataType(ClassType.class, classType.typeName); + Assert.assertEquals(expectedType.immediateAttrs.size(), classType.attributeDefinitions.length); + Assert.assertEquals(expectedType.superTypes.size(), classType.superTypes.size()); + Assert.assertEquals(classType.typeDescription, classType.typeName+DESCRIPTION); + clsTypeFound = true; + } + } + Assert.assertTrue(clsTypeFound, typeName + " type not restored"); + } + } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheConfigurationTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheConfigurationTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheConfigurationTest.java new file mode 100644 index 0000000..e62af85 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheConfigurationTest.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.typestore; + +import org.apache.atlas.typesystem.types.cache.TypeCache; +import org.testng.Assert; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import com.google.inject.Inject; + +/** + * Verify Guice can successfully instantiate and inject StoreBackTypeCache. + * StoreBackedTypeCacheTestModule Guice module uses Atlas configuration + * which has type cache implementation class set to {@link StoreBackedTypeCache}. + */ +@Guice(modules = StoreBackedTypeCacheTestModule.class) +public class StoreBackedTypeCacheConfigurationTest { + + @Inject + private TypeCache typeCache; + + @Test + public void testConfigureAsTypeCache() throws Exception { + // Verify Guice successfully instantiated and injected StoreBackTypeCache + Assert.assertTrue(typeCache instanceof StoreBackedTypeCache); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java new file mode 100644 index 0000000..ee4cc28 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java @@ -0,0 +1,212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.typestore; + +import java.util.HashMap; +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.RepositoryMetadataModule; +import org.apache.atlas.TestUtils; +import org.apache.atlas.repository.graph.GraphProvider; +import org.apache.atlas.typesystem.types.AttributeInfo; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; +import org.apache.atlas.typesystem.types.HierarchicalType; +import org.apache.atlas.typesystem.types.IDataType; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.TypeSystem; +import org.apache.atlas.typesystem.types.TypeUtils; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.util.TitanCleanup; + + +/** + * Unit test for {@link StoreBackedTypeCache} + */ +@Guice(modules = RepositoryMetadataModule.class) +public class StoreBackedTypeCacheTest { + + @Inject + private GraphProvider<TitanGraph> graphProvider; + + @Inject + private ITypeStore typeStore; + + @Inject + private StoreBackedTypeCache typeCache; + + private TypeSystem ts; + + private Map<String, ClassType> classTypesToTest = new HashMap<>(); + + @BeforeClass + public void setUp() throws Exception { + ts = TypeSystem.getInstance(); + ts.reset(); + + // Populate the type store for testing. + TestUtils.defineDeptEmployeeTypes(ts); + TestUtils.createHiveTypes(ts); + ImmutableList<String> typeNames = ts.getTypeNames(); + typeStore.store(ts, typeNames); + + ClassType type = ts.getDataType(ClassType.class, "Manager"); + classTypesToTest.put("Manager", type); + type = ts.getDataType(ClassType.class, TestUtils.TABLE_TYPE); + classTypesToTest.put(TestUtils.TABLE_TYPE, type); + } + + @AfterClass + public void tearDown() throws Exception { + ts.reset(); + try { + graphProvider.get().shutdown(); + } + catch(Exception e) { + e.printStackTrace(); + } + + try { + TitanCleanup.clear(graphProvider.get()); + } + catch(Exception e) { + e.printStackTrace(); + } + } + + @BeforeMethod + public void setupTestMethod() throws Exception { + typeCache.clear(); + } + + @Test + public void testGetClassType() throws Exception { + for (Map.Entry<String, ClassType> typeEntry : classTypesToTest.entrySet()) { + // Not cached yet + Assert.assertFalse(typeCache.isCachedInMemory(typeEntry.getKey())); + + IDataType dataType = typeCache.get(typeEntry.getKey()); + // Verify the type is now cached. + Assert.assertTrue(typeCache.isCachedInMemory(typeEntry.getKey())); + + Assert.assertTrue(dataType instanceof ClassType); + ClassType cachedType = (ClassType)dataType; + // Verify that get() also loaded and cached any dependencies of this type from the type store. + verifyHierarchicalType(cachedType, typeEntry.getValue()); + } + } + + @Test + public void testHasClassType() throws Exception { + for (Map.Entry<String, ClassType> typeEntry : classTypesToTest.entrySet()) { + // Not cached yet + Assert.assertFalse(typeCache.isCachedInMemory(typeEntry.getKey())); + + // Calling has() should result in type and its dependencies + // loaded from the type store and added to the cache. + Assert.assertTrue(typeCache.has(typeEntry.getKey())); + + // Verify the type is now cached in memory. + Assert.assertTrue(typeCache.isCachedInMemory(typeEntry.getKey())); + } + } + + @Test + public void testGetTraitType() throws Exception { + ImmutableList<String> traitNames = ts.getTypeNamesByCategory(TypeCategory.TRAIT); + for (String traitTypeName : traitNames) { + // Not cached yet + Assert.assertFalse(typeCache.isCachedInMemory(traitTypeName)); + + IDataType dataType = typeCache.get(traitTypeName); + // Verify the type is now cached. + Assert.assertTrue(typeCache.isCachedInMemory(traitTypeName)); + + Assert.assertTrue(dataType instanceof TraitType); + TraitType cachedType = (TraitType)dataType; + // Verify that get() also loaded and cached any dependencies of this type from the type store. + verifyHierarchicalType(cachedType, ts.getDataType(TraitType.class, traitTypeName)); + } + } + + @Test + public void testHasTraitType() throws Exception { + ImmutableList<String> traitNames = ts.getTypeNamesByCategory(TypeCategory.TRAIT); + for (String traitTypeName : traitNames) { + // Not cached yet + Assert.assertFalse(typeCache.isCachedInMemory(traitTypeName)); + + // Calling has() should result in type and its dependencies + // loaded from the type store and added to the cache. + Assert.assertTrue(typeCache.has(traitTypeName)); + + // Verify the type is now cached. + Assert.assertTrue(typeCache.isCachedInMemory(traitTypeName)); + } + } + + private <T extends HierarchicalType> void verifyHierarchicalType(T dataType, T expectedDataType) throws AtlasException { + Assert.assertEquals(dataType.numFields, expectedDataType.numFields); + Assert.assertEquals(dataType.immediateAttrs.size(), expectedDataType.immediateAttrs.size()); + Assert.assertEquals(dataType.fieldMapping().fields.size(), expectedDataType.fieldMapping().fields.size()); + ImmutableSet<String> superTypes = dataType.superTypes; + Assert.assertEquals(superTypes.size(), expectedDataType.superTypes.size()); + + // Verify that any attribute and super types were also cached. + for (String superTypeName : superTypes) { + Assert.assertTrue(typeCache.has(superTypeName)); + } + for (AttributeInfo attrInfo : dataType.fieldMapping().fields.values()) { + switch (attrInfo.dataType().getTypeCategory()) { + case CLASS: + case STRUCT: + case ENUM: + Assert.assertTrue(typeCache.has(attrInfo.dataType().getName()), attrInfo.dataType().getName() + " should be cached"); + break; + case ARRAY: + String elementTypeName = TypeUtils.parseAsArrayType(attrInfo.dataType().getName()); + if (!ts.getCoreTypes().contains(elementTypeName)) { + Assert.assertTrue(typeCache.has(elementTypeName), elementTypeName + " should be cached"); + } + break; + case MAP: + String[] mapTypeNames = TypeUtils.parseAsMapType(attrInfo.dataType().getName()); + for (String typeName : mapTypeNames) { + if (!ts.getCoreTypes().contains(typeName)) { + Assert.assertTrue(typeCache.has(typeName), typeName + " should be cached"); + } + } + break; + default: + break; + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTestModule.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTestModule.java b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTestModule.java new file mode 100644 index 0000000..058ed4d --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTestModule.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.typestore; + +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; +import org.apache.atlas.RepositoryMetadataModule; +import org.apache.commons.configuration.Configuration; + + +/** + * Guice module which sets TypeCache implementation class configuration property to {@link StoreBackedTypeCache}. + * + */ +public class StoreBackedTypeCacheTestModule extends RepositoryMetadataModule { + @Override + protected Configuration getConfiguration() { + try { + Configuration configuration = ApplicationProperties.get(); + configuration.setProperty(RepositoryMetadataModule.TYPE_CACHE_IMPLEMENTATION_PROPERTY, + StoreBackedTypeCache.class.getName()); + return configuration; + } catch (AtlasException e) { + throw new RuntimeException(e); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java new file mode 100644 index 0000000..4cb5c67 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.service; + +import org.apache.atlas.TestUtils; +import org.apache.atlas.repository.typestore.ITypeStore; +import org.apache.atlas.repository.typestore.StoreBackedTypeCache; +import org.apache.atlas.repository.typestore.StoreBackedTypeCacheTestModule; +import org.apache.atlas.services.MetadataService; +import org.apache.atlas.typesystem.types.TypeSystem; +import org.apache.atlas.typesystem.types.cache.TypeCache; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; + + +/** + * Verify MetadataService type lookup triggers StoreBackedTypeCache to load type from the store. + * StoreBackedTypeCacheTestModule Guice module uses Atlas configuration + * which has type cache implementation class set to {@link StoreBackedTypeCache}. + */ +@Guice(modules = StoreBackedTypeCacheTestModule.class) +public class StoreBackedTypeCacheMetadataServiceTest +{ + @Inject + private MetadataService metadataService; + + @Inject + private ITypeStore typeStore; + + @Inject + TypeCache typeCache; + + private TypeSystem ts; + + @BeforeClass + public void setUp() throws Exception { + ts = TypeSystem.getInstance(); + ts.reset(); + + // Populate the type store for testing. + TestUtils.defineDeptEmployeeTypes(ts); + TestUtils.createHiveTypes(ts); + ImmutableList<String> typeNames = ts.getTypeNames(); + typeStore.store(ts, typeNames); + ts.reset(); + } + + @Test + public void testIt() throws Exception { + Assert.assertTrue(typeCache instanceof StoreBackedTypeCache); + StoreBackedTypeCache storeBackedCache = (StoreBackedTypeCache) typeCache; + + // Cache should be empty + Assert.assertFalse(storeBackedCache.isCachedInMemory("Manager")); + + // Type lookup on MetadataService should cause Manager type to be loaded from the type store + // and cached. + Assert.assertNotNull(metadataService.getTypeDefinition("Manager")); + Assert.assertTrue(storeBackedCache.isCachedInMemory("Manager")); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java b/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java index effee2a..393b539 100644 --- a/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java +++ b/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java @@ -19,6 +19,7 @@ package org.apache.atlas.services; import com.google.inject.Provider; + import org.apache.atlas.AtlasException; import org.apache.atlas.listener.EntityChangeListener; import org.apache.atlas.listener.TypesChangeListener; @@ -26,12 +27,8 @@ import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.typestore.ITypeStore; import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.ha.HAConfiguration; -import org.apache.atlas.listener.TypesChangeListener; -import org.apache.atlas.repository.MetadataRepository; -import org.apache.atlas.repository.typestore.ITypeStore; import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.types.IDataType; -import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.commons.configuration.Configuration; import org.mockito.Matchers; import org.mockito.Mock; @@ -44,6 +41,7 @@ import java.util.HashMap; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyZeroInteractions; @@ -78,7 +76,7 @@ public class DefaultMetadataServiceMockTest { DefaultMetadataService defaultMetadataService = new DefaultMetadataService(mock(MetadataRepository.class), mock(ITypeStore.class), typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(), - new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration); + new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null); verify(typesRegistrar).registerTypes(ReservedTypesRegistrar.getTypesDir(), typeSystem, defaultMetadataService); @@ -91,10 +89,10 @@ public class DefaultMetadataServiceMockTest { DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository, typeStore, typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(), - new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration); + new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null); verifyZeroInteractions(typeStore); - verifyZeroInteractions(typeSystem); + verify(typeSystem, never()).defineTypes(Matchers.<TypesDef>any()); verifyZeroInteractions(typesRegistrar); } @@ -109,7 +107,7 @@ public class DefaultMetadataServiceMockTest { DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository, typeStore, typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(), - new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration); + new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null); defaultMetadataService.instanceIsActive(); verify(typeStore).restore(); @@ -134,7 +132,7 @@ public class DefaultMetadataServiceMockTest { DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository, typeStore, typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(), - new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration); + new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null); defaultMetadataService.instanceIsActive(); defaultMetadataService.instanceIsPassive(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java index 14f1968..90b5815 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java @@ -30,14 +30,13 @@ import java.util.concurrent.ConcurrentHashMap; import javax.inject.Singleton; -import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; import org.apache.atlas.classification.InterfaceAudience; import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.exception.TypeExistsException; import org.apache.atlas.typesystem.exception.TypeNotFoundException; -import org.apache.atlas.typesystem.types.cache.DefaultTypeCacheProvider; -import org.apache.atlas.typesystem.types.cache.ITypeCacheProvider; +import org.apache.atlas.typesystem.types.cache.DefaultTypeCache; +import org.apache.atlas.typesystem.types.cache.TypeCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +47,6 @@ import com.google.common.collect.ImmutableSet; @InterfaceAudience.Private public class TypeSystem { private static final Logger LOG = LoggerFactory.getLogger(TypeSystem.class); - private static final String CACHE_PROVIDER_CLASS_PROPERTY = "atlas.typesystem.cache.provider"; private static final TypeSystem INSTANCE = new TypeSystem(); private static ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal<SimpleDateFormat>() { @@ -60,7 +58,7 @@ public class TypeSystem { } }; - private ITypeCacheProvider typeCache; + private TypeCache typeCache = new DefaultTypeCache(); private IdType idType; private Map<String, IDataType> coreTypes; @@ -84,44 +82,18 @@ public class TypeSystem { return this; } + public void setTypeCache(TypeCache typeCache) { + this.typeCache = typeCache; + } + private void initialize() { - initCacheProvider(); coreTypes = new ConcurrentHashMap<>(); registerPrimitiveTypes(); registerCoreTypes(); } - /** - * Ideally a cache provider should have been injected in the TypeSystemProvider, - * but a singleton of TypeSystem is constructed privately within the class so that - * clients of TypeSystem would never instantiate a TypeSystem object directly in - * their code. As soon as a client makes a call to TypeSystem.getInstance(), they - * should have the singleton ready for consumption. To enable such an access pattern, - * it kind of becomes imperative to initialize the cache provider within the - * TypeSystem constructor (bypassing the GUICE way of injecting a cache provider) - */ - private void initCacheProvider() { - - // read the pluggable cache provider from Atlas configuration - final String defaultCacheProvider = DefaultTypeCacheProvider.class.getName(); - Class cacheProviderClass; - try { - cacheProviderClass = ApplicationProperties.getClass(CACHE_PROVIDER_CLASS_PROPERTY, - defaultCacheProvider, ITypeCacheProvider.class); - } catch (AtlasException e) { - throw new RuntimeException("Error getting type cache provider implementation class", e); - } - - try { - typeCache = (ITypeCacheProvider)cacheProviderClass.newInstance(); - } - catch (Exception e) { - throw new RuntimeException("Error creating instance of type cache provider implementation class " + cacheProviderClass.getName(), e); - } - } - public ImmutableList<String> getCoreTypes() { return ImmutableList.copyOf(coreTypes.keySet()); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java new file mode 100644 index 0000000..f683ed0 --- /dev/null +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java @@ -0,0 +1,229 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.typesystem.types.cache; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; +import org.apache.atlas.typesystem.types.EnumType; +import org.apache.atlas.typesystem.types.IDataType; +import org.apache.atlas.typesystem.types.StructType; +import org.apache.atlas.typesystem.types.TraitType; + +import com.google.inject.Singleton; + +/** + * Caches the types in-memory within the same process space. + */ +@Singleton +@SuppressWarnings("rawtypes") +public class DefaultTypeCache implements TypeCache { + + private Map<String, IDataType> types_ = new ConcurrentHashMap<>(); + + /* + * (non-Javadoc) + * @see + * org.apache.atlas.typesystem.types.cache.TypeCache#has(java.lang + * .String) + */ + @Override + public boolean has(String typeName) throws AtlasException { + + return types_.containsKey(typeName); + } + + /* (non-Javadoc) + * @see org.apache.atlas.typesystem.types.cache.TypeCache#has(org. + * apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String) + */ + @Override + public boolean has(TypeCategory typeCategory, String typeName) + throws AtlasException { + + assertValidTypeCategory(typeCategory); + return has(typeName); + } + + private void assertValidTypeCategory(TypeCategory typeCategory) throws + AtlasException { + + // there might no need of 'typeCategory' in this implementation for + // certain API, but for a distributed cache, it might help for the + // implementers to partition the types per their category + // while persisting so that look can be efficient + + if (typeCategory == null) { + throw new AtlasException("Category of the types to be filtered is null."); + } + + boolean validTypeCategory = typeCategory.equals(TypeCategory.CLASS) || + typeCategory.equals(TypeCategory.TRAIT) || + typeCategory.equals(TypeCategory.ENUM) || + typeCategory.equals(TypeCategory.STRUCT); + + if (!validTypeCategory) { + throw new AtlasException("Category of the types should be one of CLASS " + + "| TRAIT | ENUM | STRUCT."); + } + } + + /* + * (non-Javadoc) + * @see + * org.apache.atlas.typesystem.types.cache.TypeCache#get(java.lang + * .String) + */ + @Override + public IDataType get(String typeName) throws AtlasException { + + return types_.get(typeName); + } + + /* (non-Javadoc) + * @see org.apache.atlas.typesystem.types.cache.TypeCache#get(org.apache. + * atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String) + */ + @Override + public IDataType get(TypeCategory typeCategory, String typeName) throws AtlasException { + + assertValidTypeCategory(typeCategory); + return get(typeName); + } + + /* + * (non-Javadoc) + * @see + * org.apache.atlas.typesystem.types.cache.TypeCache#getNames(org + * .apache.atlas.typesystem.types.DataTypes.TypeCategory) + */ + @Override + public Collection<String> getTypeNames(TypeCategory typeCategory) throws AtlasException { + + assertValidTypeCategory(typeCategory); + + List<String> typeNames = new ArrayList<>(); + for (Entry<String, IDataType> typeEntry : types_.entrySet()) { + String name = typeEntry.getKey(); + IDataType type = typeEntry.getValue(); + + if (type.getTypeCategory().equals(typeCategory)) { + typeNames.add(name); + } + } + return typeNames; + } + + /* + * (non-Javadoc) + * @see + * org.apache.atlas.typesystem.types.cache.TypeCache#getAllNames() + */ + @Override + public Collection<String> getAllTypeNames() throws AtlasException { + + return types_.keySet(); + } + + /* + * (non-Javadoc) + * @see + * org.apache.atlas.typesystem.types.cache.TypeCache#put(org.apache + * .atlas.typesystem.types.IDataType) + */ + @Override + public void put(IDataType type) throws AtlasException { + + assertValidType(type); + types_.put(type.getName(), type); + } + + private void assertValidType(IDataType type) throws + AtlasException { + + if (type == null) { + throw new AtlasException("type is null."); + } + + boolean validTypeCategory = (type instanceof ClassType) || + (type instanceof TraitType) || + (type instanceof EnumType) || + (type instanceof StructType); + + if (!validTypeCategory) { + throw new AtlasException("Category of the types should be one of ClassType | " + + "TraitType | EnumType | StructType."); + } + } + + /* + * (non-Javadoc) + * @see + * org.apache.atlas.typesystem.types.cache.TypeCache#putAll(java + * .util.Collection) + */ + @Override + public void putAll(Collection<IDataType> types) throws AtlasException { + + for (IDataType type : types) { + assertValidType(type); + types_.put(type.getName(), type); + } + } + + /* + * (non-Javadoc) + * @see + * org.apache.atlas.typesystem.types.cache.TypeCache#remove(java + * .lang.String) + */ + @Override + public void remove(String typeName) throws AtlasException { + + types_.remove(typeName); + } + + /* (non-Javadoc) + * @see org.apache.atlas.typesystem.types.cache.TypeCache#remove(org. + * apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String) + */ + @Override + public void remove(TypeCategory typeCategory, String typeName) + throws AtlasException { + + assertValidTypeCategory(typeCategory); + remove(typeName); + } + + /* + * (non-Javadoc) + * @see org.apache.atlas.typesystem.types.cache.TypeCache#clear() + */ + @Override + public void clear() { + + types_.clear(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProvider.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProvider.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProvider.java deleted file mode 100644 index 700bda6..0000000 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProvider.java +++ /dev/null @@ -1,229 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.typesystem.types.cache; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; -import org.apache.atlas.typesystem.types.EnumType; -import org.apache.atlas.typesystem.types.IDataType; -import org.apache.atlas.typesystem.types.StructType; -import org.apache.atlas.typesystem.types.TraitType; - -import com.google.inject.Singleton; - -/** - * Caches the types in-memory within the same process space. - */ -@Singleton -@SuppressWarnings("rawtypes") -public class DefaultTypeCacheProvider implements ITypeCacheProvider { - - private Map<String, IDataType> types_ = new ConcurrentHashMap<>(); - - /* - * (non-Javadoc) - * @see - * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#has(java.lang - * .String) - */ - @Override - public boolean has(String typeName) throws AtlasException { - - return types_.containsKey(typeName); - } - - /* (non-Javadoc) - * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#has(org. - * apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String) - */ - @Override - public boolean has(TypeCategory typeCategory, String typeName) - throws AtlasException { - - assertValidTypeCategory(typeCategory); - return has(typeName); - } - - private void assertValidTypeCategory(TypeCategory typeCategory) throws - AtlasException { - - // there might no need of 'typeCategory' in this implementation for - // certain API, but for a distributed cache, it might help for the - // implementers to partition the types per their category - // while persisting so that look can be efficient - - if (typeCategory == null) { - throw new AtlasException("Category of the types to be filtered is null."); - } - - boolean validTypeCategory = typeCategory.equals(TypeCategory.CLASS) || - typeCategory.equals(TypeCategory.TRAIT) || - typeCategory.equals(TypeCategory.ENUM) || - typeCategory.equals(TypeCategory.STRUCT); - - if (!validTypeCategory) { - throw new AtlasException("Category of the types should be one of CLASS " - + "| TRAIT | ENUM | STRUCT."); - } - } - - /* - * (non-Javadoc) - * @see - * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#get(java.lang - * .String) - */ - @Override - public IDataType get(String typeName) throws AtlasException { - - return types_.get(typeName); - } - - /* (non-Javadoc) - * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#get(org.apache. - * atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String) - */ - @Override - public IDataType get(TypeCategory typeCategory, String typeName) throws AtlasException { - - assertValidTypeCategory(typeCategory); - return get(typeName); - } - - /* - * (non-Javadoc) - * @see - * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#getNames(org - * .apache.atlas.typesystem.types.DataTypes.TypeCategory) - */ - @Override - public Collection<String> getTypeNames(TypeCategory typeCategory) throws AtlasException { - - assertValidTypeCategory(typeCategory); - - List<String> typeNames = new ArrayList<>(); - for (Entry<String, IDataType> typeEntry : types_.entrySet()) { - String name = typeEntry.getKey(); - IDataType type = typeEntry.getValue(); - - if (type.getTypeCategory().equals(typeCategory)) { - typeNames.add(name); - } - } - return typeNames; - } - - /* - * (non-Javadoc) - * @see - * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#getAllNames() - */ - @Override - public Collection<String> getAllTypeNames() throws AtlasException { - - return types_.keySet(); - } - - /* - * (non-Javadoc) - * @see - * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#put(org.apache - * .atlas.typesystem.types.IDataType) - */ - @Override - public void put(IDataType type) throws AtlasException { - - assertValidType(type); - types_.put(type.getName(), type); - } - - private void assertValidType(IDataType type) throws - AtlasException { - - if (type == null) { - throw new AtlasException("type is null."); - } - - boolean validTypeCategory = (type instanceof ClassType) || - (type instanceof TraitType) || - (type instanceof EnumType) || - (type instanceof StructType); - - if (!validTypeCategory) { - throw new AtlasException("Category of the types should be one of ClassType | " - + "TraitType | EnumType | StructType."); - } - } - - /* - * (non-Javadoc) - * @see - * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#putAll(java - * .util.Collection) - */ - @Override - public void putAll(Collection<IDataType> types) throws AtlasException { - - for (IDataType type : types) { - assertValidType(type); - types_.put(type.getName(), type); - } - } - - /* - * (non-Javadoc) - * @see - * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#remove(java - * .lang.String) - */ - @Override - public void remove(String typeName) throws AtlasException { - - types_.remove(typeName); - } - - /* (non-Javadoc) - * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#remove(org. - * apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String) - */ - @Override - public void remove(TypeCategory typeCategory, String typeName) - throws AtlasException { - - assertValidTypeCategory(typeCategory); - remove(typeName); - } - - /* - * (non-Javadoc) - * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#clear() - */ - @Override - public void clear() { - - types_.clear(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/ITypeCacheProvider.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/ITypeCacheProvider.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/ITypeCacheProvider.java deleted file mode 100644 index 55e9ddc..0000000 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/ITypeCacheProvider.java +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.typesystem.types.cache; - -import java.util.Collection; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.IDataType; - -/** - * The types are cached to allow faster lookup when type info is needed during - * creation/updation of entities, DSL query translation/execution. - * Implementations of this can chose to plugin a distributed cache provider - * or an in-memory cache synched across nodes in an Altas cluster. <br> - * <br> - * Type entries in the cache can be one of ... <br> - * {@link org.apache.atlas.typesystem.types.ClassType} <br> - * {@link org.apache.atlas.typesystem.types.TraitType} <br> - * {@link org.apache.atlas.typesystem.types.StructType} <br> - * {@link org.apache.atlas.typesystem.types.EnumType} - */ -@SuppressWarnings("rawtypes") -public interface ITypeCacheProvider { - - /** - * @param typeName - * @return true if the type exists in cache, false otherwise. - * @throws AtlasException - */ - boolean has(String typeName) throws AtlasException; - - /** - * @param typeCategory Non-null category of type. The category can be one of - * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM. - * @param typeName - * @return true if the type of given category exists in cache, false otherwise. - * @throws AtlasException - */ - boolean has(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException; - - /** - * @param name The name of the type. - * @return returns non-null type if cached, otherwise null - * @throws AtlasException - */ - public IDataType get(String typeName) throws AtlasException; - - /** - * @param typeCategory Non-null category of type. The category can be one of - * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM. - * @param typeName - * @return returns non-null type (of the specified category) if cached, otherwise null - * @throws AtlasException - */ - public IDataType get(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException; - - /** - * @param typeCategory The category of types to filter the returned types. Cannot be null. - * The category can be one of TypeCategory.CLASS | TypeCategory.TRAIT | - * TypeCategory.STRUCT | TypeCategory.ENUM. - * @return - * @throws AtlasException - */ - Collection<String> getTypeNames(DataTypes.TypeCategory typeCategory) throws AtlasException; - - /** - * This is a convenience API to get the names of all types. - * - * @see ITypeCacheProvider#getTypeNames(org.apache.atlas.typesystem.types.DataTypes.TypeCategory) - * @return - * @throws AtlasException - */ - Collection<String> getAllTypeNames() throws AtlasException; - - /** - * @param type The type to be added to the cache. The type should not be - * null, otherwise throws NullPointerException. <br> - * Type entries in the cache can be one of ... <br> - * {@link org.apache.atlas.typesystem.types.ClassType} <br> - * {@link org.apache.atlas.typesystem.types.TraitType} <br> - * {@link org.apache.atlas.typesystem.types.StructType} <br> - * {@link org.apache.atlas.typesystem.types.EnumType} - * @throws AtlasException - */ - void put(IDataType type) throws AtlasException; - - /** - * @param types The types to be added to the cache. The type should not be - * null, otherwise throws NullPointerException. <br> - * Type entries in the cache can be one of ... <br> - * {@link org.apache.atlas.typesystem.types.ClassType} <br> - * {@link org.apache.atlas.typesystem.types.TraitType} <br> - * {@link org.apache.atlas.typesystem.types.StructType} <br> - * {@link org.apache.atlas.typesystem.types.EnumType} - * @throws AtlasException - */ - void putAll(Collection<IDataType> types) throws AtlasException; - - /** - * @param typeName Name of the type to be removed from the cache. If type - * exists, it will be removed, otherwise does nothing. - * @throws AtlasException - */ - void remove(String typeName) throws AtlasException; - - /** - * @param typeCategory Non-null category of type. The category can be one of - * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM. - * @param typeName Name of the type to be removed from the cache. If type - * exists, it will be removed, otherwise does nothing. - * @throws AtlasException - */ - void remove(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException; - - /** - * Clear the type cache - * - */ - void clear(); -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java new file mode 100644 index 0000000..27622c2 --- /dev/null +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.typesystem.types.cache; + +import java.util.Collection; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.IDataType; + +/** + * The types are cached to allow faster lookup when type info is needed during + * creation/updation of entities, DSL query translation/execution. + * Implementations of this can chose to plugin a distributed cache + * or an in-memory cache synched across nodes in an Altas cluster. <br> + * <br> + * Type entries in the cache can be one of ... <br> + * {@link org.apache.atlas.typesystem.types.ClassType} <br> + * {@link org.apache.atlas.typesystem.types.TraitType} <br> + * {@link org.apache.atlas.typesystem.types.StructType} <br> + * {@link org.apache.atlas.typesystem.types.EnumType} + */ +@SuppressWarnings("rawtypes") +public interface TypeCache { + + /** + * @param typeName + * @return true if the type exists in cache, false otherwise. + * @throws AtlasException + */ + boolean has(String typeName) throws AtlasException; + + /** + * @param typeCategory Non-null category of type. The category can be one of + * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM. + * @param typeName + * @return true if the type of given category exists in cache, false otherwise. + * @throws AtlasException + */ + boolean has(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException; + + /** + * @param name The name of the type. + * @return returns non-null type if cached, otherwise null + * @throws AtlasException + */ + public IDataType get(String typeName) throws AtlasException; + + /** + * @param typeCategory Non-null category of type. The category can be one of + * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM. + * @param typeName + * @return returns non-null type (of the specified category) if cached, otherwise null + * @throws AtlasException + */ + public IDataType get(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException; + + /** + * @param typeCategory The category of types to filter the returned types. Cannot be null. + * The category can be one of TypeCategory.CLASS | TypeCategory.TRAIT | + * TypeCategory.STRUCT | TypeCategory.ENUM. + * @return + * @throws AtlasException + */ + Collection<String> getTypeNames(DataTypes.TypeCategory typeCategory) throws AtlasException; + + /** + * This is a convenience API to get the names of all types. + * + * @see TypeCache#getTypeNames(org.apache.atlas.typesystem.types.DataTypes.TypeCategory) + * @return + * @throws AtlasException + */ + Collection<String> getAllTypeNames() throws AtlasException; + + /** + * @param type The type to be added to the cache. The type should not be + * null, otherwise throws NullPointerException. <br> + * Type entries in the cache can be one of ... <br> + * {@link org.apache.atlas.typesystem.types.ClassType} <br> + * {@link org.apache.atlas.typesystem.types.TraitType} <br> + * {@link org.apache.atlas.typesystem.types.StructType} <br> + * {@link org.apache.atlas.typesystem.types.EnumType} + * @throws AtlasException + */ + void put(IDataType type) throws AtlasException; + + /** + * @param types The types to be added to the cache. The type should not be + * null, otherwise throws NullPointerException. <br> + * Type entries in the cache can be one of ... <br> + * {@link org.apache.atlas.typesystem.types.ClassType} <br> + * {@link org.apache.atlas.typesystem.types.TraitType} <br> + * {@link org.apache.atlas.typesystem.types.StructType} <br> + * {@link org.apache.atlas.typesystem.types.EnumType} + * @throws AtlasException + */ + void putAll(Collection<IDataType> types) throws AtlasException; + + /** + * @param typeName Name of the type to be removed from the cache. If type + * exists, it will be removed, otherwise does nothing. + * @throws AtlasException + */ + void remove(String typeName) throws AtlasException; + + /** + * @param typeCategory Non-null category of type. The category can be one of + * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM. + * @param typeName Name of the type to be removed from the cache. If type + * exists, it will be removed, otherwise does nothing. + * @throws AtlasException + */ + void remove(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException; + + /** + * Clear the type cache + * + */ + void clear(); +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/9b00a9dd/typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java ---------------------------------------------------------------------- diff --git a/typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java b/typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java index 0679642..6d43359 100644 --- a/typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java +++ b/typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java @@ -58,17 +58,22 @@ public class ApplicationPropertiesTest { @Test public void testGetClass() throws Exception { + Configuration configuration = ApplicationProperties.get(); + //read from atlas-application.properties - Class cls = ApplicationProperties.getClass("atlas.TypeSystem.impl", ApplicationProperties.class.getName(), TypeSystem.class); + Class cls = ApplicationProperties.getClass(configuration, "atlas.TypeSystem.impl", + ApplicationProperties.class.getName(), TypeSystem.class); assertEquals(cls.getName(), TypeSystem.class.getName()); //default value - cls = ApplicationProperties.getClass("atlas.TypeSystem2.impl", TypeSystem.class.getName(), TypeSystem.class); + cls = ApplicationProperties.getClass(configuration, "atlas.TypeSystem2.impl", + TypeSystem.class.getName(), TypeSystem.class); assertEquals(cls.getName(), TypeSystem.class.getName()); //incompatible assignTo class, should throw AtlasException try { - cls = ApplicationProperties.getClass("atlas.TypeSystem.impl", ApplicationProperties.class.getName(), ApplicationProperties.class); + cls = ApplicationProperties.getClass(configuration, "atlas.TypeSystem.impl", + ApplicationProperties.class.getName(), ApplicationProperties.class); Assert.fail(AtlasException.class.getSimpleName() + " was expected but none thrown."); } catch (AtlasException e) {
