http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/notification/src/main/java/org/apache/atlas/notification/NotificationInterface.java ---------------------------------------------------------------------- diff --git a/notification/src/main/java/org/apache/atlas/notification/NotificationInterface.java b/notification/src/main/java/org/apache/atlas/notification/NotificationInterface.java index 0951124..d4be07b 100644 --- a/notification/src/main/java/org/apache/atlas/notification/NotificationInterface.java +++ b/notification/src/main/java/org/apache/atlas/notification/NotificationInterface.java @@ -20,7 +20,6 @@ package org.apache.atlas.notification; import org.apache.atlas.AtlasException; import org.apache.commons.configuration.Configuration; -import java.io.IOException; import java.util.List; public abstract class NotificationInterface { @@ -33,26 +32,11 @@ public abstract class NotificationInterface { HOOK, ENTITIES, TYPES } - /** - * Initialise - * @param applicationProperties - * @throws AtlasException - */ - public void initialize(Configuration applicationProperties) throws AtlasException { + public NotificationInterface(Configuration applicationProperties) throws AtlasException { this.embedded = applicationProperties.getBoolean(PROPERTY_EMBEDDED, false); } /** - * Start embedded notification service on atlast server - * @throws IOException - */ - public final void startService() throws IOException { - if (embedded) { - _startService(); - } - } - - /** * Is the notification service embedded in atlas server * @return */ @@ -60,18 +44,9 @@ public abstract class NotificationInterface { return embedded; } - protected abstract void _startService() throws IOException; - - /** - * Shutdown - close all the connections - */ - public final void shutdown() { - _shutdown(); - } - - protected abstract void _shutdown(); - public abstract List<NotificationConsumer> createConsumers(NotificationType type, int numConsumers); public abstract void send(NotificationType type, String... messages) throws NotificationException; + + public abstract void close(); }
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/notification/src/main/java/org/apache/atlas/notification/NotificationModule.java ---------------------------------------------------------------------- diff --git a/notification/src/main/java/org/apache/atlas/notification/NotificationModule.java b/notification/src/main/java/org/apache/atlas/notification/NotificationModule.java index db17e35..c73f9cb 100644 --- a/notification/src/main/java/org/apache/atlas/notification/NotificationModule.java +++ b/notification/src/main/java/org/apache/atlas/notification/NotificationModule.java @@ -18,11 +18,21 @@ package org.apache.atlas.notification; import com.google.inject.AbstractModule; +import com.google.inject.Singleton; +import com.google.inject.multibindings.Multibinder; import org.apache.atlas.kafka.KafkaNotification; +import org.apache.atlas.kafka.KafkaNotificationProvider; +import org.apache.atlas.service.Service; public class NotificationModule extends AbstractModule { + @Override protected void configure() { - bind(NotificationInterface.class).to(KafkaNotification.class).asEagerSingleton(); + bind(NotificationInterface.class).to(KafkaNotification.class).in(Singleton.class); + bind(KafkaNotification.class).toProvider(KafkaNotificationProvider.class).in(Singleton.class); + + Multibinder<Service> serviceBinder = Multibinder.newSetBinder(binder(), Service.class); + serviceBinder.addBinding().to(KafkaNotification.class); + serviceBinder.addBinding().to(NotificationHookConsumer.class); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationTest.java ---------------------------------------------------------------------- diff --git a/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationTest.java b/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationTest.java index 02752dc..735655c 100644 --- a/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationTest.java +++ b/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationTest.java @@ -18,12 +18,10 @@ package org.apache.atlas.kafka; import com.google.inject.Inject; -import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; import org.apache.atlas.notification.NotificationConsumer; import org.apache.atlas.notification.NotificationInterface; import org.apache.atlas.notification.NotificationModule; -import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.RandomStringUtils; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -35,18 +33,15 @@ import org.testng.annotations.Test; public class KafkaNotificationTest { @Inject - private NotificationInterface kafka; + private KafkaNotification kafka; @BeforeClass public void setUp() throws Exception { - Configuration conf = ApplicationProperties.get(); - conf.setProperty(KafkaNotification.PROPERTY_PREFIX + ".data", "target/data/kafka" + random()); - kafka.initialize(conf); - kafka.startService(); + kafka.start(); } @Test - public void testSendMessage() throws AtlasException { + public void testSendReceiveMessage() throws AtlasException { String msg1 = "message" + random(); String msg2 = "message" + random(); kafka.send(NotificationInterface.NotificationType.HOOK, msg1, msg2); @@ -63,6 +58,6 @@ public class KafkaNotificationTest { @AfterClass public void teardown() throws Exception { - kafka.shutdown(); + kafka.stop(); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index e8d18a2..e29736a 100755 --- a/pom.xml +++ b/pom.xml @@ -392,6 +392,7 @@ </profile> </profiles> <modules> + <module>common</module> <module>typesystem</module> <module>notification</module> <module>client</module> @@ -972,6 +973,12 @@ <dependency> <groupId>org.apache.atlas</groupId> + <artifactId>atlas-common</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.atlas</groupId> <artifactId>atlas-webapp</artifactId> <version>${project.version}</version> <type>war</type> @@ -1298,7 +1305,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-site-plugin</artifactId> - <version>3.2</version> + <version>3.3</version> </plugin> <plugin> @@ -1348,13 +1355,6 @@ <goal>testCompile</goal> </goals> </execution> - <execution> - <id>attach-scaladocs</id> - <phase>verify</phase> - <goals> - <goal>doc-jar</goal> - </goals> - </execution> </executions> <configuration> <scalaVersion>${scala.version}</scalaVersion> @@ -1428,6 +1428,9 @@ -Xmx1024m -XX:MaxPermSize=512m -Dlog4j.configuration=atlas-log4j.xml -Djava.net.preferIPv4Stack=true </argLine> + <excludes> + <exclude>**/*Base*</exclude> + </excludes> </configuration> <dependencies> <dependency> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 9be6e3d..767bd20 100644 --- a/release-log.txt +++ b/release-log.txt @@ -4,6 +4,7 @@ Apache Atlas Release Notes --trunk - unreleased INCOMPATIBLE CHANGES: +ATLAS-58 Make hive hook reliable (shwethags) ATLAS-54 Rename configs in hive hook (shwethags) ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java b/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java index 3d47293..b9689f4 100644 --- a/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java +++ b/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java @@ -32,6 +32,7 @@ public class GraphTransactionInterceptor implements MethodInterceptor { @Inject GraphProvider<TitanGraph> graphProvider; + @Override public Object invoke(MethodInvocation invocation) throws Throwable { if (titanGraph == null) { titanGraph = graphProvider.get(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java index fbd01de..34f2ba3 100755 --- a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java +++ b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java @@ -18,11 +18,11 @@ package org.apache.atlas; +import com.google.inject.Singleton; import com.google.inject.matcher.Matchers; import com.google.inject.multibindings.Multibinder; import com.google.inject.throwingproviders.ThrowingProviderBinder; import com.thinkaurelius.titan.core.TitanGraph; -import com.tinkerpop.blueprints.Graph; import org.aopalliance.intercept.MethodInterceptor; import org.apache.atlas.discovery.DiscoveryService; import org.apache.atlas.discovery.HiveLineageService; @@ -38,6 +38,7 @@ import org.apache.atlas.repository.typestore.GraphBackedTypeStore; import org.apache.atlas.repository.typestore.ITypeStore; import org.apache.atlas.services.DefaultMetadataService; import org.apache.atlas.services.MetadataService; +import org.apache.atlas.typesystem.types.TypeSystem; /** * Guice module for Repository module. @@ -55,6 +56,8 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { // bind the MetadataRepositoryService interface to an implementation bind(MetadataRepository.class).to(GraphBackedMetadataRepository.class).asEagerSingleton(); + bind(TypeSystem.class).in(Singleton.class); + // bind the ITypeStore interface to an implementation bind(ITypeStore.class).to(GraphBackedTypeStore.class).asEagerSingleton(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/repository/src/main/java/org/apache/atlas/listener/EntityChangeListener.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/listener/EntityChangeListener.java b/repository/src/main/java/org/apache/atlas/listener/EntityChangeListener.java index f58d6de..6465e92 100755 --- a/repository/src/main/java/org/apache/atlas/listener/EntityChangeListener.java +++ b/repository/src/main/java/org/apache/atlas/listener/EntityChangeListener.java @@ -21,6 +21,8 @@ package org.apache.atlas.listener; import org.apache.atlas.AtlasException; import org.apache.atlas.typesystem.ITypedReferenceableInstance; +import java.util.Collection; + /** * Entity (a Typed instance) change notification listener. */ @@ -29,10 +31,10 @@ public interface EntityChangeListener { /** * This is upon adding a new typed instance to the repository. * - * @param typedInstance a typed instance + * @param typedInstances a typed instance * @throws AtlasException */ - void onEntityAdded(ITypedReferenceableInstance typedInstance) throws AtlasException; + void onEntityAdded(Collection<ITypedReferenceableInstance> typedInstances) throws AtlasException; /** * This is upon adding a new trait to a typed instance. http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/repository/src/main/java/org/apache/atlas/repository/EntityExistsException.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/EntityExistsException.java b/repository/src/main/java/org/apache/atlas/repository/EntityExistsException.java new file mode 100644 index 0000000..7ea7e41 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/EntityExistsException.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.typesystem.IReferenceableInstance; + +public class EntityExistsException extends AtlasException { + public EntityExistsException(IReferenceableInstance typedInstance, Exception e) { + super("Model violation for type "+ typedInstance.getTypeName(), e); + } + + public EntityExistsException(IReferenceableInstance typedInstance) { + super("Model violation for type "+ typedInstance.getTypeName()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java index 1b0faa5..97d1fe2 100755 --- a/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java @@ -19,7 +19,6 @@ package org.apache.atlas.repository; import org.apache.atlas.AtlasException; -import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedStruct; import org.apache.atlas.typesystem.types.AttributeInfo; @@ -76,11 +75,12 @@ public interface MetadataRepository { /** * Creates an entity definition (instance) corresponding to a given type. * - * @param entity entity (typed instance) + * @param entities entity (typed instance) * @return a globally unique identifier * @throws RepositoryException + * @throws EntityExistsException */ - String createEntity(IReferenceableInstance entity) throws RepositoryException; + String[] createEntities(ITypedReferenceableInstance... entities) throws RepositoryException, EntityExistsException; /** * Fetch the complete definition of an entity given its GUID. @@ -158,4 +158,13 @@ public interface MetadataRepository { * @param value property value */ void updateEntity(String guid, String property, String value) throws RepositoryException; + + /** + * Returns the entity for the given type and qualified name + * @param entityType + * @param attribute + * @param value + * @return entity instance + */ + ITypedReferenceableInstance getEntityDefinition(String entityType, String attribute, String value) throws AtlasException; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java index cc10ea3..b44ce60 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java @@ -19,6 +19,8 @@ package org.apache.atlas.repository.graph; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.thinkaurelius.titan.core.SchemaViolationException; import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.TitanProperty; import com.thinkaurelius.titan.core.TitanVertex; @@ -29,6 +31,7 @@ import com.tinkerpop.blueprints.Vertex; import org.apache.atlas.AtlasException; import org.apache.atlas.GraphTransaction; import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.EntityExistsException; import org.apache.atlas.repository.EntityNotFoundException; import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.RepositoryException; @@ -37,7 +40,6 @@ import org.apache.atlas.typesystem.ITypedInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedStruct; import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.typesystem.persistence.MapIds; import org.apache.atlas.typesystem.types.AttributeInfo; import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.DataTypes; @@ -62,9 +64,9 @@ import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; /** * An implementation backed by a Graph database provided @@ -78,8 +80,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository { private static final String EDGE_LABEL_PREFIX = "__"; - private final AtomicInteger ID_SEQ = new AtomicInteger(0); - private final TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(); private final GraphToTypedInstanceMapper graphToInstanceMapper = new GraphToTypedInstanceMapper(); @@ -89,7 +89,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository { @Inject public GraphBackedMetadataRepository(GraphProvider<TitanGraph> graphProvider) throws AtlasException { this.typeSystem = TypeSystem.getInstance(); - this.titanGraph = graphProvider.get(); } @@ -126,6 +125,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository { return getQualifiedName(dataType, aInfo.name); } + public String getFieldNameInVertex(IDataType<?> dataType, String attrName) throws AtlasException { + return getQualifiedName(dataType, attrName); + } @Override public String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo) { return getEdgeLabel(dataType.getName(), aInfo.name); @@ -142,10 +144,13 @@ public class GraphBackedMetadataRepository implements MetadataRepository { @Override @GraphTransaction - public String createEntity(IReferenceableInstance typedInstance) throws RepositoryException { - LOG.info("adding entity={}", typedInstance); + public String[] createEntities(ITypedReferenceableInstance... entities) throws RepositoryException, + EntityExistsException { + LOG.info("adding entities={}", entities); try { - return instanceToGraphMapper.mapTypedInstanceToGraph(typedInstance); + return instanceToGraphMapper.mapTypedInstanceToGraph(entities); + } catch (EntityExistsException e) { + throw e; } catch (AtlasException e) { throw new RepositoryException(e); } @@ -159,19 +164,38 @@ public class GraphBackedMetadataRepository implements MetadataRepository { Vertex instanceVertex = getVertexForGUID(guid); try { - LOG.debug("Found a vertex {} for guid {}", instanceVertex, guid); return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex); - } catch (AtlasException e) { throw new RepositoryException(e); } } + @Override + @GraphTransaction + public ITypedReferenceableInstance getEntityDefinition(String entityType, String attribute, String value) + throws AtlasException { + LOG.info("Retrieving entity with type={} and {}={}", entityType, attribute, value); + IDataType type = typeSystem.getDataType(IDataType.class, entityType); + String propertyKey = getFieldNameInVertex(type, attribute); + Vertex instanceVertex = getVertexForProperty(propertyKey, value); + + String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY); + return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex); + } + + private Vertex getVertexForGUID(String guid) throws EntityNotFoundException { - Vertex instanceVertex = GraphHelper.findVertexByGUID(titanGraph, guid); + return getVertexForProperty(Constants.GUID_PROPERTY_KEY, guid); + } + + private Vertex getVertexForProperty(String propertyKey, Object value) throws EntityNotFoundException { + Vertex instanceVertex = GraphHelper.findVertex(titanGraph, propertyKey, value); if (instanceVertex == null) { - LOG.debug("Could not find a vertex for guid={}", guid); - throw new EntityNotFoundException("Could not find an entity in the repository for guid: " + guid); + LOG.debug("Could not find a vertex with {}={}", propertyKey, value); + throw new EntityNotFoundException("Could not find an entity in the repository with " + propertyKey + "=" + + value); + } else { + LOG.debug("Found a vertex {} with {}={}", instanceVertex, propertyKey, value); } return instanceVertex; @@ -244,7 +268,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { instanceVertex, Collections.<Id, Vertex>emptyMap()); // update the traits in entity once adding trait instance is successful - ((TitanVertex) instanceVertex).addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, traitName); + GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName); } catch (RepositoryException e) { throw e; @@ -304,7 +328,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { // add it back again for (String traitName : traitNames) { - ((TitanVertex) instanceVertex).addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, traitName); + GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName); } } @@ -368,16 +392,18 @@ public class GraphBackedMetadataRepository implements MetadataRepository { private final class EntityProcessor implements ObjectGraphWalker.NodeProcessor { - public final Map<Id, Id> idToNewIdMap; public final Map<Id, IReferenceableInstance> idToInstanceMap; public final Map<Id, Vertex> idToVertexMap; public EntityProcessor() { - idToNewIdMap = new HashMap<>(); - idToInstanceMap = new HashMap<>(); + idToInstanceMap = new LinkedHashMap<>(); idToVertexMap = new HashMap<>(); } + public void cleanUp() { + idToInstanceMap.clear(); + } + @Override public void processNode(ObjectGraphWalker.Node nd) throws AtlasException { IReferenceableInstance ref = null; @@ -394,10 +420,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository { if (id != null) { if (id.isUnassigned()) { - if (!idToNewIdMap.containsKey(id)) { - idToNewIdMap.put(id, new Id(ID_SEQ.getAndIncrement(), 0, id.className)); - } - if (ref != null) { if (idToInstanceMap.containsKey(id)) { // Oops throw new RepositoryException( @@ -410,42 +432,74 @@ public class GraphBackedMetadataRepository implements MetadataRepository { } } - private void createVerticesForClassTypes(List<ITypedReferenceableInstance> newInstances) throws AtlasException { - for (ITypedReferenceableInstance typedInstance : newInstances) { + private List<ITypedReferenceableInstance> createVerticesForClassType( + List<ITypedReferenceableInstance> typedInstances) throws AtlasException { + + List<ITypedReferenceableInstance> instancesCreated = new ArrayList<>(); + for (ITypedReferenceableInstance typedInstance : typedInstances) { final Id id = typedInstance.getId(); if (!idToVertexMap.containsKey(id)) { Vertex instanceVertex; if (id.isAssigned()) { // has a GUID - instanceVertex = GraphHelper.findVertexByGUID(titanGraph, id.id); + instanceVertex = getVertexForGUID(id.id); } else { + //Check if there is already an instance with the same unique attribute value ClassType classType = typeSystem.getDataType(ClassType.class, typedInstance.getTypeName()); - instanceVertex = GraphHelper - .createVertexWithIdentity(titanGraph, typedInstance, classType.getAllSuperTypeNames()); + instanceVertex = instanceToGraphMapper.getVertexForInstanceByUniqueAttribute(classType, typedInstance); + if (instanceVertex == null) { + instanceVertex = GraphHelper.createVertexWithIdentity(titanGraph, typedInstance, + classType.getAllSuperTypeNames()); + instancesCreated.add(typedInstance); + + instanceToGraphMapper.mapInstanceToVertex(id, typedInstance, instanceVertex, + classType.fieldMapping().fields, idToVertexMap, true); + + } } idToVertexMap.put(id, instanceVertex); } } + return instancesCreated; } } private final class TypedInstanceToGraphMapper { - private String mapTypedInstanceToGraph(IReferenceableInstance typedInstance) throws AtlasException { - + private String[] mapTypedInstanceToGraph(ITypedReferenceableInstance[] typedInstances) + throws AtlasException { EntityProcessor entityProcessor = new EntityProcessor(); - try { - LOG.debug("Walking the object graph for instance {}", typedInstance.getTypeName()); - new ObjectGraphWalker(typeSystem, entityProcessor, typedInstance).walk(); - } catch (AtlasException me) { - throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me); - } + List<String> guids = new ArrayList<>(); + for (ITypedReferenceableInstance typedInstance : typedInstances) { + try { + LOG.debug("Walking the object graph for instance {}", typedInstance.getTypeName()); + entityProcessor.cleanUp(); + new ObjectGraphWalker(typeSystem, entityProcessor, typedInstance).walk(); + } catch (AtlasException me) { + throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me); + } + + List<ITypedReferenceableInstance> newTypedInstances = discoverInstances(entityProcessor); + List<ITypedReferenceableInstance> instancesCreated = + entityProcessor.createVerticesForClassType(newTypedInstances); - List<ITypedReferenceableInstance> newTypedInstances = discoverInstances(entityProcessor); - entityProcessor.createVerticesForClassTypes(newTypedInstances); - String guid = addDiscoveredInstances(typedInstance, entityProcessor, newTypedInstances); - addFullTextProperty(entityProcessor, newTypedInstances); - return guid; + for (ITypedReferenceableInstance instance : instancesCreated) { + try { + //new vertex, set all the properties + addDiscoveredInstance(entityProcessor, instance); + } catch(SchemaViolationException e) { + throw new EntityExistsException(typedInstance, e); + } + } + + addFullTextProperty(entityProcessor, instancesCreated); + + //Return guid for + Vertex instanceVertex = entityProcessor.idToVertexMap.get(typedInstance.getId()); + String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY); + guids.add(guid); + } + return guids.toArray(new String[guids.size()]); } private void addFullTextProperty(EntityProcessor entityProcessor, @@ -455,7 +509,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { Id id = typedInstance.getId(); Vertex instanceVertex = entityProcessor.idToVertexMap.get(id); String fullText = getFullTextForVertex(instanceVertex, true); - addProperty(instanceVertex, Constants.ENTITY_TEXT_PROPERTY_KEY, fullText); + GraphHelper.setProperty(instanceVertex, Constants.ENTITY_TEXT_PROPERTY_KEY, fullText); } } @@ -572,63 +626,78 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ClassType cT = typeSystem.getDataType(ClassType.class, transientInstance.getTypeName()); ITypedReferenceableInstance newInstance = cT.convert(transientInstance, Multiplicity.REQUIRED); newTypedInstances.add(newInstance); - - // Now replace old references with new Ids - MapIds mapIds = new MapIds(entityProcessor.idToNewIdMap); - new ObjectGraphWalker(typeSystem, mapIds, newTypedInstances).walk(); - } catch (AtlasException me) { throw new RepositoryException( String.format("Failed to create Instance(id = %s", transientInstance.getId()), me); } } - return newTypedInstances; + //Reverse the list to create the entities in dependency order + return Lists.reverse(newTypedInstances); } - private String addDiscoveredInstances(IReferenceableInstance entity, EntityProcessor entityProcessor, - List<ITypedReferenceableInstance> newTypedInstances) throws AtlasException { + /** + * For the given type, finds an unique attribute and checks if there is an existing instance with the same + * unique value + * @param classType + * @param instance + * @return + * @throws AtlasException + */ + Vertex getVertexForInstanceByUniqueAttribute(ClassType classType, IReferenceableInstance instance) + throws AtlasException { + for (AttributeInfo attributeInfo : classType.fieldMapping().fields.values()) { + if (attributeInfo.isUnique) { + String propertyKey = getFieldNameInVertex(classType, attributeInfo); + try { + return getVertexForProperty(propertyKey, instance.get(attributeInfo.name)); + } catch(EntityNotFoundException e) { + //Its ok if there is no entity with the same unique value + } + } + } - String typedInstanceGUID = null; - for (ITypedReferenceableInstance typedInstance : newTypedInstances) { // Traverse over newInstances - LOG.debug("Adding typed instance {}", typedInstance.getTypeName()); + return null; + } - Id id = typedInstance.getId(); - if (id == null) { // oops - throw new RepositoryException("id cannot be null"); - } + private void addDiscoveredInstance(EntityProcessor entityProcessor, ITypedReferenceableInstance typedInstance) + throws AtlasException { + LOG.debug("Adding typed instance {}", typedInstance.getTypeName()); - Vertex instanceVertex = entityProcessor.idToVertexMap.get(id); + Id id = typedInstance.getId(); + if (id == null) { // oops + throw new RepositoryException("id cannot be null"); + } - // add the attributes for the instance - ClassType classType = typeSystem.getDataType(ClassType.class, typedInstance.getTypeName()); - final Map<String, AttributeInfo> fields = classType.fieldMapping().fields; + Vertex instanceVertex = entityProcessor.idToVertexMap.get(id); - mapInstanceToVertex(id, typedInstance, instanceVertex, fields, entityProcessor.idToVertexMap); + // add the attributes for the instance + ClassType classType = typeSystem.getDataType(ClassType.class, typedInstance.getTypeName()); + final Map<String, AttributeInfo> fields = classType.fieldMapping().fields; - for (String traitName : typedInstance.getTraits()) { - LOG.debug("mapping trait {}", traitName); - ((TitanVertex) instanceVertex).addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, traitName); - ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName); + mapInstanceToVertex(id, typedInstance, instanceVertex, fields, entityProcessor.idToVertexMap, false); - // add the attributes for the trait instance - mapTraitInstanceToVertex(traitInstance, typedInstance, instanceVertex, - entityProcessor.idToVertexMap); - } + for (String traitName : typedInstance.getTraits()) { + LOG.debug("mapping trait {}", traitName); + GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName); + ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName); - if (typedInstance.getId() == entity.getId()) { // save the guid for return - typedInstanceGUID = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY); - } + // add the attributes for the trait instance + mapTraitInstanceToVertex(traitInstance, typedInstance, instanceVertex, + entityProcessor.idToVertexMap); } - - return typedInstanceGUID; } private void mapInstanceToVertex(Id id, ITypedInstance typedInstance, Vertex instanceVertex, - Map<String, AttributeInfo> fields, Map<Id, Vertex> idToVertexMap) throws AtlasException { + Map<String, AttributeInfo> fields, Map<Id, Vertex> idToVertexMap, boolean mapOnlyUniqueAttributes) + throws AtlasException { LOG.debug("Mapping instance {} of {} to vertex {}", typedInstance, typedInstance.getTypeName(), instanceVertex); for (AttributeInfo attributeInfo : fields.values()) { + if (mapOnlyUniqueAttributes && !attributeInfo.isUnique) { + continue; + } + final IDataType dataType = attributeInfo.dataType(); mapAttributesToVertex(id, typedInstance, instanceVertex, idToVertexMap, attributeInfo, dataType); } @@ -653,7 +722,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { //handles both int and string for enum EnumValue enumValue = (EnumValue) dataType.convert(typedInstance.get(attributeInfo.name), Multiplicity.REQUIRED); - addProperty(instanceVertex, propertyName, enumValue.value); + GraphHelper.setProperty(instanceVertex, propertyName, enumValue.value); break; case ARRAY: @@ -677,8 +746,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { break; case CLASS: - Id referenceId = (Id) typedInstance.get(attributeInfo.name); - mapClassReferenceAsEdge(instanceVertex, idToVertexMap, edgeLabel, referenceId); + mapClassReferenceAsEdge(instanceVertex, idToVertexMap, edgeLabel, (ITypedReferenceableInstance) attrValue); break; default: @@ -707,7 +775,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { } // for dereference on way out - addProperty(instanceVertex, propertyName, values); + GraphHelper.setProperty(instanceVertex, propertyName, values); } private void mapMapCollectionToVertex(Id id, ITypedInstance typedInstance, Vertex instanceVertex, @@ -726,11 +794,11 @@ public class GraphBackedMetadataRepository implements MetadataRepository { String myPropertyName = propertyName + "." + entry.getKey().toString(); String value = mapCollectionEntryToVertex(id, instanceVertex, attributeInfo, idToVertexMap, elementType, entry.getValue(), myPropertyName); - addProperty(instanceVertex, myPropertyName, value); + GraphHelper.setProperty(instanceVertex, myPropertyName, value); } // for dereference on way out - addProperty(instanceVertex, propertyName, new ArrayList(collection.keySet())); + GraphHelper.setProperty(instanceVertex, propertyName, new ArrayList(collection.keySet())); } private String mapCollectionEntryToVertex(Id id, Vertex instanceVertex, AttributeInfo attributeInfo, @@ -757,8 +825,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository { return structElementEdge.getId().toString(); case CLASS: - Id referenceId = (Id) value; - return mapClassReferenceAsEdge(instanceVertex, idToVertexMap, edgeLabel, referenceId); + return mapClassReferenceAsEdge(instanceVertex, idToVertexMap, edgeLabel, + (ITypedReferenceableInstance) value); default: throw new IllegalArgumentException("Unknown type category: " + elementType.getTypeCategory()); @@ -766,11 +834,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository { } private String mapClassReferenceAsEdge(Vertex instanceVertex, Map<Id, Vertex> idToVertexMap, String propertyKey, - Id id) throws AtlasException { - if (id != null) { + ITypedReferenceableInstance typedReference) throws AtlasException { + if (typedReference != null) { Vertex referenceVertex; + Id id = typedReference instanceof Id ? (Id) typedReference : typedReference.getId(); if (id.isAssigned()) { - referenceVertex = GraphHelper.findVertexByGUID(titanGraph, id.id); + referenceVertex = getVertexForGUID(id.id); } else { referenceVertex = idToVertexMap.get(id); } @@ -796,7 +865,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { // map all the attributes to this newly created vertex mapInstanceToVertex(id, structInstance, structInstanceVertex, structInstance.fieldMapping().fields, - idToVertexMap); + idToVertexMap, false); return structInstanceVertex; } @@ -820,7 +889,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { // map all the attributes to this newly created vertex mapInstanceToVertex(typedInstanceId, traitInstance, traitInstanceVertex, - traitInstance.fieldMapping().fields, idToVertexMap); + traitInstance.fieldMapping().fields, idToVertexMap, false); // add an edge to the newly created vertex from the parent String relationshipLabel = getEdgeLabel(typedInstanceTypeName, traitName); @@ -861,15 +930,10 @@ public class GraphBackedMetadataRepository implements MetadataRepository { //Convert Property value to Long while persisting propertyValue = dateVal.getTime(); } - addProperty(instanceVertex, vertexPropertyName, propertyValue); + GraphHelper.setProperty(instanceVertex, vertexPropertyName, propertyValue); } } - private void addProperty(Vertex vertex, String propertyName, Object value) { - LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex); - vertex.setProperty(propertyName, value); - } - public final class GraphToTypedInstanceMapper { public ITypedReferenceableInstance mapGraphToTypedInstance(String guid, Vertex instanceVertex) @@ -971,6 +1035,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY); LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel, guid); if (attributeInfo.isComposite) { + //Also, when you retrieve a type's instance, you get the complete object graph of the composites LOG.debug("Found composite, mapping vertex to instance"); return mapGraphToTypedInstance(guid, referenceVertex); } else { @@ -1092,6 +1157,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel, guid); if (attributeInfo.isComposite) { + //Also, when you retrieve a type's instance, you get the complete object graph of the composites LOG.debug("Found composite, mapping vertex to instance"); return mapGraphToTypedInstance(guid, referenceVertex); } else { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index ceb8a65..aa532c4 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -46,34 +46,35 @@ public final class GraphHelper { } public static Vertex createVertexWithIdentity(Graph graph, ITypedReferenceableInstance typedInstance, - Set<String> superTypeNames) { - final Vertex vertexWithIdentity = - createVertexWithoutIdentity(graph, typedInstance.getTypeName(), typedInstance.getId(), superTypeNames); + Set<String> superTypeNames) { + final Vertex vertexWithIdentity = createVertexWithoutIdentity(graph, typedInstance.getTypeName(), + typedInstance.getId(), superTypeNames); // add identity final String guid = UUID.randomUUID().toString(); - vertexWithIdentity.setProperty(Constants.GUID_PROPERTY_KEY, guid); + setProperty(vertexWithIdentity, Constants.GUID_PROPERTY_KEY, guid); return vertexWithIdentity; } public static Vertex createVertexWithoutIdentity(Graph graph, String typeName, Id typedInstanceId, - Set<String> superTypeNames) { + Set<String> superTypeNames) { + LOG.debug("Creating vertex for type {} id {}", typeName, typedInstanceId._getId()); final Vertex vertexWithoutIdentity = graph.addVertex(null); // add type information - vertexWithoutIdentity.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName); + setProperty(vertexWithoutIdentity, Constants.ENTITY_TYPE_PROPERTY_KEY, typeName); // add super types for (String superTypeName : superTypeNames) { - ((TitanVertex) vertexWithoutIdentity).addProperty(Constants.SUPER_TYPES_PROPERTY_KEY, superTypeName); + addProperty(vertexWithoutIdentity, Constants.SUPER_TYPES_PROPERTY_KEY, superTypeName); } // add version information - vertexWithoutIdentity.setProperty(Constants.VERSION_PROPERTY_KEY, typedInstanceId.version); + setProperty(vertexWithoutIdentity, Constants.VERSION_PROPERTY_KEY, typedInstanceId.version); // add timestamp information - vertexWithoutIdentity.setProperty(Constants.TIMESTAMP_PROPERTY_KEY, System.currentTimeMillis()); + setProperty(vertexWithoutIdentity, Constants.TIMESTAMP_PROPERTY_KEY, System.currentTimeMillis()); return vertexWithoutIdentity; } @@ -84,12 +85,12 @@ public final class GraphHelper { return titanGraph.addEdge(null, fromVertex, toVertex, edgeLabel); } - public static Vertex findVertexByGUID(TitanGraph titanGraph, String value) { - LOG.debug("Finding vertex for key={}, value={}", Constants.GUID_PROPERTY_KEY, value); + public static Vertex findVertex(TitanGraph titanGraph, String propertyKey, Object value) { + LOG.debug("Finding vertex for {}={}", propertyKey, value); - GraphQuery query = titanGraph.query().has(Constants.GUID_PROPERTY_KEY, value); + GraphQuery query = titanGraph.query().has(propertyKey, value); Iterator<Vertex> results = query.vertices().iterator(); - // returning one since guid should be unique + // returning one since entityType, qualifiedName should be unique return results.hasNext() ? results.next() : null; } @@ -107,6 +108,16 @@ public final class GraphHelper { + edge.getVertex(Direction.IN) + "]"; } + public static void setProperty(Vertex vertex, String propertyName, Object value) { + LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex); + vertex.setProperty(propertyName, value); + } + + public static void addProperty(Vertex vertex, String propertyName, Object value) { + LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex); + ((TitanVertex)vertex).addProperty(propertyName, value); + } + /* public static void dumpToLog(final Graph graph) { LOG.debug("*******************Graph Dump****************************"); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java index 56168db..ea39f92 100755 --- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java +++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java @@ -39,6 +39,7 @@ import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.json.InstanceSerialization; import org.apache.atlas.typesystem.json.TypesSerialization; import org.apache.atlas.typesystem.types.AttributeDefinition; +import org.apache.atlas.typesystem.types.AttributeInfo; import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.DataTypes; import org.apache.atlas.typesystem.types.EnumTypeDefinition; @@ -49,11 +50,14 @@ import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeUtils; +import org.apache.atlas.typesystem.types.ValueConversionException; import org.apache.atlas.typesystem.types.utils.TypesUtil; +import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import scala.actors.threadpool.Arrays; import javax.inject.Inject; import javax.inject.Singleton; @@ -107,23 +111,21 @@ public class DefaultMetadataService implements MetadataService { } private static final AttributeDefinition NAME_ATTRIBUTE = - TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE); + TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE); private static final AttributeDefinition DESCRIPTION_ATTRIBUTE = TypesUtil.createOptionalAttrDef("description", DataTypes.STRING_TYPE); @InterfaceAudience.Private private void createSuperTypes() throws AtlasException { - if (typeSystem.isRegistered(AtlasClient.DATA_SET_SUPER_TYPE)) { - return; // this is already registered - } - HierarchicalTypeDefinition<ClassType> infraType = TypesUtil .createClassTypeDef(AtlasClient.INFRASTRUCTURE_SUPER_TYPE, ImmutableList.<String>of(), NAME_ATTRIBUTE, DESCRIPTION_ATTRIBUTE); + createType(infraType); HierarchicalTypeDefinition<ClassType> datasetType = TypesUtil .createClassTypeDef(AtlasClient.DATA_SET_SUPER_TYPE, ImmutableList.<String>of(), NAME_ATTRIBUTE, DESCRIPTION_ATTRIBUTE); + createType(datasetType); HierarchicalTypeDefinition<ClassType> processType = TypesUtil .createClassTypeDef(AtlasClient.PROCESS_SUPER_TYPE, ImmutableList.<String>of(), NAME_ATTRIBUTE, @@ -132,12 +134,23 @@ public class DefaultMetadataService implements MetadataService { Multiplicity.OPTIONAL, false, null), new AttributeDefinition("outputs", DataTypes.arrayTypeName(AtlasClient.DATA_SET_SUPER_TYPE), Multiplicity.OPTIONAL, false, null)); + createType(processType); - TypesDef typesDef = TypeUtils - .getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), - ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), - ImmutableList.of(infraType, datasetType, processType)); - createType(TypesSerialization.toJson(typesDef)); + HierarchicalTypeDefinition<ClassType> referenceableType = TypesUtil + .createClassTypeDef(AtlasClient.REFERENCEABLE_SUPER_TYPE, ImmutableList.<String>of(), + TypesUtil.createUniqueRequiredAttrDef(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, + DataTypes.STRING_TYPE)); + createType(referenceableType); + } + + private void createType(HierarchicalTypeDefinition<ClassType> type) throws AtlasException { + if (!typeSystem.isRegistered(type.typeName)) { + TypesDef typesDef = TypeUtils + .getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), + ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), + ImmutableList.of(type)); + createType(TypesSerialization.toJson(typesDef)); + } } /** @@ -150,17 +163,9 @@ public class DefaultMetadataService implements MetadataService { @Override public JSONObject createType(String typeDefinition) throws AtlasException { ParamChecker.notEmpty(typeDefinition, "type definition cannot be empty"); + validateTypeDefinition(typeDefinition); - TypesDef typesDef; - try { - typesDef = TypesSerialization.fromJson(typeDefinition); - if (typesDef.isEmpty()) { - throw new AtlasException("Invalid type definition"); - } - } catch (Exception e) { - LOG.error("Unable to deserialize json={}", typeDefinition, e); - throw new IllegalArgumentException("Unable to deserialize json ", e); - } + TypesDef typesDef = validateTypeDefinition(typeDefinition); try { final Map<String, IDataType> typesAdded = typeSystem.defineTypes(typesDef); @@ -185,6 +190,19 @@ public class DefaultMetadataService implements MetadataService { } } + private TypesDef validateTypeDefinition(String typeDefinition) { + try { + TypesDef typesDef = TypesSerialization.fromJson(typeDefinition); + if (typesDef.isEmpty()) { + throw new IllegalArgumentException("Invalid type definition"); + } + return typesDef; + } catch (Exception e) { + LOG.error("Unable to deserialize json={}", typeDefinition, e); + throw new IllegalArgumentException("Unable to deserialize json " + typeDefinition, e); + } + } + /** * Return the definition for the given type. * @@ -220,36 +238,43 @@ public class DefaultMetadataService implements MetadataService { /** * Creates an entity, instance of the type. * - * @param entityInstanceDefinition definition - * @return guid + * @param entityInstanceDefinition json array of entity definitions + * @return guids - json array of guids */ @Override - public String createEntity(String entityInstanceDefinition) throws AtlasException { + public String createEntities(String entityInstanceDefinition) throws AtlasException { ParamChecker.notEmpty(entityInstanceDefinition, "Entity instance definition cannot be empty"); - ITypedReferenceableInstance entityTypedInstance = deserializeClassInstance(entityInstanceDefinition); + ITypedReferenceableInstance[] typedInstances = deserializeClassInstances(entityInstanceDefinition); - final String guid = repository.createEntity(entityTypedInstance); + final String[] guids = repository.createEntities(typedInstances); - onEntityAddedToRepo(entityTypedInstance); - return guid; + onEntityAddedToRepo(Arrays.asList(typedInstances)); + return new JSONArray(Arrays.asList(guids)).toString(); } - private ITypedReferenceableInstance deserializeClassInstance(String entityInstanceDefinition) + private ITypedReferenceableInstance[] deserializeClassInstances(String entityInstanceDefinition) throws AtlasException { - - final Referenceable entityInstance; try { - entityInstance = InstanceSerialization.fromJsonReferenceable(entityInstanceDefinition, true); + JSONArray referableInstances = new JSONArray(entityInstanceDefinition); + ITypedReferenceableInstance[] instances = new ITypedReferenceableInstance[referableInstances.length()]; + for (int index = 0; index < referableInstances.length(); index++) { + Referenceable entityInstance = + InstanceSerialization.fromJsonReferenceable(referableInstances.getString(index), true); + final String entityTypeName = entityInstance.getTypeName(); + ParamChecker.notEmpty(entityTypeName, "Entity type cannot be null"); + + ClassType entityType = typeSystem.getDataType(ClassType.class, entityTypeName); + ITypedReferenceableInstance typedInstrance = entityType.convert(entityInstance, Multiplicity.REQUIRED); + instances[index] = typedInstrance; + } + return instances; + } catch(ValueConversionException e) { + throw e; } catch (Exception e) { // exception from deserializer LOG.error("Unable to deserialize json={}", entityInstanceDefinition, e); throw new IllegalArgumentException("Unable to deserialize json"); } - final String entityTypeName = entityInstance.getTypeName(); - ParamChecker.notEmpty(entityTypeName, "Entity type cannot be null"); - - ClassType entityType = typeSystem.getDataType(ClassType.class, entityTypeName); - return entityType.convert(entityInstance, Multiplicity.REQUIRED); } /** @@ -266,6 +291,29 @@ public class DefaultMetadataService implements MetadataService { return InstanceSerialization.toJson(instance, true); } + @Override + public String getEntityDefinition(String entityType, String attribute, String value) throws AtlasException { + validateTypeExists(entityType); + validateUniqueAttribute(entityType, attribute); + + final ITypedReferenceableInstance instance = repository.getEntityDefinition(entityType, attribute, value); + return InstanceSerialization.toJson(instance, true); + } + + /** + * Validate that attribute is unique attribute + * @param entityType + * @param attributeName + */ + private void validateUniqueAttribute(String entityType, String attributeName) throws AtlasException { + ClassType type = typeSystem.getDataType(ClassType.class, entityType); + AttributeInfo attribute = type.fieldMapping().fields.get(attributeName); + if (!attribute.isUnique) { + throw new IllegalArgumentException( + String.format("%s.%s is not a unique attribute", entityType, attributeName)); + } + } + /** * Return the list of entity names for the given type in the repository. * @@ -291,9 +339,9 @@ public class DefaultMetadataService implements MetadataService { private void validateTypeExists(String entityType) throws AtlasException { ParamChecker.notEmpty(entityType, "entity type cannot be null"); - // verify if the type exists - if (!typeSystem.isRegistered(entityType)) { - throw new TypeNotFoundException("type is not defined for : " + entityType); + IDataType type = typeSystem.getDataType(IDataType.class, entityType); + if (type.getTypeCategory() != DataTypes.TypeCategory.CLASS) { + throw new IllegalArgumentException("type " + entityType + " not a CLASS type"); } } @@ -401,11 +449,11 @@ public class DefaultMetadataService implements MetadataService { } } - private void onEntityAddedToRepo(ITypedReferenceableInstance typedInstance) + private void onEntityAddedToRepo(Collection<ITypedReferenceableInstance> typedInstances) throws AtlasException { for (EntityChangeListener listener : entityChangeListeners) { - listener.onEntityAdded(typedInstance); + listener.onEntityAdded(typedInstances); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/repository/src/main/java/org/apache/atlas/services/MetadataService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/MetadataService.java b/repository/src/main/java/org/apache/atlas/services/MetadataService.java index 8f82f88..c806d3a 100755 --- a/repository/src/main/java/org/apache/atlas/services/MetadataService.java +++ b/repository/src/main/java/org/apache/atlas/services/MetadataService.java @@ -66,7 +66,7 @@ public interface MetadataService { * @param entityDefinition definition * @return guid */ - String createEntity(String entityDefinition) throws AtlasException; + String createEntities(String entityDefinition) throws AtlasException; /** * Return the definition for the given guid. @@ -77,6 +77,16 @@ public interface MetadataService { String getEntityDefinition(String guid) throws AtlasException; /** + * Return the definition given type and attribute. The attribute has to be unique attribute for the type + * @param entityType - type name + * @param attribute - attribute name + * @param value - attribute value + * @return + * @throws AtlasException + */ + String getEntityDefinition(String entityType, String attribute, String value) throws AtlasException; + + /** * Return the list of entity names for the given type in the repository. * * @param entityType type http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java b/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java index 18b1d1b..4195955 100755 --- a/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java +++ b/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java @@ -19,11 +19,8 @@ package org.apache.atlas; import com.thinkaurelius.titan.core.TitanGraph; -import com.thinkaurelius.titan.core.util.TitanCleanup; import org.apache.atlas.repository.graph.GraphProvider; -import org.apache.atlas.typesystem.types.TypeSystem; import org.testng.Assert; -import org.testng.annotations.AfterClass; import org.testng.annotations.Guice; import org.testng.annotations.Test; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/repository/src/test/java/org/apache/atlas/TestUtils.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java index de41793..214c387 100755 --- a/repository/src/test/java/org/apache/atlas/TestUtils.java +++ b/repository/src/test/java/org/apache/atlas/TestUtils.java @@ -26,20 +26,25 @@ import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.Referenceable; +import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.types.AttributeDefinition; import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.EnumType; import org.apache.atlas.typesystem.types.EnumTypeDefinition; import org.apache.atlas.typesystem.types.EnumValue; import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; +import org.apache.atlas.typesystem.types.IDataType; import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TypeSystem; +import org.apache.atlas.typesystem.types.TypeUtils; +import org.apache.atlas.typesystem.types.utils.TypesUtil; +import org.apache.commons.lang.RandomStringUtils; import org.testng.Assert; import java.io.File; +import java.util.Collection; import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef; import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef; @@ -52,6 +57,8 @@ import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeD */ public final class TestUtils { + public static final long TEST_DATE_IN_LONG = 1418265358440L; + private TestUtils() { } @@ -92,7 +99,6 @@ public final class TestUtils { EnumTypeDefinition orgLevelEnum = new EnumTypeDefinition("OrgLevel", new EnumValue("L1", 1), new EnumValue("L2", 2)); - ts.defineEnumType(orgLevelEnum); StructTypeDefinition addressDetails = createStructTypeDef("Address", createRequiredAttrDef("street", DataTypes.STRING_TYPE), @@ -105,7 +111,7 @@ public final class TestUtils { HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", ImmutableList.<String>of(), createRequiredAttrDef("name", DataTypes.STRING_TYPE), - createOptionalAttrDef("orgLevel", ts.getDataType(EnumType.class, "OrgLevel")), + createOptionalAttrDef("orgLevel", "OrgLevel"), createOptionalAttrDef("address", "Address"), new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"), new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates")); @@ -118,12 +124,13 @@ public final class TestUtils { createTraitTypeDef("SecurityClearance", ImmutableList.<String>of(), createRequiredAttrDef("level", DataTypes.INT_TYPE)); - ts.defineTypes(ImmutableList.of(addressDetails), ImmutableList.of(securityClearanceTypeDef), + ts.defineTypes(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails), + ImmutableList.of(securityClearanceTypeDef), ImmutableList.of(deptTypeDef, personTypeDef, managerTypeDef)); } public static Referenceable createDeptEg1(TypeSystem ts) throws AtlasException { - Referenceable hrDept = new Referenceable("Department"); + Referenceable hrDept = new Referenceable(ENTITY_TYPE); Referenceable john = new Referenceable("Person"); Referenceable jane = new Referenceable("Manager", "SecurityClearance"); Referenceable johnAddr = new Referenceable("Address"); @@ -156,4 +163,110 @@ public final class TestUtils { return hrDept; } + + public static final String ENTITY_TYPE = "Department"; + public static final String DATABASE_TYPE = "hive_database"; + public static final String DATABASE_NAME = "foo"; + public static final String TABLE_TYPE = "hive_table"; + public static final String TABLE_NAME = "bar"; + public static final String CLASSIFICATION = "classification"; + public static final String PII = "PII"; + public static final String SUPER_TYPE_NAME = "Base"; + + public static TypesDef defineHiveTypes() { + HierarchicalTypeDefinition<ClassType> superTypeDefinition = + createClassTypeDef(SUPER_TYPE_NAME, ImmutableList.<String>of(), + createOptionalAttrDef("namespace", DataTypes.STRING_TYPE), + createOptionalAttrDef("cluster", DataTypes.STRING_TYPE), + createOptionalAttrDef("colo", DataTypes.STRING_TYPE)); + + HierarchicalTypeDefinition<ClassType> databaseTypeDefinition = + createClassTypeDef(DATABASE_TYPE, ImmutableList.of(SUPER_TYPE_NAME), + TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), + createOptionalAttrDef("created", DataTypes.DATE_TYPE), + createRequiredAttrDef("description", DataTypes.STRING_TYPE)); + + + StructTypeDefinition structTypeDefinition = new StructTypeDefinition("serdeType", + new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE), + createRequiredAttrDef("serde", DataTypes.STRING_TYPE)}); + + EnumValue values[] = {new EnumValue("MANAGED", 1), new EnumValue("EXTERNAL", 2),}; + + EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values); + + HierarchicalTypeDefinition<ClassType> columnsDefinition = + createClassTypeDef("column_type", ImmutableList.<String>of(), + createRequiredAttrDef("name", DataTypes.STRING_TYPE), + createRequiredAttrDef("type", DataTypes.STRING_TYPE)); + + StructTypeDefinition partitionDefinition = new StructTypeDefinition("partition_type", + new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE),}); + + HierarchicalTypeDefinition<ClassType> tableTypeDefinition = + createClassTypeDef(TABLE_TYPE, ImmutableList.of(SUPER_TYPE_NAME), + TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), + createRequiredAttrDef("description", DataTypes.STRING_TYPE), + createRequiredAttrDef("type", DataTypes.STRING_TYPE), + createOptionalAttrDef("created", DataTypes.DATE_TYPE), + // enum + new AttributeDefinition("tableType", "tableType", Multiplicity.REQUIRED, false, null), + // array of strings + new AttributeDefinition("columnNames", + String.format("array<%s>", DataTypes.STRING_TYPE.getName()), Multiplicity.OPTIONAL, + false, null), + // array of classes + new AttributeDefinition("columns", String.format("array<%s>", "column_type"), + Multiplicity.OPTIONAL, true, null), + // array of structs + new AttributeDefinition("partitions", String.format("array<%s>", "partition_type"), + Multiplicity.OPTIONAL, true, null), + // map of primitives + new AttributeDefinition("parametersMap", + DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()), + Multiplicity.OPTIONAL, true, null), + // map of classes - todo - enable this + // new AttributeDefinition("columnsMap", + // DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), + // "column_type"), + // Multiplicity.COLLECTION, true, null), + // map of structs todo - enable this + // new AttributeDefinition("partitionsMap", + // DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), + // "partition_type"), + // Multiplicity.COLLECTION, true, null), + // struct reference + new AttributeDefinition("serde1", "serdeType", Multiplicity.OPTIONAL, false, null), + new AttributeDefinition("serde2", "serdeType", Multiplicity.OPTIONAL, false, null), + // class reference + new AttributeDefinition("database", DATABASE_TYPE, Multiplicity.REQUIRED, true, null)); + + HierarchicalTypeDefinition<TraitType> piiTypeDefinition = + createTraitTypeDef(PII, ImmutableList.<String>of()); + + HierarchicalTypeDefinition<TraitType> classificationTypeDefinition = + createTraitTypeDef(CLASSIFICATION, ImmutableList.<String>of(), + createRequiredAttrDef("tag", DataTypes.STRING_TYPE)); + + HierarchicalTypeDefinition<TraitType> fetlClassificationTypeDefinition = + createTraitTypeDef("fetl" + CLASSIFICATION, ImmutableList.of(CLASSIFICATION), + createRequiredAttrDef("tag", DataTypes.STRING_TYPE)); + + return TypeUtils.getTypesDef(ImmutableList.of(enumTypeDefinition), + ImmutableList.of(structTypeDefinition, partitionDefinition), + ImmutableList.of(classificationTypeDefinition, fetlClassificationTypeDefinition, piiTypeDefinition), + ImmutableList.of(superTypeDefinition, databaseTypeDefinition, columnsDefinition, tableTypeDefinition)); + } + + public static Collection<IDataType> createHiveTypes(TypeSystem typeSystem) throws Exception { + if (!typeSystem.isRegistered(TABLE_TYPE)) { + TypesDef typesDef = defineHiveTypes(); + return typeSystem.defineTypes(typesDef).values(); + } + return null; + } + + public static final String randomString() { + return RandomStringUtils.randomAlphanumeric(10); + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java index 127e6d7..c72ffb2 100755 --- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java @@ -85,7 +85,7 @@ public class GraphBackedDiscoveryServiceTest { ClassType deptType = typeSystem.getDataType(ClassType.class, "Department"); ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED); - repositoryService.createEntity(hrDept2); + repositoryService.createEntities(hrDept2); } private void setupSampleData() throws ScriptException { @@ -303,6 +303,6 @@ public class GraphBackedDiscoveryServiceTest { ClassType deptType = TypeSystem.getInstance().getDataType(ClassType.class, "D"); ITypedReferenceableInstance typedInstance = deptType.convert(instance, Multiplicity.REQUIRED); - repositoryService.createEntity(typedInstance); + repositoryService.createEntities(typedInstance); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java index 2c7d61a..db51ae5 100644 --- a/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java @@ -54,9 +54,6 @@ import org.testng.annotations.Guice; import org.testng.annotations.Test; import javax.inject.Inject; -import java.io.File; -import java.io.IOException; -import java.util.HashSet; import java.util.List; /** @@ -534,7 +531,10 @@ public class HiveLineageServiceTest { String entityJSON = InstanceSerialization.toJson(referenceable, true); System.out.println("Submitting new entity= " + entityJSON); - String guid = metadataService.createEntity(entityJSON); + JSONArray jsonArray = new JSONArray(); + jsonArray.put(entityJSON); + String response = metadataService.createEntities(jsonArray.toString()); + String guid = new JSONArray(response).getString(0); System.out.println("created instance for type " + typeName + ", guid: " + guid); // return the reference to created instance with guid http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1bfda02a/repository/src/test/java/org/apache/atlas/repository/BaseTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/BaseTest.java b/repository/src/test/java/org/apache/atlas/repository/BaseTest.java index 8924c37..a43fc93 100755 --- a/repository/src/test/java/org/apache/atlas/repository/BaseTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/BaseTest.java @@ -18,172 +18,13 @@ package org.apache.atlas.repository; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import org.apache.atlas.AtlasException; -import org.apache.atlas.repository.memory.MemRepository; -import org.apache.atlas.typesystem.ITypedReferenceableInstance; -import org.apache.atlas.typesystem.Referenceable; -import org.apache.atlas.typesystem.Struct; -import org.apache.atlas.typesystem.types.AttributeDefinition; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.HierarchicalType; -import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; -import org.apache.atlas.typesystem.types.IDataType; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.StructType; -import org.apache.atlas.typesystem.types.StructTypeDefinition; -import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TypeSystem; -import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.junit.Before; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.Date; -import java.util.Map; - public abstract class BaseTest { - - public static final String STRUCT_TYPE_1 = "t1"; - public static final String STRUCT_TYPE_2 = "t2"; - public static final String TEST_DATE = "2014-12-11T02:35:58.440Z"; - public static final long TEST_DATE_IN_LONG = 1418265358440L; - protected IRepository repo; - - public static Struct createStruct() throws AtlasException { - StructType structType = (StructType) TypeSystem.getInstance().getDataType(StructType.class, STRUCT_TYPE_1); - Struct s = new Struct(structType.getName()); - s.set("a", 1); - s.set("b", true); - s.set("c", (byte) 1); - s.set("d", (short) 2); - s.set("e", 1); - s.set("f", 1); - s.set("g", 1L); - s.set("h", 1.0f); - s.set("i", 1.0); - s.set("j", BigInteger.valueOf(1L)); - s.set("k", new BigDecimal(1)); - s.set("l", new Date(1418265358440L)); - s.set("m", Lists.<Integer>asList(Integer.valueOf(1), new Integer[]{Integer.valueOf(1)})); - s.set("n", Lists.<BigDecimal>asList(BigDecimal.valueOf(1.1), new BigDecimal[]{BigDecimal.valueOf(1.1)})); - Map<String, Double> hm = Maps.<String, Double>newHashMap(); - hm.put("a", 1.0); - hm.put("b", 2.0); - s.set("o", hm); - return s; - } - - protected final TypeSystem getTypeSystem() { - return TypeSystem.getInstance(); - } - - protected final IRepository getRepository() { - return repo; - } - @Before public void setup() throws Exception { - TypeSystem ts = TypeSystem.getInstance(); ts.reset(); - repo = new MemRepository(ts); - - StructType structType = - ts.defineStructType(STRUCT_TYPE_1, true, TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE), - TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE), - TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE), - TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE), - TypesUtil.createOptionalAttrDef("e", DataTypes.INT_TYPE), - TypesUtil.createOptionalAttrDef("f", DataTypes.INT_TYPE), - TypesUtil.createOptionalAttrDef("g", DataTypes.LONG_TYPE), - TypesUtil.createOptionalAttrDef("h", DataTypes.FLOAT_TYPE), - TypesUtil.createOptionalAttrDef("i", DataTypes.DOUBLE_TYPE), - TypesUtil.createOptionalAttrDef("j", DataTypes.BIGINTEGER_TYPE), - TypesUtil.createOptionalAttrDef("k", DataTypes.BIGDECIMAL_TYPE), - TypesUtil.createOptionalAttrDef("l", DataTypes.DATE_TYPE), - TypesUtil.createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)), - TypesUtil.createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)), TypesUtil - .createOptionalAttrDef("o", - ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE))); - - StructType recursiveStructType = - ts.defineStructType(STRUCT_TYPE_2, true, TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE), - TypesUtil.createOptionalAttrDef("s", STRUCT_TYPE_2)); - } - - protected Map<String, IDataType> defineTraits(HierarchicalTypeDefinition... tDefs) throws AtlasException { - - return getTypeSystem().defineTraitTypes(tDefs); - } - - /* - * Class Hierarchy is: - * Department(name : String, employees : Array[Person]) - * Person(name : String, department : Department, manager : Manager) - * Manager(subordinates : Array[Person]) extends Person - * - * Persons can have SecurityClearance(level : Int) clearance. - */ - protected void defineDeptEmployeeTypes(TypeSystem ts) throws AtlasException { - - HierarchicalTypeDefinition<ClassType> deptTypeDef = TypesUtil - .createClassTypeDef("Department", ImmutableList.<String>of(), - TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE), - new AttributeDefinition("employees", String.format("array<%s>", "Person"), - Multiplicity.COLLECTION, true, "department")); - HierarchicalTypeDefinition<ClassType> personTypeDef = TypesUtil - .createClassTypeDef("Person", ImmutableList.<String>of(), - TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE), - new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"), - new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates")); - HierarchicalTypeDefinition<ClassType> managerTypeDef = TypesUtil - .createClassTypeDef("Manager", ImmutableList.<String>of("Person"), - new AttributeDefinition("subordinates", String.format("array<%s>", "Person"), - Multiplicity.COLLECTION, false, "manager")); - - HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = TypesUtil - .createTraitTypeDef("SecurityClearance", ImmutableList.<String>of(), - TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE)); - - ts.defineTypes(ImmutableList.<StructTypeDefinition>of(), - ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(securityClearanceTypeDef), - ImmutableList.<HierarchicalTypeDefinition<ClassType>>of(deptTypeDef, personTypeDef, managerTypeDef)); - - ImmutableList<HierarchicalType> types = ImmutableList - .of(ts.getDataType(HierarchicalType.class, "SecurityClearance"), - ts.getDataType(ClassType.class, "Department"), ts.getDataType(ClassType.class, "Person"), - ts.getDataType(ClassType.class, "Manager")); - - repo.defineTypes(types); - - } - - protected Referenceable createDeptEg1(TypeSystem ts) throws AtlasException { - Referenceable hrDept = new Referenceable("Department"); - Referenceable john = new Referenceable("Person"); - Referenceable jane = new Referenceable("Manager", "SecurityClearance"); - - hrDept.set("name", "hr"); - john.set("name", "John"); - john.set("department", hrDept); - jane.set("name", "Jane"); - jane.set("department", hrDept); - - john.set("manager", jane); - - hrDept.set("employees", ImmutableList.<Referenceable>of(john, jane)); - - jane.set("subordinates", ImmutableList.<Referenceable>of(john)); - - jane.getTrait("SecurityClearance").set("level", 1); - - ClassType deptType = ts.getDataType(ClassType.class, "Department"); - ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED); - - return hrDept; } }
