Repository: incubator-atlas Updated Branches: refs/heads/master 4f4ab9b98 -> acfe9a48f
ATLAS-1526: updated AtlasAttribute with helper methods to derive isComposite/reverseAttribute flags Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/acfe9a48 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/acfe9a48 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/acfe9a48 Branch: refs/heads/master Commit: acfe9a48fa3b1a4f75f52b6d43d0bd6ee964c441 Parents: 4f4ab9b Author: Madhan Neethiraj <[email protected]> Authored: Sat Feb 4 03:25:44 2017 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Sun Feb 5 17:34:04 2017 -0800 ---------------------------------------------------------------------- addons/models/0030-hive_model.json | 18 +- addons/models/0080-storm_model.json | 6 +- .../atlas/model/typedef/AtlasStructDef.java | 1 - .../org/apache/atlas/type/AtlasEntityType.java | 27 +-- .../org/apache/atlas/type/AtlasStructType.java | 109 ++++++++---- .../apache/atlas/type/TestAtlasEntityType.java | 1 - .../store/graph/v1/AtlasStructDefStoreV1.java | 165 +++++++++++-------- .../store/graph/v1/GraphMutationContext.java | 4 +- .../atlas/typesystem/types/utils/TypesUtil.java | 4 + .../java/org/apache/atlas/util/RestUtils.java | 142 +++++++--------- .../adapters/AtlasStructFormatConverter.java | 2 +- .../atlas/web/resources/TypesResource.java | 37 +---- 12 files changed, 271 insertions(+), 245 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/acfe9a48/addons/models/0030-hive_model.json ---------------------------------------------------------------------- diff --git a/addons/models/0030-hive_model.json b/addons/models/0030-hive_model.json index 561cf5b..5ae16cf 100644 --- a/addons/models/0030-hive_model.json +++ b/addons/models/0030-hive_model.json @@ -231,14 +231,7 @@ "name": "partitionKeys", "typeName": "array<hive_column>", "cardinality": "SINGLE", - "constraintDefs": [ - { - "type": "foreignKey", - "params": { - "onDelete": "update" - } - } - ], + "constraintDefs": [], "isIndexable": false, "isOptional": true, "isUnique": false @@ -255,14 +248,7 @@ "name": "columns", "typeName": "array<hive_column>", "cardinality": "SINGLE", - "constraintDefs": [ - { - "type": "foreignKey", - "params": { - "onDelete": "update" - } - } - ], + "constraintDefs": [], "isIndexable": false, "isOptional": true, "isUnique": false http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/acfe9a48/addons/models/0080-storm_model.json ---------------------------------------------------------------------- diff --git a/addons/models/0080-storm_model.json b/addons/models/0080-storm_model.json index 59f3228..344227d 100644 --- a/addons/models/0080-storm_model.json +++ b/addons/models/0080-storm_model.json @@ -54,11 +54,7 @@ "name": "nodes", "typeName": "array<storm_node>", "cardinality": "LIST", - "constraintDefs": [ - { - "type": "foreignKey" - } - ], + "constraintDefs": [], "isIndexable": false, "isOptional": false, "isUnique": false http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/acfe9a48/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java index ff64415..2c00f54 100644 --- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java +++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java @@ -470,7 +470,6 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { public static final String CONSTRAINT_PARAM_REF_ATTRIBUTE = "refAttribute"; public static final String CONSTRAINT_PARAM_ON_DELETE = "onDelete"; public static final String CONSTRAINT_PARAM_VAL_CASCADE = "cascade"; - public static final String CONSTRAINT_PARAM_VAL_UPDATE = "update"; private String type; // foreignKey/mappedFromRef/valueInRange private Map<String, Object> params; // onDelete=cascade/refAttribute=attr2/min=0,max=23 http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/acfe9a48/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java index 0099307..761eef5 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java @@ -20,7 +20,6 @@ package org.apache.atlas.type; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.typedef.AtlasEntityDef; @@ -134,7 +133,7 @@ public class AtlasEntityType extends AtlasStructType { for (Map.Entry<String, AtlasAttribute> e : mappedFromRefAttributes.entrySet()) { AtlasAttribute attribute = e.getValue(); - if(StringUtils.equals(attribute.getStructType().getTypeName(), typeName) && StringUtils.equals(attribute.getName(), attribName)) { + if(StringUtils.equals(attribute.getDefinedInType().getTypeName(), typeName) && StringUtils.equals(attribute.getName(), attribName)) { ret = e.getKey(); break; @@ -358,11 +357,11 @@ public class AtlasEntityType extends AtlasStructType { AtlasType attribType = attribute.getAttributeType(); - if (attribType.getTypeCategory() == TypeCategory.ARRAY) { + if (attribType instanceof AtlasArrayType) { attribType = ((AtlasArrayType)attribType).getElementType(); } - if (attribType.getTypeCategory() != TypeCategory.ENTITY) { + if (!(attribType instanceof AtlasEntityType)) { throw new AtlasBaseException(AtlasErrorCode.CONSTRAINT_NOT_SATISFIED, getTypeName(), attribDef.getName(), CONSTRAINT_TYPE_MAPPED_FROM_REF, attribDef.getTypeName()); @@ -427,11 +426,23 @@ public class AtlasEntityType extends AtlasStructType { public String toTypeName() { return fromAttribute.getTypeName(); } - public AtlasStructType fromType() { return fromAttribute.getStructType(); } + public AtlasStructType fromType() { return fromAttribute.getDefinedInType(); } public AtlasAttribute fromAttribute() { return fromAttribute; } - public AtlasEntityType toType() { return (AtlasEntityType)fromAttribute.getAttributeType(); } + public AtlasEntityType toType() { + AtlasType attrType = fromAttribute.getAttributeType(); + + if (attrType instanceof AtlasArrayType) { + attrType = ((AtlasArrayType)attrType).getElementType(); + } + + if (attrType instanceof AtlasEntityType) { + return (AtlasEntityType)attrType; + } + + return null; + } public AtlasConstraintDef getConstraint() { return refConstraint; } @@ -439,10 +450,6 @@ public class AtlasEntityType extends AtlasStructType { return StringUtils.equals(getOnDeleteAction(), CONSTRAINT_PARAM_VAL_CASCADE); } - public boolean isOnDeleteUpdate() { - return StringUtils.equals(getOnDeleteAction(), CONSTRAINT_PARAM_VAL_UPDATE); - } - private String getOnDeleteAction() { Map<String, Object> params = refConstraint.getParams(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/acfe9a48/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java index 3a815b7..f444e93 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java @@ -19,12 +19,12 @@ package org.apache.atlas.type; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef; +import org.apache.atlas.type.AtlasEntityType.ForeignKeyReference; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; @@ -38,7 +38,6 @@ import java.util.List; import java.util.Map; import java.util.Set; - /** * class that implements behaviour of a struct-type. */ @@ -109,11 +108,6 @@ public class AtlasStructType extends AtlasType { AtlasConstraintDef.CONSTRAINT_PARAM_VAL_CASCADE); } - public boolean isForeignKeyOnDeleteActionUpdate(String attributeName) { - return StringUtils.equals(getForeignKeyOnDeleteAction(attributeName), - AtlasConstraintDef.CONSTRAINT_PARAM_VAL_UPDATE); - } - @Override public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { Map<String, AtlasAttribute> a = new HashMap<>(); @@ -153,10 +147,14 @@ public class AtlasStructType extends AtlasType { AtlasAttribute attribute = getAttribute(attributeName); AtlasConstraintDef constraint = e.getValue(); - AtlasType attributeType = attribute.getAttributeType(); + AtlasType attrType = attribute.getAttributeType(); + + if (attrType instanceof AtlasArrayType) { + attrType = ((AtlasArrayType)attrType).getElementType(); + } - if (attributeType instanceof AtlasEntityType) { - ((AtlasEntityType)attributeType).addForeignKeyReference(attribute, constraint); + if (attrType instanceof AtlasEntityType) { + ((AtlasEntityType)attrType).addForeignKeyReference(attribute, constraint); } } } @@ -406,21 +404,21 @@ public class AtlasStructType extends AtlasType { continue; } - if (this.getTypeCategory() != TypeCategory.ENTITY) { + if (!(this instanceof AtlasEntityType)) { throw new AtlasBaseException(AtlasErrorCode.UNSUPPORTED_CONSTRAINT, AtlasConstraintDef.CONSTRAINT_TYPE_FOREIGN_KEY, getTypeName(), attribute.getName()); } - AtlasType attribType = attribute.getAttributeType(); + AtlasType attrType = attribute.getAttributeType(); - if (attribType.getTypeCategory() == TypeCategory.ARRAY) { - attribType = ((AtlasArrayType) attribType).getElementType(); + if (attrType instanceof AtlasArrayType) { + attrType = ((AtlasArrayType) attrType).getElementType(); } - if (attribType.getTypeCategory() != TypeCategory.ENTITY) { + if (!(attrType instanceof AtlasEntityType)) { throw new AtlasBaseException(AtlasErrorCode.CONSTRAINT_NOT_SATISFIED, getTypeName(), attribute.getName(), AtlasConstraintDef.CONSTRAINT_TYPE_FOREIGN_KEY, - attribType.getTypeName()); + attrType.getTypeName()); } if (ret == null) { @@ -446,21 +444,21 @@ public class AtlasStructType extends AtlasType { public static class AtlasAttribute { - private final AtlasStructType structType; + private final AtlasStructType definedInType; private final AtlasType attributeType; private final AtlasAttributeDef attributeDef; private final String qualifiedName; - public AtlasAttribute(AtlasStructType structType, AtlasAttributeDef attrDef, AtlasType attributeType) { - this.structType = structType; + public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType) { + this.definedInType = definedInType; this.attributeDef = attrDef; this.attributeType = attributeType; - this.qualifiedName = getQualifiedAttributeName(structType.getStructDef(), attributeDef.getName()); + this.qualifiedName = getQualifiedAttributeName(definedInType.getStructDef(), attributeDef.getName()); } - public AtlasStructType getStructType() { return structType; } + public AtlasStructType getDefinedInType() { return definedInType; } - public AtlasStructDef getStructDef() { return structType.getStructDef(); } + public AtlasStructDef getDefinedInDef() { return definedInType.getStructDef(); } public AtlasType getAttributeType() { return attributeType; @@ -480,20 +478,73 @@ public class AtlasStructType extends AtlasType { return qualifiedName; } + public boolean isForeignKeyWithOnDeleteCascade() { + return definedInType.isForeignKeyOnDeleteActionCascade(getName()); + } + + /* + * true - if attribute-type has foreign-key(onDelete=cascade) reference to this type + * false - in all cases + * + * "legacyIsComposite" can not be computed and cached in the constructor - as definedInType is not fully + * populated at the time AtlasAttribute object is constructed. + */ + public boolean legacyIsComposite() { + boolean ret = false; + + if (definedInType instanceof AtlasEntityType) { + AtlasEntityType entityType = (AtlasEntityType) definedInType; + AtlasType attrType = attributeType; + + if (attrType instanceof AtlasArrayType) { + attrType = ((AtlasArrayType)attrType).getElementType(); + } + + if (attrType instanceof AtlasEntityType) { + for (ForeignKeyReference fkRef : entityType.getForeignKeyReferences()) { + if (fkRef.isOnDeleteCascade() && StringUtils.equals(fkRef.fromTypeName(), attrType.getTypeName())) { + ret = true; + break; + } + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("*** {}.{}: isComposite={} ***", definedInType.getTypeName(), getName(), ret); + } + + return ret; + } + /* - * "isContainedAttribute" can not be computed and cached in the constructor - as structType is not fully + * return the name of the attribute in attribute-type that has mappedFromRef constraint on this attribute + * + * "legacyReverseAttribute" can not be computed and cached in the constructor - as definedInType is not fully * populated at the time AtlasAttribute object is constructed. */ - public boolean isContainedAttribute() { - if ( structType.isForeignKeyOnDeleteActionUpdate(attributeDef.getName()) ) { - return true; + public String legacyReverseAttribute() { + String ret = null; + + if (definedInType instanceof AtlasEntityType) { + AtlasType attrType = attributeType; + + if (attrType instanceof AtlasArrayType) { + attrType = ((AtlasArrayType)attrType).getElementType(); + } + + if (attrType instanceof AtlasEntityType) { + AtlasEntityType attribEntityType = (AtlasEntityType) attrType; + + ret = attribEntityType.getMappedFromRefAttribute(definedInType.getTypeName(), getName()); + } } - if ( structType instanceof AtlasEntityType) { - return ((AtlasEntityType) structType).isMappedFromRefAttribute(attributeDef.getName()); + if (LOG.isDebugEnabled()) { + LOG.debug("*** {}.{}: reverseAttribute={} ***", definedInType.getTypeName(), getName(), ret); } - return false; + return ret; } public static String getQualifiedAttributeName(AtlasStructDef structDef, String attrName) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/acfe9a48/intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java b/intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java index dc5bbf1..032dce0 100644 --- a/intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java +++ b/intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java @@ -154,7 +154,6 @@ public class TestAtlasEntityType { assertEquals(fkRef.fromAttributeName(), ATTR_TABLE); assertEquals(fkRef.toTypeName(), TYPE_TABLE); assertTrue(fkRef.isOnDeleteCascade()); - assertFalse(fkRef.isOnDeleteUpdate()); assertEquals(typeTable.getForeignKeyAttributes().size(), 0); assertEquals(typeTable.getMappedFromRefAttributes().size(), 1); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/acfe9a48/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java index 7476c42..d48c87e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java @@ -29,15 +29,17 @@ import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasStructDefStore; import org.apache.atlas.repository.util.FilterUtil; -import org.apache.atlas.type.AtlasArrayType; -import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasStructType; +import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeUtil; +import org.apache.atlas.typesystem.types.AttributeDefinition; +import org.apache.atlas.typesystem.types.AttributeInfo; import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; +import org.codehaus.jettison.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -394,7 +396,7 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { String propertyKey = AtlasGraphUtilsV1.getTypeDefPropertyKey(structDef, attributeDef.getName()); - AtlasGraphUtilsV1.setProperty(vertex, propertyKey, toJsonFromAttributeDef(attributeDef, structType)); + AtlasGraphUtilsV1.setProperty(vertex, propertyKey, toJsonFromAttribute(structType.getAttribute(attributeDef.getName()))); attrNames.add(attributeDef.getName()); } @@ -438,7 +440,7 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At String propertyKey = AtlasGraphUtilsV1.getTypeDefPropertyKey(structDef, attributeDef.getName()); - AtlasGraphUtilsV1.setProperty(vertex, propertyKey, toJsonFromAttributeDef(attributeDef, structType)); + AtlasGraphUtilsV1.setProperty(vertex, propertyKey, toJsonFromAttribute(structType.getAttribute(attributeDef.getName()))); } } @@ -496,32 +498,10 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At } } - private static String toJsonFromAttributeDef(AtlasAttributeDef attributeDef, AtlasStructType structType) { - boolean isComposite = false; - String reverseAttribName = null; - - if (structType instanceof AtlasEntityType) { - AtlasEntityType entityType = (AtlasEntityType)structType; - - isComposite = entityType.isMappedFromRefAttribute(attributeDef.getName()) || - entityType.isForeignKeyOnDeleteActionUpdate(attributeDef.getName()); - } - - // find the attribute in the referenced entity that has mappedFromRef to this attribute - if (structType.isForeignKeyAttribute(attributeDef.getName())) { - AtlasType attribType = structType.getAttributeType(attributeDef.getName()); - - if (attribType.getTypeCategory() == org.apache.atlas.model.TypeCategory.ARRAY) { - attribType = ((AtlasArrayType)attribType).getElementType(); - } - - if (attribType.getTypeCategory() == org.apache.atlas.model.TypeCategory.ENTITY) { - AtlasEntityType attribEntityType = (AtlasEntityType)attribType; - - reverseAttribName = attribEntityType.getMappedFromRefAttribute(structType.getTypeName(), - attributeDef.getName()); - } - } + private static String toJsonFromAttribute(AtlasAttribute attribute) { + AtlasAttributeDef attributeDef = attribute.getAttributeDef(); + boolean isComposite = attribute.legacyIsComposite(); + String reverseAttribName = attribute.legacyReverseAttribute(); Map<String, Object> attribInfo = new HashMap<>(); @@ -531,6 +511,7 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At attribInfo.put("isIndexable", attributeDef.getIsIndexable()); attribInfo.put("isComposite", isComposite); attribInfo.put("reverseAttributeName", reverseAttribName); + attribInfo.put("isForeignKeyWithOnDeleteCascade", attribute.isForeignKeyWithOnDeleteCascade()); final int lower; final int upper; @@ -572,7 +553,7 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At String attrTypeName = ret.getTypeName(); if (AtlasTypeUtil.isArrayType(attrTypeName)) { - Set<String> typeNames = AtlasTypeUtil.getReferencedTypeNames(ret.getTypeName()); + Set<String> typeNames = AtlasTypeUtil.getReferencedTypeNames(attrTypeName); if (typeNames.size() > 0) { attrTypeName = typeNames.iterator().next(); @@ -582,57 +563,97 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At if (!AtlasTypeUtil.isBuiltInType(attrTypeName)) { AtlasVertex attributeType = typeDefStore.findTypeVertexByName(attrTypeName); - // check for isComposite/reverseAttributeName for entity types + /* determine constraints to add to this attribute + - add mappedFromRef if attribute-type has an attribute that refers to this attribute via reverseAttributeName + example: hive_table.sd referenced from hive_storagedesc.table with reverseAttributeName=sd + - add foreignKey(onDelete=cascade) if attribute-type has an attribute that refers to this struct with isComposite=true + example: hive_storagedesc referenced from hive_table.sd with isComposite=true + example: hive_column referenced from hive_table.columns with isComposite=true + */ if (attributeType != null && typeDefStore.isTypeVertex(attributeType, TypeCategory.CLASS)) { - String reverseAttribName = (String) attribInfo.get("reverseAttributeName"); - Boolean isComposite = (Boolean) attribInfo.get("isComposite"); + boolean attributeTypeHasIsCompositeRef = false; + String attributeTypeRevAttribRefFrom = null; - if (StringUtils.isNotBlank(reverseAttribName) || isComposite) { - if (AtlasTypeUtil.isMapType(attrTypeName)) { - throw new AtlasBaseException(AtlasErrorCode.CONSTRAINT_NOT_SUPPORTED_ON_MAP_TYPE, - structDef.getName(), ret.getName(), attrTypeName); - } + List<String> attrNames = attributeType.getProperty(AtlasGraphUtilsV1.getTypeDefPropertyKey(attrTypeName), List.class); - String refAttributeName = null; - List<String> attrNames = attributeType.getProperty( - AtlasGraphUtilsV1.getTypeDefPropertyKey(attrTypeName), List.class); + if (CollectionUtils.isNotEmpty(attrNames)) { + for (String attrName : attrNames) { + String attribJson = attributeType.getProperty( + AtlasGraphUtilsV1.getTypeDefPropertyKey(attrTypeName, attrName), String.class); - if (CollectionUtils.isNotEmpty(attrNames)) { - for (String attrName : attrNames) { - String attribJson = attributeType.getProperty( - AtlasGraphUtilsV1.getTypeDefPropertyKey(attrTypeName, attrName), String.class); + if (StringUtils.isBlank(attribJson)) { + continue; + } - Map refAttrInfo = AtlasType.fromJson(attribJson, Map.class); - String refAttribType = (String) refAttrInfo.get("dataType"); - String refAttribRevAttribName = (String) refAttrInfo.get("reverseAttributeName"); + Map refAttrInfo = AtlasType.fromJson(attribJson, Map.class); + + if (refAttrInfo == null) { + continue; + } + + String refAttribType = (String) refAttrInfo.get("dataType"); - if (StringUtils.equals(refAttribType, structDef.getName()) && - StringUtils.equals(refAttribRevAttribName, ret.getName())) { - refAttributeName = (String) refAttrInfo.get("name"); + if (AtlasTypeUtil.isArrayType(refAttribType)) { + Set<String> typeNames = AtlasTypeUtil.getReferencedTypeNames(refAttribType); - break; + if (typeNames.size() > 0) { + refAttribType = typeNames.iterator().next(); } } - } - if (isComposite) { - if (StringUtils.isNotBlank(refAttributeName)) { // ex: hive_table.columns, hive_column.table - Map<String, Object> params = new HashMap<>(); - params.put(AtlasConstraintDef.CONSTRAINT_PARAM_REF_ATTRIBUTE, refAttributeName); + if (!StringUtils.equals(refAttribType, structDef.getName())) { + continue; + } + + if (StringUtils.isBlank(attributeTypeRevAttribRefFrom)) { + String refAttribRevAttribName = (String) refAttrInfo.get("reverseAttributeName"); + + if (StringUtils.equals(refAttribRevAttribName, ret.getName())) { + attributeTypeRevAttribRefFrom = (String) refAttrInfo.get("name"); + } + } + + if (!attributeTypeHasIsCompositeRef) { + Object val = refAttrInfo.get("isComposite"); + + if (val instanceof Boolean) { + attributeTypeHasIsCompositeRef = (Boolean) val; + } if (val != null) { + attributeTypeHasIsCompositeRef = Boolean.parseBoolean(val.toString()); + } + } - ret.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_MAPPED_FROM_REF, params)); - } else { // ex: hive_table.partitionKeys, with no reverseAttribute-reference - ret.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_FOREIGN_KEY)); + if (StringUtils.isNotBlank(attributeTypeRevAttribRefFrom) && attributeTypeHasIsCompositeRef) { + break; } } + } + + boolean isForeignKeyWithOnDeleteCascade = attributeTypeHasIsCompositeRef; - if (StringUtils.isNotBlank(reverseAttribName)) { // ex: hive_column.table - Map<String, Object> params = new HashMap<>(); - params.put(CONSTRAINT_PARAM_ON_DELETE, CONSTRAINT_PARAM_VAL_CASCADE); + if (!isForeignKeyWithOnDeleteCascade) { + Object val = attribInfo.get("isForeignKeyWithOnDeleteCascade"); - ret.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_FOREIGN_KEY, params)); + if (val instanceof Boolean) { + isForeignKeyWithOnDeleteCascade = (Boolean) val; + } else if (val != null) { + isForeignKeyWithOnDeleteCascade = Boolean.parseBoolean(val.toString()); } } + + if (StringUtils.isNotBlank(attributeTypeRevAttribRefFrom)) { + Map<String, Object> params = new HashMap<>(); + params.put(AtlasConstraintDef.CONSTRAINT_PARAM_REF_ATTRIBUTE, attributeTypeRevAttribRefFrom); + + ret.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_MAPPED_FROM_REF, params)); + } + + if (isForeignKeyWithOnDeleteCascade) { // ex: hive_column.table + Map<String, Object> params = new HashMap<>(); + params.put(CONSTRAINT_PARAM_ON_DELETE, CONSTRAINT_PARAM_VAL_CASCADE); + + ret.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_FOREIGN_KEY, params)); + } } } @@ -664,4 +685,18 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At return ret; } + + public static AttributeDefinition toAttributeDefintion(AtlasAttribute attribute) { + AttributeDefinition ret = null; + + String jsonString = toJsonFromAttribute(attribute); + + try { + ret = AttributeInfo.fromJson(jsonString); + } catch (JSONException excp) { + LOG.error("failed in converting to AttributeDefinition: " + jsonString, excp); + } + + return ret; + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/acfe9a48/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/GraphMutationContext.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/GraphMutationContext.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/GraphMutationContext.java index 0a1b91d..7dbedc7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/GraphMutationContext.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/GraphMutationContext.java @@ -160,11 +160,11 @@ public class GraphMutationContext { } public AtlasStructType getParentType() { - return attribute.getStructType(); + return attribute.getDefinedInType(); } public AtlasStructDef getStructDef() { - return attribute.getStructDef(); + return attribute.getDefinedInDef(); } public AtlasAttributeDef getAttributeDef() { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/acfe9a48/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java index 7923a36..5c977c1 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java @@ -93,6 +93,10 @@ public class TypesUtil { return new StructTypeDefinition(name, description, attrDefs); } + public static StructTypeDefinition createStructTypeDef(String name, String description, String version, AttributeDefinition... attrDefs) { + return new StructTypeDefinition(name, description, version, attrDefs); + } + public static HierarchicalTypeDefinition<ClassType> createClassTypeDef(String name, ImmutableSet<String> superTypes, AttributeDefinition... attrDefs) { return createClassTypeDef(name, null, superTypes, attrDefs); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/acfe9a48/webapp/src/main/java/org/apache/atlas/util/RestUtils.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/util/RestUtils.java b/webapp/src/main/java/org/apache/atlas/util/RestUtils.java index f928a06..cecf636 100644 --- a/webapp/src/main/java/org/apache/atlas/util/RestUtils.java +++ b/webapp/src/main/java/org/apache/atlas/util/RestUtils.java @@ -32,9 +32,13 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinali import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef; import org.apache.atlas.model.typedef.AtlasTypeDefHeader; import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.repository.store.graph.v1.AtlasStructDefStoreV1; import org.apache.atlas.type.AtlasArrayType; +import org.apache.atlas.type.AtlasClassificationType; import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasEnumType; import org.apache.atlas.type.AtlasStructType; +import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeUtil; @@ -72,51 +76,72 @@ public final class RestUtils { private RestUtils() {} private static final Logger LOG = LoggerFactory.getLogger(RestUtils.class); - public static TypesDef toTypesDef(AtlasEnumDef enumDef) { + public static TypesDef toTypesDef(AtlasType type, AtlasTypeRegistry typeRegistry) throws AtlasBaseException { + final TypesDef ret; + + if (type instanceof AtlasEnumType) { + ret = RestUtils.enumToTypesDef((AtlasEnumType)type); + } else if (type instanceof AtlasEntityType) { + ret = RestUtils.entityToTypesDef((AtlasEntityType)type, typeRegistry); + } else if (type instanceof AtlasClassificationType) { + ret = RestUtils.classificationToTypesDef((AtlasClassificationType)type, typeRegistry); + } else if (type instanceof AtlasStructType) { + ret = RestUtils.structToTypesDef((AtlasStructType)type, typeRegistry); + } else { + ret = new TypesDef(); + } + + return ret; + } + + private static TypesDef enumToTypesDef(AtlasEnumType enumType) { TypesDef ret = null; - if (enumDef != null) { - String enumName = enumDef.getName(); - String enumDesc = enumDef.getDescription(); - String enumVersion = enumDef.getTypeVersion(); - EnumValue[] enumValues = getEnumValues(enumDef.getElementDefs()); + AtlasEnumDef enumDef = enumType.getEnumDef(); - if (enumName != null && enumValues != null && enumValues.length > 0) { - EnumTypeDefinition enumTypeDef = new EnumTypeDefinition(enumName, enumDesc, enumVersion, enumValues); + String enumName = enumDef.getName(); + String enumDesc = enumDef.getDescription(); + String enumVersion = enumDef.getTypeVersion(); + EnumValue[] enumValues = getEnumValues(enumDef.getElementDefs()); - ret = TypesUtil.getTypesDef(ImmutableList.of(enumTypeDef), - ImmutableList.<StructTypeDefinition>of(), - ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), - ImmutableList.<HierarchicalTypeDefinition<ClassType>>of()); - } + if (enumName != null && enumValues != null && enumValues.length > 0) { + EnumTypeDefinition enumTypeDef = new EnumTypeDefinition(enumName, enumDesc, enumVersion, enumValues); + + ret = TypesUtil.getTypesDef(ImmutableList.of(enumTypeDef), + ImmutableList.<StructTypeDefinition>of(), + ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), + ImmutableList.<HierarchicalTypeDefinition<ClassType>>of()); } return ret; } - public static TypesDef toTypesDef(AtlasStructDef structDef, AtlasTypeRegistry registry) throws AtlasBaseException { - String typeName = structDef.getName(); - String typeDesc = structDef.getDescription(); - AttributeDefinition[] attributes = getAttributes(structDef, registry); - StructTypeDefinition structType = TypesUtil.createStructTypeDef(typeName, typeDesc, attributes); + private static TypesDef structToTypesDef(AtlasStructType structType, AtlasTypeRegistry registry) + throws AtlasBaseException { + String typeName = structType.getStructDef().getName(); + String typeDesc = structType.getStructDef().getDescription(); + String typeVersion = structType.getStructDef().getTypeVersion(); + AttributeDefinition[] attributes = getAttributes(structType, registry); + StructTypeDefinition structTypeDef = TypesUtil.createStructTypeDef(typeName, typeDesc, typeVersion, attributes); TypesDef ret = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), - ImmutableList.of(structType), + ImmutableList.of(structTypeDef), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), ImmutableList.<HierarchicalTypeDefinition<ClassType>>of()); return ret; } - public static TypesDef toTypesDef(AtlasEntityDef entityDef, AtlasTypeRegistry registry) throws AtlasBaseException { - String typeName = entityDef.getName(); - String typeDesc = entityDef.getDescription(); - String typeVersion = entityDef.getTypeVersion(); - ImmutableSet superTypes = ImmutableSet.copyOf(entityDef.getSuperTypes()); - AttributeDefinition[] attributes = getAttributes(entityDef, registry); + private static TypesDef entityToTypesDef(AtlasEntityType entityType, AtlasTypeRegistry registry) + throws AtlasBaseException { + String typeName = entityType.getEntityDef().getName(); + String typeDesc = entityType.getEntityDef().getDescription(); + String typeVersion = entityType.getEntityDef().getTypeVersion(); + ImmutableSet superTypes = ImmutableSet.copyOf(entityType.getEntityDef().getSuperTypes()); + AttributeDefinition[] attributes = getAttributes(entityType, registry); HierarchicalTypeDefinition<ClassType> classType = TypesUtil.createClassTypeDef(typeName, typeDesc, typeVersion, - superTypes, attributes); + superTypes, attributes); TypesDef ret = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), @@ -125,12 +150,13 @@ public final class RestUtils { return ret; } - public static TypesDef toTypesDef(AtlasClassificationDef classifDef, AtlasTypeRegistry registry) throws AtlasBaseException { - String typeName = classifDef.getName(); - String typeDesc = classifDef.getDescription(); - String typeVersion = classifDef.getTypeVersion(); - ImmutableSet superTypes = ImmutableSet.copyOf(classifDef.getSuperTypes()); - AttributeDefinition[] attributes = getAttributes(classifDef, registry); + private static TypesDef classificationToTypesDef(AtlasClassificationType classificationType, + AtlasTypeRegistry registry) throws AtlasBaseException { + String typeName = classificationType.getClassificationDef().getName(); + String typeDesc = classificationType.getClassificationDef().getDescription(); + String typeVersion = classificationType.getClassificationDef().getTypeVersion(); + ImmutableSet superTypes = ImmutableSet.copyOf(classificationType.getClassificationDef().getSuperTypes()); + AttributeDefinition[] attributes = getAttributes(classificationType, registry); HierarchicalTypeDefinition traitType = TypesUtil.createTraitTypeDef(typeName, typeDesc, typeVersion, superTypes, attributes); @@ -441,59 +467,15 @@ public final class RestUtils { return ret; } - private static AttributeDefinition[] getAttributes(AtlasStructDef structDef, AtlasTypeRegistry registry) throws AtlasBaseException { + private static AttributeDefinition[] getAttributes(AtlasStructType structType, AtlasTypeRegistry registry) throws AtlasBaseException { List<AttributeDefinition> ret = new ArrayList<>(); - List<AtlasAttributeDef> attrDefs = structDef.getAttributeDefs(); + List<AtlasAttributeDef> attrDefs = structType.getStructDef().getAttributeDefs(); if (CollectionUtils.isNotEmpty(attrDefs)) { - for (AtlasAttributeDef attrDef : attrDefs) { - String name = attrDef.getName(); - String dataTypeName = attrDef.getTypeName(); - Boolean isUnique = attrDef.getIsUnique(); - Boolean isIndexable = attrDef.getIsIndexable(); - String reverseAttribName = null; - boolean isComposite; - - // Multiplicity mapping - final int lower; - final int upper; - - if (attrDef.getCardinality() == Cardinality.SINGLE) { - lower = attrDef.getIsOptional() ? 0 : 1; - upper = 1; - } else { - if(attrDef.getIsOptional()) { - lower = 0; - } else { - lower = attrDef.getValuesMinCount() < 1 ? 1 : attrDef.getValuesMinCount(); - } + AtlasAttribute attribute = structType.getAttribute(attrDef.getName()); - upper = attrDef.getValuesMaxCount() < 2 ? Integer.MAX_VALUE : attrDef.getValuesMaxCount(); - } - - Multiplicity multiplicity = new Multiplicity(lower, upper, Cardinality.SET.equals(attrDef.getCardinality())); - - // Constraint checks: - // 1. [ mappedFromRef -> isComposite ] - // 2. [ foreignKey(onDelete=cascade) -> reverseAttribute ] - AtlasStructType structType = (AtlasStructType) registry.getType(structDef.getName()); - boolean isForeignKey = structType.isForeignKeyAttribute(attrDef.getName()); - boolean isMappedFromRef = (structType instanceof AtlasEntityType) && ((AtlasEntityType)structType).isMappedFromRefAttribute(attrDef.getName()); - AtlasType attrType = structType.getAttributeType(attrDef.getName()); - - if (attrType != null && isForeignKey) { - if (attrType.getTypeCategory() == TypeCategory.ARRAY) { - attrType = ((AtlasArrayType) attrType).getElementType(); - } - - if (attrType.getTypeCategory() == TypeCategory.ENTITY) { - reverseAttribName = ((AtlasEntityType) attrType). - getMappedFromRefAttribute(structType.getTypeName(), attrDef.getName()); - } - } - isComposite = isMappedFromRef || (isForeignKey && StringUtils.isBlank(reverseAttribName)); - ret.add(new AttributeDefinition(name, dataTypeName, multiplicity, isComposite, isUnique, isIndexable, reverseAttribName)); + ret.add(AtlasStructDefStoreV1.toAttributeDefintion(attribute)); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/acfe9a48/webapp/src/main/java/org/apache/atlas/web/adapters/AtlasStructFormatConverter.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/adapters/AtlasStructFormatConverter.java b/webapp/src/main/java/org/apache/atlas/web/adapters/AtlasStructFormatConverter.java index f486cda..4158f01 100644 --- a/webapp/src/main/java/org/apache/atlas/web/adapters/AtlasStructFormatConverter.java +++ b/webapp/src/main/java/org/apache/atlas/web/adapters/AtlasStructFormatConverter.java @@ -136,7 +136,7 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter { Object v1Value = null; AtlasFormatConverter attrConverter = null; - if (attrType.getTypeCategory() == TypeCategory.ENTITY && !attr.isContainedAttribute()) { + if (attrType.getTypeCategory() == TypeCategory.ENTITY && !attr.legacyIsComposite()) { attrConverter = new AtlasObjectIdConverter(converterRegistry, typeRegistry); v1Value = attrConverter.fromV2ToV1(v2Value, attrType, context); } else { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/acfe9a48/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java index c45096f..1081c45 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java @@ -21,17 +21,10 @@ package org.apache.atlas.web.resources; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.core.ResourceContext; import org.apache.atlas.AtlasClient; -import org.apache.atlas.AtlasException; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.TypeCategory; -import org.apache.atlas.model.typedef.AtlasClassificationDef; -import org.apache.atlas.model.typedef.AtlasEntityDef; -import org.apache.atlas.model.typedef.AtlasEnumDef; -import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.typesystem.TypesDef; -import org.apache.atlas.typesystem.exception.TypeExistsException; import org.apache.atlas.typesystem.json.TypesSerialization; import org.apache.atlas.util.RestUtils; import org.apache.atlas.utils.AtlasPerfTracer; @@ -231,38 +224,12 @@ public class TypesResource { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesResource.getDefinition(" + typeName + ")"); } - TypesREST typesRest = resourceContext.getResource(TypesREST.class); JSONObject response = new JSONObject(); try { - TypeCategory typeCategory = typeRegistry.getType(typeName).getTypeCategory(); - TypesDef typesDef = null; - - if (typeCategory != null) { - switch (typeCategory) { - case ENUM: - AtlasEnumDef enumDef = typesRest.getEnumDefByName(typeName); - typesDef = RestUtils.toTypesDef(enumDef); - break; - case STRUCT: - AtlasStructDef structDef = typesRest.getStructDefByName(typeName); - typesDef = RestUtils.toTypesDef(structDef, typeRegistry); - break; - case ENTITY: - AtlasEntityDef entityDef = typesRest.getEntityDefByName(typeName); - typesDef = RestUtils.toTypesDef(entityDef, typeRegistry); - break; - case CLASSIFICATION: - AtlasClassificationDef classificationDef = typesRest.getClassificationDefByName(typeName); - typesDef = RestUtils.toTypesDef(classificationDef, typeRegistry); - break; - default: - typesDef = new TypesDef(); - break; - } - } + TypesDef typesDef = RestUtils.toTypesDef(typeRegistry.getType(typeName), typeRegistry);; + String typeDefinition = TypesSerialization.toJson(typesDef); - final String typeDefinition = TypesSerialization.toJson(typesDef); response.put(AtlasClient.TYPENAME, typeName); response.put(AtlasClient.DEFINITION, new JSONObject(typeDefinition)); response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId());
