This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch branch-1.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-1.0 by this push:
new 7c471dc ATLAS-3034: Perf enhancement to avoid unnecessary lookup when
creating new relationships
7c471dc is described below
commit 7c471dc313daf7c9e30e77024ac3e2c94a9a5403
Author: Sarath Subramanian <ssubraman...@hortonworks.com>
AuthorDate: Wed Jan 23 10:35:35 2019 -0800
ATLAS-3034: Perf enhancement to avoid unnecessary lookup when creating new
relationships
(cherry picked from commit a9aa5b0e1b0a7c2867aef16358e3c3c1977bb140)
---
.../store/graph/AtlasRelationshipStore.java | 4 +
.../store/graph/v2/AtlasRelationshipStoreV2.java | 124 +++++++++++----------
.../store/graph/v2/EntityGraphMapper.java | 32 ++----
3 files changed, 77 insertions(+), 83 deletions(-)
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
index 4468951..53e29e1 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
@@ -59,6 +59,10 @@ public interface AtlasRelationshipStore {
AtlasEdge getOrCreate(AtlasVertex end1Vertex, AtlasVertex end2Vertex,
AtlasRelationship relationship) throws AtlasBaseException;
+ AtlasEdge getRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex,
AtlasRelationship relationship);
+
+ AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex
end2Vertex, AtlasRelationship relationship) throws AtlasBaseException;
+
/**
* Retrieve a relationship if it exists or creates a new relationship
instance.
* @param relationship relationship instance definition
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
index 6bc36b3..2791ef3 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
@@ -81,9 +81,8 @@ import static
org.apache.atlas.repository.graph.GraphHelper.getBlockedClassifica
import static
org.apache.atlas.repository.graph.GraphHelper.getClassificationEntityGuid;
import static
org.apache.atlas.repository.graph.GraphHelper.getClassificationName;
import static
org.apache.atlas.repository.graph.GraphHelper.getClassificationVertices;
-import static
org.apache.atlas.repository.graph.GraphHelper.getOutGoingEdgesByLabel;
+import static
org.apache.atlas.repository.graph.GraphHelper.getIncomingEdgesByLabel;
import static org.apache.atlas.repository.graph.GraphHelper.getPropagateTags;
-import static
org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getIdFromVertex;
import static
org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getState;
import static
org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getTypeName;
@@ -117,8 +116,6 @@ public class AtlasRelationshipStoreV2 implements
AtlasRelationshipStore {
AtlasVertex end1Vertex = getVertexFromEndPoint(relationship.getEnd1());
AtlasVertex end2Vertex = getVertexFromEndPoint(relationship.getEnd2());
- validateRelationship(end1Vertex, end2Vertex,
relationship.getTypeName(), relationship.getAttributes());
-
AtlasEdge edge = createRelationship(end1Vertex, end2Vertex,
relationship);
AtlasRelationship ret = edge != null ?
entityRetriever.mapEdgeToAtlasRelationship(edge) : null;
@@ -289,36 +286,37 @@ public class AtlasRelationshipStoreV2 implements
AtlasRelationshipStore {
@Override
public AtlasEdge getOrCreate(AtlasVertex end1Vertex, AtlasVertex
end2Vertex, AtlasRelationship relationship) throws AtlasBaseException {
- AtlasEdge ret = getRelationshipEdge(end1Vertex, end2Vertex,
relationship.getTypeName());
+ AtlasEdge ret = getRelationship(end1Vertex, end2Vertex, relationship);
if (ret == null) {
- validateRelationship(end1Vertex, end2Vertex,
relationship.getTypeName(), relationship.getAttributes());
-
- ret = createRelationship(end1Vertex, end2Vertex, relationship);
+ ret = createRelationship(end1Vertex, end2Vertex, relationship,
false);
}
return ret;
}
@Override
+ public AtlasEdge getRelationship(AtlasVertex fromVertex, AtlasVertex
toVertex, AtlasRelationship relationship) {
+ String relationshipLabel = getRelationshipEdgeLabel(fromVertex,
toVertex, relationship.getTypeName());
+
+ return getRelationshipEdge(fromVertex, toVertex, relationshipLabel);
+ }
+
+ @Override
public AtlasRelationship getOrCreate(AtlasRelationship relationship)
throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> getOrCreate({})", relationship);
}
- validateRelationship(relationship);
-
AtlasVertex end1Vertex =
getVertexFromEndPoint(relationship.getEnd1());
AtlasVertex end2Vertex =
getVertexFromEndPoint(relationship.getEnd2());
AtlasRelationship ret = null;
// check if relationship exists
- AtlasEdge relationshipEdge = getRelationshipEdge(end1Vertex,
end2Vertex, relationship.getTypeName());
+ AtlasEdge relationshipEdge = getRelationship(end1Vertex, end2Vertex,
relationship);
if (relationshipEdge == null) {
- validateRelationship(relationship);
-
- relationshipEdge = createRelationship(end1Vertex, end2Vertex,
relationship);
+ relationshipEdge = createRelationship(end1Vertex, end2Vertex,
relationship, false);
}
if (relationshipEdge != null){
@@ -332,15 +330,26 @@ public class AtlasRelationshipStoreV2 implements
AtlasRelationshipStore {
return ret;
}
- private AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex
end2Vertex, AtlasRelationship relationship) throws AtlasBaseException {
- AtlasEdge ret = null;
+ @Override
+ public AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex
end2Vertex, AtlasRelationship relationship) throws AtlasBaseException {
+ return createRelationship(end1Vertex, end2Vertex, relationship, true);
+ }
+
+ public AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex
end2Vertex, AtlasRelationship relationship, boolean existingRelationshipCheck)
throws AtlasBaseException {
+ AtlasEdge ret;
try {
- ret = getRelationshipEdge(end1Vertex, end2Vertex,
relationship.getTypeName());
+ validateRelationship(end1Vertex, end2Vertex,
relationship.getTypeName(), relationship.getAttributes());
- if (ret != null) {
- throw new
AtlasBaseException(AtlasErrorCode.RELATIONSHIP_ALREADY_EXISTS,
relationship.getTypeName(),
-
AtlasGraphUtilsV2.getIdFromVertex(end1Vertex),
AtlasGraphUtilsV2.getIdFromVertex(end2Vertex));
+ String relationshipLabel = getRelationshipEdgeLabel(end1Vertex,
end2Vertex, relationship.getTypeName());
+
+ if (existingRelationshipCheck) {
+ ret = getRelationshipEdge(end1Vertex, end2Vertex,
relationshipLabel);
+
+ if (ret != null) {
+ throw new
AtlasBaseException(AtlasErrorCode.RELATIONSHIP_ALREADY_EXISTS,
relationship.getTypeName(),
+
AtlasGraphUtilsV2.getIdFromVertex(end1Vertex),
AtlasGraphUtilsV2.getIdFromVertex(end2Vertex));
+ }
}
AtlasRelationshipType relationType =
typeRegistry.getRelationshipTypeByName(relationship.getTypeName());
@@ -349,10 +358,40 @@ public class AtlasRelationshipStoreV2 implements
AtlasRelationshipStore {
AtlasEntityHeader end1Entity =
entityRetriever.toAtlasEntityHeaderWithClassifications(end1Vertex);
AtlasEntityHeader end2Entity =
entityRetriever.toAtlasEntityHeaderWithClassifications(end2Vertex);
- AtlasAuthorizationUtils.verifyAccess(new
AtlasRelationshipAccessRequest(typeRegistry, AtlasPrivilege.RELATIONSHIP_ADD,
relationship.getTypeName(), end1Entity, end2Entity));
+ AtlasAuthorizationUtils.verifyAccess(new
AtlasRelationshipAccessRequest(typeRegistry, AtlasPrivilege.RELATIONSHIP_ADD,
+
relationship.getTypeName(), end1Entity, end2Entity));
}
- ret = createRelationshipEdge(end1Vertex, end2Vertex, relationship);
+ if (existingRelationshipCheck) {
+ ret = graphHelper.getOrCreateEdge(end1Vertex, end2Vertex,
relationshipLabel);
+
+ } else {
+ ret = graphHelper.addEdge(end1Vertex, end2Vertex,
relationshipLabel);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Created relationship edge from [{}] --> [{}]
using edge label: [{}]", getTypeName(end1Vertex), getTypeName(end2Vertex),
relationshipLabel);
+ }
+ }
+
+ // map additional properties to relationship edge
+ if (ret != null) {
+ // Accept a valid (assigned) guid from the supplied
relationship, or generate one.
+ String relationshipGuid = relationship.getGuid();
+ PropagateTags tagPropagation =
getRelationshipTagPropagation(end1Vertex, end2Vertex, relationship);
+ final String guid =
AtlasTypeUtil.isAssignedGuid(relationshipGuid) ? relationshipGuid :
UUID.randomUUID().toString();
+
+ AtlasGraphUtilsV2.setEncodedProperty(ret,
ENTITY_TYPE_PROPERTY_KEY, relationship.getTypeName());
+ AtlasGraphUtilsV2.setEncodedProperty(ret,
RELATIONSHIP_GUID_PROPERTY_KEY, guid);
+ AtlasGraphUtilsV2.setEncodedProperty(ret,
VERSION_PROPERTY_KEY, getRelationshipVersion(relationship));
+ AtlasGraphUtilsV2.setEncodedProperty(ret, PROVENANCE_TYPE_KEY,
relationship.getProvenanceType());
+ AtlasGraphUtilsV2.setEncodedProperty(ret,
RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, tagPropagation.name());
+
+ // blocked propagated classifications
+ handleBlockedClassifications(ret,
relationship.getBlockedPropagatedClassifications());
+
+ // propagate tags
+ deleteDelegate.getHandler().addTagPropagation(ret,
tagPropagation);
+ }
if (MapUtils.isNotEmpty(relationType.getAllAttributes())) {
for (AtlasAttribute attr :
relationType.getAllAttributes().values()) {
@@ -691,10 +730,9 @@ public class AtlasRelationshipStoreV2 implements
AtlasRelationshipStore {
type.getNormalizedValue(relationship);
}
- public AtlasEdge getRelationshipEdge(AtlasVertex fromVertex, AtlasVertex
toVertex, String relationshipType) {
- String relationshipLabel =
getRelationshipEdgeLabel(fromVertex, toVertex, relationshipType);
- Iterator<AtlasEdge> edgesIterator =
getOutGoingEdgesByLabel(fromVertex, relationshipLabel);
- AtlasEdge ret = null;
+ public AtlasEdge getRelationshipEdge(AtlasVertex fromVertex, AtlasVertex
toVertex, String relationshipLabel) {
+ AtlasEdge ret = null;
+ Iterator<AtlasEdge> edgesIterator = getIncomingEdgesByLabel(toVertex,
relationshipLabel);
while (edgesIterator != null && edgesIterator.hasNext()) {
AtlasEdge edge = edgesIterator.next();
@@ -702,8 +740,7 @@ public class AtlasRelationshipStoreV2 implements
AtlasRelationshipStore {
if (edge != null) {
Status status = graphHelper.getStatus(edge);
- if ((status == null || status == ACTIVE) &&
-
StringUtils.equals(getIdFromVertex(edge.getInVertex()),
getIdFromVertex(toVertex))) {
+ if ((status == null || status == ACTIVE) &&
edge.getOutVertex().equals(fromVertex)) {
ret = edge;
break;
}
@@ -733,37 +770,6 @@ public class AtlasRelationshipStoreV2 implements
AtlasRelationshipStore {
return ret;
}
- private AtlasEdge createRelationshipEdge(AtlasVertex fromVertex,
AtlasVertex toVertex, AtlasRelationship relationship) throws
RepositoryException, AtlasBaseException {
- String relationshipLabel = getRelationshipEdgeLabel(fromVertex,
toVertex, relationship.getTypeName());
- PropagateTags tagPropagation =
getRelationshipTagPropagation(fromVertex, toVertex, relationship);
- AtlasEdge ret =
graphHelper.getOrCreateEdge(fromVertex, toVertex, relationshipLabel);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Created relationship edge from [{}] --> [{}] using edge
label: [{}]", getTypeName(fromVertex), getTypeName(toVertex),
relationshipLabel);
- }
-
- // map additional properties to relationship edge
- if (ret != null) {
- // Accept a valid (assigned) guid from the supplied relationship,
or generate one.
- String relationshipGuid = relationship.getGuid();
- final String guid = AtlasTypeUtil.isAssignedGuid(relationshipGuid)
? relationshipGuid : UUID.randomUUID().toString();
-
- AtlasGraphUtilsV2.setEncodedProperty(ret,
ENTITY_TYPE_PROPERTY_KEY, relationship.getTypeName());
- AtlasGraphUtilsV2.setEncodedProperty(ret,
RELATIONSHIP_GUID_PROPERTY_KEY, guid);
- AtlasGraphUtilsV2.setEncodedProperty(ret, VERSION_PROPERTY_KEY,
getRelationshipVersion(relationship));
- AtlasGraphUtilsV2.setEncodedProperty(ret, PROVENANCE_TYPE_KEY,
relationship.getProvenanceType());
- AtlasGraphUtilsV2.setEncodedProperty(ret,
RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, tagPropagation.name());
-
- // blocked propagated classifications
- handleBlockedClassifications(ret,
relationship.getBlockedPropagatedClassifications());
-
- // propagate tags
- deleteDelegate.getHandler().addTagPropagation(ret, tagPropagation);
- }
-
- return ret;
- }
-
private PropagateTags getRelationshipTagPropagation(AtlasVertex
fromVertex, AtlasVertex toVertex, AtlasRelationship relationship) {
AtlasRelationshipType relationshipType =
typeRegistry.getRelationshipTypeByName(relationship.getTypeName());
AtlasRelationshipEndDef endDef1 =
relationshipType.getRelationshipDef().getEndDef1();
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
index 98734cc..afa471f 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
@@ -819,7 +819,6 @@ public class EntityGraphMapper {
AtlasType type =
typeRegistry.getType(AtlasGraphUtilsV2.getTypeName(entityVertex));
AtlasRelationshipEdgeDirection edgeDirection =
ctx.getAttribute().getRelationshipEdgeDirection();
- String edgeLabel =
ctx.getAttribute().getRelationshipEdgeLabel();
if (type instanceof AtlasEntityType) {
AtlasEntityType entityType = (AtlasEntityType) type;
@@ -844,10 +843,17 @@ public class EntityGraphMapper {
fromVertex = entityVertex;
toVertex = attributeVertex;
}
- boolean relationshipExists =
isRelationshipExists(fromVertex, toVertex, edgeLabel);
ret = getOrCreateRelationship(fromVertex, toVertex,
relationshipName, relationshipAttributes);
+ boolean isCreated = GraphHelper.getCreatedTime(ret) ==
RequestContext.get().getRequestTime();
+
+ if (isCreated) {
+ // if relationship did not exist before and new
relationship was created
+ // record entity update on both relationship vertices
+ recordEntityUpdate(attributeVertex);
+ }
+
// for import use the relationship guid provided
if (RequestContext.get().isImportInProgress()) {
String relationshipGuid =
getRelationshipGuid(ctx.getValue());
@@ -856,12 +862,6 @@ public class EntityGraphMapper {
AtlasGraphUtilsV2.setEncodedProperty(ret,
RELATIONSHIP_GUID_PROPERTY_KEY, relationshipGuid);
}
}
-
- // if relationship did not exist before and new
relationship was created
- // record entity update on both relationship vertices
- if (!relationshipExists) {
- recordEntityUpdate(attributeVertex);
- }
}
} else {
// use legacy way to create/update edges
@@ -1851,22 +1851,6 @@ public class EntityGraphMapper {
return relationshipStore.getOrCreate(end1Vertex, end2Vertex, new
AtlasRelationship(relationshipName, relationshipAttributes));
}
- private boolean isRelationshipExists(AtlasVertex fromVertex, AtlasVertex
toVertex, String edgeLabel) {
- boolean ret = false;
- Iterator<AtlasEdge> edges =
graphHelper.getOutGoingEdgesByLabel(fromVertex, edgeLabel);
-
- while (edges != null && edges.hasNext()) {
- AtlasEdge edge = edges.next();
- AtlasVertex inVertex = edge.getInVertex();
-
- if (inVertex != null &&
StringUtils.equals(getIdFromVertex(inVertex), getIdFromVertex(toVertex))) {
- ret = true;
- }
- }
-
- return ret;
- }
-
private void recordEntityUpdate(AtlasVertex vertex) throws
AtlasBaseException {
RequestContext req = RequestContext.get();
