pareshddevalia commented on code in PR #399:
URL: https://github.com/apache/atlas/pull/399#discussion_r2204917199
##########
repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java:
##########
@@ -413,6 +429,131 @@ private void addEntity(AtlasEntityWithExtInfo
entityWithExtInfo, ExportContext c
context.reportProgress();
}
+ public void getEntityGuids(AtlasEntityWithExtInfo entityWithExtInfo,
ExportContext context, RelationshipAttributesExtractor
relationshipAttributesExtractor) throws AtlasBaseException {
+ if
(!context.classificationEntity.containsKey(entityWithExtInfo.getEntity().getGuid()))
{
+ if
(CollectionUtils.isNotEmpty(entityWithExtInfo.getEntity().getClassifications()))
{
+ for (AtlasClassification c :
entityWithExtInfo.getEntity().getClassifications()) {
+ context.classificationEntity
+ .computeIfAbsent(c.getEntityGuid(), key -> new
UniqueList<>())
+ .add(c.getTypeName());
+ }
+ }
+ }
+ String entityGuid = getClassificationLineage(entityWithExtInfo,
context, relationshipAttributesExtractor);
+ if (entityGuid != null) {
+ if (!context.visitedVertices.contains(entityGuid)) {
+ context.visitedVertices.add(entityGuid);
+ getEntityGuids(new
AtlasEntityWithExtInfo(entityGraphRetriever.toAtlasEntity(entityGuid)),
context, relationshipAttributesExtractor);
+ }
+ }
+ }
+
+ public String getClassificationLineage(AtlasEntityWithExtInfo
entityWithExtInfo, ExportContext context, RelationshipAttributesExtractor
relationshipAttributesExtractor) throws AtlasBaseException {
+ Iterable tagPropagationEdge = new ArrayList();
+ Boolean isParent = false;
+ AtlasVertex adjacentVertex = null;
+ AtlasVertex entityVertexStart =
entityGraphRetriever.getEntityVertex(entityWithExtInfo.getEntity().getGuid());
+ if
(CollectionUtils.isNotEmpty(entityWithExtInfo.getEntity().getClassifications()))
{
+ for (AtlasClassification classification :
entityWithExtInfo.getEntity().getClassifications()) {
+ if
(context.propagatedEdgeMap.containsKey(getGuid(entityVertexStart)) &&
Review Comment:
getGuid(entityVertexStart) method is begin called repetitively.
Try to store them and access it.
##########
repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java:
##########
@@ -413,6 +429,131 @@ private void addEntity(AtlasEntityWithExtInfo
entityWithExtInfo, ExportContext c
context.reportProgress();
}
+ public void getEntityGuids(AtlasEntityWithExtInfo entityWithExtInfo,
ExportContext context, RelationshipAttributesExtractor
relationshipAttributesExtractor) throws AtlasBaseException {
+ if
(!context.classificationEntity.containsKey(entityWithExtInfo.getEntity().getGuid()))
{
+ if
(CollectionUtils.isNotEmpty(entityWithExtInfo.getEntity().getClassifications()))
{
+ for (AtlasClassification c :
entityWithExtInfo.getEntity().getClassifications()) {
+ context.classificationEntity
+ .computeIfAbsent(c.getEntityGuid(), key -> new
UniqueList<>())
+ .add(c.getTypeName());
+ }
+ }
+ }
+ String entityGuid = getClassificationLineage(entityWithExtInfo,
context, relationshipAttributesExtractor);
+ if (entityGuid != null) {
+ if (!context.visitedVertices.contains(entityGuid)) {
+ context.visitedVertices.add(entityGuid);
+ getEntityGuids(new
AtlasEntityWithExtInfo(entityGraphRetriever.toAtlasEntity(entityGuid)),
context, relationshipAttributesExtractor);
+ }
+ }
+ }
+
+ public String getClassificationLineage(AtlasEntityWithExtInfo
entityWithExtInfo, ExportContext context, RelationshipAttributesExtractor
relationshipAttributesExtractor) throws AtlasBaseException {
+ Iterable tagPropagationEdge = new ArrayList();
+ Boolean isParent = false;
+ AtlasVertex adjacentVertex = null;
+ AtlasVertex entityVertexStart =
entityGraphRetriever.getEntityVertex(entityWithExtInfo.getEntity().getGuid());
+ if
(CollectionUtils.isNotEmpty(entityWithExtInfo.getEntity().getClassifications()))
{
+ for (AtlasClassification classification :
entityWithExtInfo.getEntity().getClassifications()) {
+ if
(context.propagatedEdgeMap.containsKey(getGuid(entityVertexStart)) &&
+
context.propagatedEdgeMap.get(getGuid(entityVertexStart)).contains(classification.getTypeName()))
{
+ continue;
+ }
+
+
context.propagatedEdgeMap.computeIfAbsent(entityWithExtInfo.getEntity().getGuid(),
key -> new UniqueList<>())
+ .add(classification.getTypeName());
+ //We are iterating on classifications here again because while
traversing
+ // there could be more classifications on entities where
classifications are blocked on edges
+
+ String direction =
relationshipAttributesExtractor.isLineageType(getTypeName(entityVertexStart))
+ ? "__Process.inputs"
+ : "__Process.outputs";
+
+ tagPropagationEdge = entityVertexStart.query()// found 2 edges
+ .direction(direction.equals("__Process.inputs") ?
AtlasEdgeDirection.OUT : AtlasEdgeDirection.IN)
+ .label(direction)
+ .edges();
+ Iterator<AtlasEdge> iterator =
tagPropagationEdge.iterator().hasNext() ? tagPropagationEdge.iterator() : null;
+
+ if (iterator != null) {
+ while (iterator.hasNext()) {
+ AtlasEdge edge = null;
+ AtlasEdge propagationEdge = iterator.next();
+ if
(relationshipAttributesExtractor.isLineageType(getTypeName(entityVertexStart)))
{
+ edge =
checkPropagatedEdge(propagationEdge.getInVertex(),
classification.getTypeName(), classification.getEntityGuid(), edge);
+ //this is getting executed for hive process for
incoming edge from table
+ }
+ else {
+ edge =
checkPropagatedEdge(propagationEdge.getOutVertex(),
classification.getTypeName(), classification.getEntityGuid(), edge);
+ //this is checked for outvertex in case of table
or other propagating entity which is not a process
+ }
+ if (edge == null) {
+ isParent =
checkIfParentVertex(propagationEdge.getInVertex(),
classification.getTypeName());
+ //This is checking if propagated edge is not found
then entity must be a parent of a classification
+ }
+
+ if (edge != null || isParent) {
+ entityLineageProcess(entityVertexStart, context,
propagationEdge,
+ iterator, entityGraphRetriever,
relationshipAttributesExtractor, classification);
+ }
+ }
+ }
+ }
+ }
+ return (adjacentVertex == null) ? null : getGuid(adjacentVertex);
+ }
+
+ private void entityLineageProcess(AtlasVertex entityVertexStart,
ExportContext context,
+ AtlasEdge propagationEdge,
Iterator<AtlasEdge> iterator,
+ EntityGraphRetriever
entityGraphRetriever,
+ RelationshipAttributesExtractor
relationshipAttributesExtractor, AtlasClassification classification) throws
AtlasBaseException {
+ if
(context.classificationEntity.containsKey(getGuid(entityVertexStart)) &&
+
context.classificationEntity.get(getGuid(entityVertexStart)).contains(classification.getTypeName()))
{
+ return;
+ }
+ AtlasVertex outVertex = propagationEdge.getOutVertex();
+ AtlasVertex inVertex = propagationEdge.getInVertex();
+
+ AtlasVertex adjacentVertex =
StringUtils.equals(outVertex.getIdForDisplay(),
entityVertexStart.getIdForDisplay()) ? inVertex : outVertex;
+ AtlasEntity entity =
entityGraphRetriever.toAtlasEntity(getGuid(adjacentVertex));
+ //Here we check if adjacent entity is parent of ongoing classification
+ //Here we check for iterator.
+ if (context.classificationEntity.containsKey(getGuid(adjacentVertex))
&&
+
context.classificationEntity.get(getGuid(adjacentVertex)).contains(classification.getTypeName())
+ || iterator.hasNext()) {
+ if (!context.guidsToProcess.contains(getGuid(adjacentVertex))) {
+ context.guidsToProcess.add(getGuid(adjacentVertex));
+ }
+
+ getEntityGuids(new AtlasEntityWithExtInfo(entity), context,
relationshipAttributesExtractor);
+ //we have to send this to getEntityGuids
+ //to cover any other classifications if applied on adjacent vertex
and got missed incase of blocktag
+ }
+ if (!context.guidsToProcess.contains(getGuid(adjacentVertex))) {
+ context.guidsToProcess.add(getGuid(adjacentVertex));
+ }
+ }
+
+ //This method executes when a process or a table has propagated
classification edge
+ private AtlasEdge checkPropagatedEdge(AtlasVertex vertex, String
classification, String guid, AtlasEdge edgeID) {
+ if (getPropagatedClassificationEdge(vertex, classification, guid) !=
null) {
Review Comment:
same getPropagatedClassificationEdge method called twice
##########
repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java:
##########
@@ -413,6 +429,131 @@ private void addEntity(AtlasEntityWithExtInfo
entityWithExtInfo, ExportContext c
context.reportProgress();
}
+ public void getEntityGuids(AtlasEntityWithExtInfo entityWithExtInfo,
ExportContext context, RelationshipAttributesExtractor
relationshipAttributesExtractor) throws AtlasBaseException {
+ if
(!context.classificationEntity.containsKey(entityWithExtInfo.getEntity().getGuid()))
{
+ if
(CollectionUtils.isNotEmpty(entityWithExtInfo.getEntity().getClassifications()))
{
+ for (AtlasClassification c :
entityWithExtInfo.getEntity().getClassifications()) {
+ context.classificationEntity
+ .computeIfAbsent(c.getEntityGuid(), key -> new
UniqueList<>())
+ .add(c.getTypeName());
+ }
+ }
+ }
+ String entityGuid = getClassificationLineage(entityWithExtInfo,
context, relationshipAttributesExtractor);
+ if (entityGuid != null) {
+ if (!context.visitedVertices.contains(entityGuid)) {
+ context.visitedVertices.add(entityGuid);
+ getEntityGuids(new
AtlasEntityWithExtInfo(entityGraphRetriever.toAtlasEntity(entityGuid)),
context, relationshipAttributesExtractor);
+ }
+ }
+ }
+
+ public String getClassificationLineage(AtlasEntityWithExtInfo
entityWithExtInfo, ExportContext context, RelationshipAttributesExtractor
relationshipAttributesExtractor) throws AtlasBaseException {
+ Iterable tagPropagationEdge = new ArrayList();
+ Boolean isParent = false;
+ AtlasVertex adjacentVertex = null;
+ AtlasVertex entityVertexStart =
entityGraphRetriever.getEntityVertex(entityWithExtInfo.getEntity().getGuid());
+ if
(CollectionUtils.isNotEmpty(entityWithExtInfo.getEntity().getClassifications()))
{
+ for (AtlasClassification classification :
entityWithExtInfo.getEntity().getClassifications()) {
+ if
(context.propagatedEdgeMap.containsKey(getGuid(entityVertexStart)) &&
+
context.propagatedEdgeMap.get(getGuid(entityVertexStart)).contains(classification.getTypeName()))
{
+ continue;
+ }
+
+
context.propagatedEdgeMap.computeIfAbsent(entityWithExtInfo.getEntity().getGuid(),
key -> new UniqueList<>())
+ .add(classification.getTypeName());
+ //We are iterating on classifications here again because while
traversing
+ // there could be more classifications on entities where
classifications are blocked on edges
+
+ String direction =
relationshipAttributesExtractor.isLineageType(getTypeName(entityVertexStart))
Review Comment:
getTypeName(entityVertexStart) method called twice
##########
repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java:
##########
@@ -413,6 +429,131 @@ private void addEntity(AtlasEntityWithExtInfo
entityWithExtInfo, ExportContext c
context.reportProgress();
}
+ public void getEntityGuids(AtlasEntityWithExtInfo entityWithExtInfo,
ExportContext context, RelationshipAttributesExtractor
relationshipAttributesExtractor) throws AtlasBaseException {
+ if
(!context.classificationEntity.containsKey(entityWithExtInfo.getEntity().getGuid()))
{
+ if
(CollectionUtils.isNotEmpty(entityWithExtInfo.getEntity().getClassifications()))
{
+ for (AtlasClassification c :
entityWithExtInfo.getEntity().getClassifications()) {
+ context.classificationEntity
+ .computeIfAbsent(c.getEntityGuid(), key -> new
UniqueList<>())
+ .add(c.getTypeName());
+ }
+ }
+ }
+ String entityGuid = getClassificationLineage(entityWithExtInfo,
context, relationshipAttributesExtractor);
+ if (entityGuid != null) {
+ if (!context.visitedVertices.contains(entityGuid)) {
+ context.visitedVertices.add(entityGuid);
+ getEntityGuids(new
AtlasEntityWithExtInfo(entityGraphRetriever.toAtlasEntity(entityGuid)),
context, relationshipAttributesExtractor);
+ }
+ }
+ }
+
+ public String getClassificationLineage(AtlasEntityWithExtInfo
entityWithExtInfo, ExportContext context, RelationshipAttributesExtractor
relationshipAttributesExtractor) throws AtlasBaseException {
+ Iterable tagPropagationEdge = new ArrayList();
+ Boolean isParent = false;
+ AtlasVertex adjacentVertex = null;
+ AtlasVertex entityVertexStart =
entityGraphRetriever.getEntityVertex(entityWithExtInfo.getEntity().getGuid());
+ if
(CollectionUtils.isNotEmpty(entityWithExtInfo.getEntity().getClassifications()))
{
+ for (AtlasClassification classification :
entityWithExtInfo.getEntity().getClassifications()) {
+ if
(context.propagatedEdgeMap.containsKey(getGuid(entityVertexStart)) &&
+
context.propagatedEdgeMap.get(getGuid(entityVertexStart)).contains(classification.getTypeName()))
{
+ continue;
+ }
+
+
context.propagatedEdgeMap.computeIfAbsent(entityWithExtInfo.getEntity().getGuid(),
key -> new UniqueList<>())
+ .add(classification.getTypeName());
+ //We are iterating on classifications here again because while
traversing
+ // there could be more classifications on entities where
classifications are blocked on edges
+
+ String direction =
relationshipAttributesExtractor.isLineageType(getTypeName(entityVertexStart))
+ ? "__Process.inputs"
+ : "__Process.outputs";
+
+ tagPropagationEdge = entityVertexStart.query()// found 2 edges
Review Comment:
Iterable assigned with new ArrayList(), but it has overwritten with a
query().edges() result, which is not an ArrayList. Remove the unused iterable.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
