This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-0.8
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-0.8 by this push:
new 6ef9521 ATLAS-3037: createOrUpate() API updated to skip entities that
don't have any change in attribute values
6ef9521 is described below
commit 6ef9521a986d8de24b17aab9b1b0f74d25177794
Author: Madhan Neethiraj <mad...@apache.org>
AuthorDate: Sun Jan 27 01:38:55 2019 -0800
ATLAS-3037: createOrUpate() API updated to skip entities that don't have
any change in attribute values
---
.../apache/atlas/model/instance/AtlasStruct.java | 15 +++
.../java/org/apache/atlas/type/AtlasArrayType.java | 127 ++++++++++++++++++
.../org/apache/atlas/type/AtlasBuiltInTypes.java | 101 ++++++++++++++-
.../apache/atlas/type/AtlasClassificationType.java | 11 ++
.../org/apache/atlas/type/AtlasEntityType.java | 12 ++
.../java/org/apache/atlas/type/AtlasMapType.java | 67 ++++++++++
.../org/apache/atlas/type/AtlasStructType.java | 61 +++++++++
.../main/java/org/apache/atlas/type/AtlasType.java | 30 ++++-
.../test/java/org/apache/atlas/TestUtilsV2.java | 143 +++++----------------
.../store/graph/v1/AtlasEntityStoreV1.java | 82 ++++++++++--
.../store/graph/v1/EntityGraphRetriever.java | 12 ++
.../repository/impexp/ExportSkipLineageTest.java | 10 +-
.../impexp/ZipFileResourceTestUtils.java | 2 +-
.../store/graph/v1/AtlasDeleteHandlerV1Test.java | 67 +++++++---
.../store/graph/v1/AtlasEntityStoreV1Test.java | 16 ++-
.../src/test/resources/reporting-v1-full.zip | Bin 10801 -> 11462 bytes
.../web/resources/AdminExportImportTestIT.java | 2 +-
17 files changed, 604 insertions(+), 154 deletions(-)
diff --git
a/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java
b/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java
index 737e1cc..943ea20 100644
--- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java
+++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java
@@ -55,6 +55,9 @@ import org.codehaus.jackson.map.annotate.JsonSerialize;
public class AtlasStruct implements Serializable {
private static final long serialVersionUID = 1L;
+ public static final String KEY_TYPENAME = "typeName";
+ public static final String KEY_ATTRIBUTES = "attributes";
+
public static final String SERIALIZED_DATE_FORMAT_STR =
"yyyyMMdd-HH:mm:ss.SSS-Z";
@Deprecated
public static final DateFormat DATE_FORMATTER = new
SimpleDateFormat(SERIALIZED_DATE_FORMAT_STR);
@@ -87,6 +90,18 @@ public class AtlasStruct implements Serializable {
}
}
+ public AtlasStruct(Map map) {
+ if (map != null) {
+ Object typeName = map.get(KEY_TYPENAME);
+ Map attributes = (map.get(KEY_ATTRIBUTES) instanceof Map) ?
(Map) map.get(KEY_ATTRIBUTES) : map;
+
+ if (typeName != null) {
+ setTypeName(typeName.toString());
+ }
+
+ setAttributes(new HashMap<>(attributes));
+ }
+ }
public String getTypeName() {
return typeName;
}
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java
b/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java
index 89ba220..ee522ae 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java
@@ -28,6 +28,7 @@ import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import static
org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.COUNT_NOT_SET;
@@ -161,6 +162,86 @@ public class AtlasArrayType extends AtlasType {
}
@Override
+ public boolean areEqualValues(Object val1, Object val2, Map<String,
String> guidAssignments) {
+ boolean ret = true;
+
+ if (val1 == null) {
+ ret = isEmptyArrayValue(val2);
+ } else if (val2 == null) {
+ ret = isEmptyArrayValue(val1);
+ } else {
+ if (val1.getClass().isArray() && val2.getClass().isArray()) {
+ int len = Array.getLength(val1);
+
+ if (len != Array.getLength(val2)) {
+ ret = false;
+ } else {
+ for (int i = 0; i < len; i++) {
+ if (!elementType.areEqualValues(Array.get(val1, i),
Array.get(val2, i), guidAssignments)) {
+ ret = false;
+
+ break;
+ }
+ }
+ }
+ } else if ((val1 instanceof Set) && (val2 instanceof Set)) {
+ Set set1 = (Set) val1;
+ Set set2 = (Set) val2;
+
+ if (set1.size() != set2.size()) {
+ ret = false;
+ } else {
+ for (Object elem1 : set1) {
+ boolean foundInSet2 = false;
+
+ for (Object elem2 : set2) {
+ if (elementType.areEqualValues(elem1, elem2,
guidAssignments)) {
+ foundInSet2 = true;
+
+ break;
+ }
+ }
+
+ if (!foundInSet2) {
+ ret = false;
+
+ break;
+ }
+ }
+ }
+ } else {
+ List list1 = getListFromValue(val1);
+
+ if (list1 == null) {
+ ret = false;
+ } else {
+ List list2 = getListFromValue(val2);
+
+ if (list2 == null) {
+ ret = false;
+ } else {
+ int len = list1.size();
+
+ if (len != list2.size()) {
+ ret = false;
+ } else {
+ for (int i = 0; i < len; i++) {
+ if (!elementType.areEqualValues(list1.get(i),
list2.get(i), guidAssignments)) {
+ ret = false;
+
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ @Override
public boolean isValidValueForUpdate(Object obj) {
if (obj != null) {
if (obj instanceof List || obj instanceof Set) {
@@ -435,4 +516,50 @@ public class AtlasArrayType extends AtlasType {
return true;
}
+
+ private boolean isEmptyArrayValue(Object val) {
+ if (val == null) {
+ return true;
+ } else if (val instanceof Collection) {
+ return ((Collection) val).isEmpty();
+ } else if (val.getClass().isArray()) {
+ return Array.getLength(val) == 0;
+ } else if (val instanceof String){
+ List list = AtlasType.fromJson(val.toString(), List.class);
+
+ return list == null || list.isEmpty();
+ }
+
+ return false;
+ }
+
+ private List getListFromValue(Object val) {
+ final List ret;
+
+ if (val instanceof List) {
+ ret = (List) val;
+ } else if (val instanceof Collection) {
+ int len = ((Collection) val).size();
+
+ ret = new ArrayList<>(len);
+
+ for (Object elem : ((Collection) val)) {
+ ret.add(elem);
+ }
+ } else if (val.getClass().isArray()) {
+ int len = Array.getLength(val);
+
+ ret = new ArrayList<>(len);
+
+ for (int i = 0; i < len; i++) {
+ ret.add(Array.get(val, i));
+ }
+ } else if (val instanceof String){
+ ret = AtlasType.fromJson(val.toString(), List.class);
+ } else {
+ ret = null;
+ }
+
+ return ret;
+ }
}
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java
b/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java
index 1039de6..95dbc86 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java
@@ -29,6 +29,7 @@ import java.math.BigInteger;
import java.text.ParseException;
import java.util.Date;
import java.util.Map;
+import java.util.Objects;
/**
@@ -255,6 +256,7 @@ public class AtlasBuiltInTypes {
*/
public static class AtlasFloatType extends AtlasType {
private static final Float DEFAULT_VALUE = 0f;
+ private static final Float FLOAT_EPSILON = 0.00000001f;
public AtlasFloatType() {
super(AtlasBaseTypeDef.ATLAS_TYPE_FLOAT, TypeCategory.PRIMITIVE);
@@ -275,6 +277,33 @@ public class AtlasBuiltInTypes {
}
@Override
+ public boolean areEqualValues(Object val1, Object val2, Map<String,
String> guidAssignments) {
+ final boolean ret;
+
+ if (val1 == null) {
+ ret = val2 == null;
+ } else if (val2 == null) {
+ ret = false;
+ } else {
+ Float floatVal1 = getNormalizedValue(val1);
+
+ if (floatVal1 == null) {
+ ret = false;
+ } else {
+ Float floatVal2 = getNormalizedValue(val2);
+
+ if (floatVal2 == null) {
+ ret = false;
+ } else {
+ ret = Math.abs(floatVal1 - floatVal2) < FLOAT_EPSILON;
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ @Override
public Float getNormalizedValue(Object obj) {
if (obj != null) {
if (obj instanceof Float) {
@@ -303,7 +332,8 @@ public class AtlasBuiltInTypes {
* class that implements behaviour of double type.
*/
public static class AtlasDoubleType extends AtlasType {
- private static final Double DEFAULT_VALUE = 0d;
+ private static final Double DEFAULT_VALUE = 0d;
+ private static final Double DOUBLE_EPSILON = 0.00000001d;
public AtlasDoubleType() {
super(AtlasBaseTypeDef.ATLAS_TYPE_DOUBLE, TypeCategory.PRIMITIVE);
@@ -324,6 +354,33 @@ public class AtlasBuiltInTypes {
}
@Override
+ public boolean areEqualValues(Object val1, Object val2, Map<String,
String> guidAssignments) {
+ final boolean ret;
+
+ if (val1 == null) {
+ ret = val2 == null;
+ } else if (val2 == null) {
+ ret = false;
+ } else {
+ Double doubleVal1 = getNormalizedValue(val1);
+
+ if (doubleVal1 == null) {
+ ret = false;
+ } else {
+ Double doubleVal2 = getNormalizedValue(val2);
+
+ if (doubleVal2 == null) {
+ ret = false;
+ } else {
+ ret = Math.abs(doubleVal1 - doubleVal2) <
DOUBLE_EPSILON;
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ @Override
public Double getNormalizedValue(Object obj) {
Double ret;
@@ -570,6 +627,48 @@ public class AtlasBuiltInTypes {
}
@Override
+ public boolean areEqualValues(Object val1, Object val2, Map<String,
String> guidAssignments) {
+ boolean ret = true;
+
+ if (val1 == null) {
+ ret = val2 == null;
+ } else if (val2 == null) {
+ ret = false;
+ } else {
+ AtlasObjectId v1 = getNormalizedValue(val1);
+ AtlasObjectId v2 = getNormalizedValue(val2);
+
+ if (v1 == null || v2 == null) {
+ ret = false;
+ } else {
+ String guid1 = v1.getGuid();
+ String guid2 = v2.getGuid();
+
+ if (guidAssignments != null ) {
+ if (guidAssignments.containsKey(guid1)) {
+ guid1 = guidAssignments.get(guid1);
+ }
+
+ if (guidAssignments.containsKey(guid2)) {
+ guid2 = guidAssignments.get(guid2);
+ }
+ }
+
+ boolean isV1AssignedGuid =
AtlasTypeUtil.isAssignedGuid(guid1);
+ boolean isV2AssignedGuid =
AtlasTypeUtil.isAssignedGuid(guid2);
+
+ if (isV1AssignedGuid == isV2AssignedGuid) { // if both
have assigned/unassigned guids, compare guids
+ ret = Objects.equals(guid1, guid2);
+ } else { // if one has assigned and other unassigned guid,
compare typeName and unique-attribute
+ ret = Objects.equals(v1.getTypeName(),
v2.getTypeName()) && Objects.equals(v1.getUniqueAttributes(),
v2.getUniqueAttributes());
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ @Override
public AtlasObjectId getNormalizedValue(Object obj) {
if (obj != null) {
if (obj instanceof AtlasObjectId) {
diff --git
a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
index 707d7b2..66f23d8 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
@@ -200,6 +200,17 @@ public class AtlasClassificationType extends
AtlasStructType {
}
@Override
+ public boolean areEqualValues(Object val1, Object val2, Map<String,
String> guidAssignments) {
+ for (AtlasClassificationType superType : superTypes) {
+ if (!superType.areEqualValues(val1, val2, guidAssignments)) {
+ return false;
+ }
+ }
+
+ return super.areEqualValues(val1, val2, guidAssignments);
+ }
+
+ @Override
public boolean isValidValueForUpdate(Object obj) {
if (obj != null) {
for (AtlasClassificationType superType : superTypes) {
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
index e29f3a4..bc080cc 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
@@ -234,6 +234,18 @@ public class AtlasEntityType extends AtlasStructType {
}
@Override
+ public boolean areEqualValues(Object val1, Object val2, Map<String,
String> guidAssignments) {
+ for (AtlasEntityType superType : superTypes) {
+ if (!superType.areEqualValues(val1, val2, guidAssignments)) {
+ return false;
+ }
+ }
+
+ return super.areEqualValues(val1, val2, guidAssignments);
+ }
+
+
+ @Override
public boolean isValidValueForUpdate(Object obj) {
if (obj != null) {
for (AtlasEntityType superType : superTypes) {
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java
b/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java
index 8b702a7..4451e8b 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java
@@ -124,6 +124,45 @@ public class AtlasMapType extends AtlasType {
}
@Override
+ public boolean areEqualValues(Object val1, Object val2, Map<String,
String> guidAssignments) {
+ boolean ret = true;
+
+ if (val1 == null) {
+ ret = isEmptyMapValue(val2);
+ } else if (val2 == null) {
+ ret = isEmptyMapValue(val1);
+ } else {
+ Map map1 = getMapFromValue(val1);
+
+ if (map1 == null) {
+ ret = false;
+ } else {
+ Map map2 = getMapFromValue(val2);
+
+ if (map2 == null) {
+ ret = false;
+ } else {
+ int len = map1.size();
+
+ if (len != map2.size()) {
+ ret = false;
+ } else {
+ for (Object key : map1.keySet()) {
+ if (!valueType.areEqualValues(map1.get(key),
map2.get(key), guidAssignments)) {
+ ret = false;
+
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ @Override
public boolean isValidValueForUpdate(Object obj) {
if (obj != null) {
if (obj instanceof Map) {
@@ -299,4 +338,32 @@ public class AtlasMapType extends AtlasType {
return attributeType;
}
}
+
+ private boolean isEmptyMapValue(Object val) {
+ if (val == null) {
+ return true;
+ } else if (val instanceof Map) {
+ return ((Map) val).isEmpty();
+ } else if (val instanceof String) {
+ Map map = AtlasType.fromJson(val.toString(), Map.class);
+
+ return map == null || map.isEmpty();
+ }
+
+ return false;
+ }
+
+ private Map getMapFromValue(Object val) {
+ final Map ret;
+
+ if (val instanceof Map) {
+ ret = (Map) val;
+ } else if (val instanceof String) {
+ ret = AtlasType.fromJson(val.toString(), Map.class);
+ } else {
+ ret = null;
+ }
+
+ return ret;
+ }
}
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
index 45c58bd..8d44a82 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
@@ -228,6 +228,44 @@ public class AtlasStructType extends AtlasType {
}
@Override
+ public boolean areEqualValues(Object val1, Object val2, Map<String,
String> guidAssignments) {
+ boolean ret = true;
+
+ if (val1 == null) {
+ ret = val2 == null;
+ } else if (val2 == null) {
+ ret = false;
+ } else {
+ AtlasStruct structVal1 = getStructFromValue(val1);
+
+ if (structVal1 == null) {
+ ret = false;
+ } else {
+ AtlasStruct structVal2 = getStructFromValue(val2);
+
+ if (structVal2 == null) {
+ ret = false;
+ } else if
(!StringUtils.equalsIgnoreCase(structVal1.getTypeName(),
structVal2.getTypeName())) {
+ ret = false;
+ } else {
+ for (AtlasAttribute attribute :
getAllAttributes().values()) {
+ Object attrValue1 =
structVal1.getAttribute(attribute.getName());
+ Object attrValue2 =
structVal2.getAttribute(attribute.getName());
+
+ if
(!attribute.getAttributeType().areEqualValues(attrValue1, attrValue2,
guidAssignments)) {
+ ret = false;
+
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ @Override
public boolean isValidValueForUpdate(Object obj) {
if (obj != null) {
Map<String, Object> attributes;
@@ -601,6 +639,29 @@ public class AtlasStructType extends AtlasType {
return Collections.unmodifiableMap(ret);
}
+ private AtlasStruct getStructFromValue(Object val) {
+ final AtlasStruct ret;
+
+ if (val instanceof AtlasStruct) {
+ ret = (AtlasStruct) val;
+ } else if (val instanceof Map) {
+ ret = new AtlasStruct((Map) val);
+ } else if (val instanceof String) {
+ Map map = AtlasType.fromJson(val.toString(), Map.class);
+
+ if (map == null) {
+ ret = null;
+ } else {
+ ret = new AtlasStruct((Map) val);
+ }
+ } else {
+ ret = null;
+ }
+
+ return ret;
+ }
+
+
public static class AtlasAttribute {
private final AtlasStructType definedInType;
private final AtlasType attributeType;
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasType.java
b/intg/src/main/java/org/apache/atlas/type/AtlasType.java
index ae7044d..e9b89b2 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasType.java
@@ -26,8 +26,8 @@ import org.codehaus.jackson.map.ObjectMapper;
import java.io.IOException;
import java.util.List;
-
-
+import java.util.Map;
+import java.util.Objects;
/**
@@ -72,6 +72,32 @@ public abstract class AtlasType {
public abstract boolean isValidValue(Object obj);
+ public boolean areEqualValues(Object val1, Object val2, Map<String,
String> guidAssignments) {
+ final boolean ret;
+
+ if (val1 == null) {
+ ret = val2 == null;
+ } else if (val2 == null) {
+ ret = false;
+ } else {
+ Object normalizedVal1 = getNormalizedValue(val1);
+
+ if (normalizedVal1 == null) {
+ ret = false;
+ } else {
+ Object normalizedVal2 = getNormalizedValue(val2);
+
+ if (normalizedVal2 == null) {
+ ret = false;
+ } else {
+ ret = Objects.equals(normalizedVal1, normalizedVal2);
+ }
+ }
+ }
+
+ return ret;
+ }
+
public abstract Object getNormalizedValue(Object obj);
public boolean validateValue(Object obj, String objName, List<String>
messages) {
diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
index 18f8b25..1c78172 100755
--- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
+++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
@@ -86,22 +86,22 @@ public final class TestUtilsV2 {
));
AtlasStructDef addressDetails =
- createStructTypeDef("Address", "Address"+_description,
+ createStructTypeDef(ADDRESS_TYPE, "Address"+_description,
AtlasTypeUtil.createRequiredAttrDef("street",
"string"),
AtlasTypeUtil.createRequiredAttrDef("city", "string"));
AtlasEntityDef deptTypeDef =
AtlasTypeUtil.createClassTypeDef(DEPARTMENT_TYPE,
"Department"+_description, ImmutableSet.<String>of(),
AtlasTypeUtil.createUniqueRequiredAttrDef("name",
"string"),
- new AtlasAttributeDef("employees",
String.format("array<%s>", "Employee"), true,
+ new AtlasAttributeDef("employees",
String.format("array<%s>", EMPLOYEE_TYPE), true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
new ArrayList<AtlasStructDef.AtlasConstraintDef>()
{{
add(new
AtlasStructDef.AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF));
}}));
- AtlasEntityDef personTypeDef =
AtlasTypeUtil.createClassTypeDef("Person", "Person"+_description,
ImmutableSet.<String>of(),
+ AtlasEntityDef personTypeDef =
AtlasTypeUtil.createClassTypeDef(PERSON_TYPE, "Person"+_description,
ImmutableSet.<String>of(),
AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"),
- AtlasTypeUtil.createOptionalAttrDef("address", "Address"),
+ AtlasTypeUtil.createOptionalAttrDef("address", ADDRESS_TYPE),
AtlasTypeUtil.createOptionalAttrDef("birthday", "date"),
AtlasTypeUtil.createOptionalAttrDef("hasPets", "boolean"),
AtlasTypeUtil.createOptionalAttrDef("numberOfCars", "byte"),
@@ -112,13 +112,13 @@ public final class TestUtilsV2 {
AtlasTypeUtil.createOptionalAttrDef("approximationOfPi",
"bigdecimal")
);
- AtlasEntityDef employeeTypeDef =
AtlasTypeUtil.createClassTypeDef("Employee", "Employee"+_description,
ImmutableSet.of("Person"),
+ AtlasEntityDef employeeTypeDef =
AtlasTypeUtil.createClassTypeDef(EMPLOYEE_TYPE, "Employee"+_description,
ImmutableSet.of(PERSON_TYPE),
AtlasTypeUtil.createOptionalAttrDef("orgLevel", "OrgLevel"),
- new AtlasAttributeDef("department", "Department", false,
+ new AtlasAttributeDef("department", DEPARTMENT_TYPE, false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
false, false,
new ArrayList<AtlasConstraintDef>()),
- new AtlasAttributeDef("manager", "Manager", true,
+ new AtlasAttributeDef("manager", MANAGER_TYPE, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
new ArrayList<AtlasConstraintDef>() {{
@@ -141,8 +141,8 @@ public final class TestUtilsV2 {
put(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE,
"employees");
}}));
- AtlasEntityDef managerTypeDef =
AtlasTypeUtil.createClassTypeDef("Manager", "Manager"+_description,
ImmutableSet.of("Employee"),
- new AtlasAttributeDef("subordinates",
String.format("array<%s>", "Employee"), false,
AtlasAttributeDef.Cardinality.SET,
+ AtlasEntityDef managerTypeDef =
AtlasTypeUtil.createClassTypeDef(MANAGER_TYPE, "Manager"+_description,
ImmutableSet.of(EMPLOYEE_TYPE),
+ new AtlasAttributeDef("subordinates",
String.format("array<%s>", EMPLOYEE_TYPE), false,
AtlasAttributeDef.Cardinality.SET,
1, 10, false, false,
Collections.<AtlasConstraintDef>emptyList()));
@@ -199,7 +199,7 @@ public final class TestUtilsV2 {
));
AtlasStructDef addressDetails =
- createStructTypeDef("Address", "Address"+_description,
+ createStructTypeDef(ADDRESS_TYPE, "Address"+_description,
AtlasTypeUtil.createRequiredAttrDef("street",
"string"),
AtlasTypeUtil.createRequiredAttrDef("city", "string"),
AtlasTypeUtil.createOptionalAttrDef("zip", "int"));
@@ -209,17 +209,17 @@ public final class TestUtilsV2 {
ImmutableSet.<String>of(),
AtlasTypeUtil.createUniqueRequiredAttrDef("name",
"string"),
AtlasTypeUtil.createOptionalAttrDef("dep-code",
"string"),
- new AtlasAttributeDef("employees",
String.format("array<%s>", "Employee"), true,
+ new AtlasAttributeDef("employees",
String.format("array<%s>", EMPLOYEE_TYPE), true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
new ArrayList<AtlasStructDef.AtlasConstraintDef>()
{{
add(new
AtlasStructDef.AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF));
}}));
- AtlasEntityDef personTypeDef =
AtlasTypeUtil.createClassTypeDef("Person", "Person"+_description,
+ AtlasEntityDef personTypeDef =
AtlasTypeUtil.createClassTypeDef(PERSON_TYPE, "Person"+_description,
ImmutableSet.<String>of(),
AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"),
AtlasTypeUtil.createOptionalAttrDef("email", "string"),
- AtlasTypeUtil.createOptionalAttrDef("address", "Address"),
+ AtlasTypeUtil.createOptionalAttrDef("address", ADDRESS_TYPE),
AtlasTypeUtil.createOptionalAttrDef("birthday", "date"),
AtlasTypeUtil.createOptionalAttrDef("hasPets", "boolean"),
AtlasTypeUtil.createOptionalAttrDef("numberOfCars", "byte"),
@@ -230,15 +230,15 @@ public final class TestUtilsV2 {
AtlasTypeUtil.createOptionalAttrDef("approximationOfPi",
"bigdecimal")
);
- AtlasEntityDef employeeTypeDef =
AtlasTypeUtil.createClassTypeDef("Employee", "Employee"+_description,
- ImmutableSet.of("Person"),
+ AtlasEntityDef employeeTypeDef =
AtlasTypeUtil.createClassTypeDef(EMPLOYEE_TYPE, "Employee"+_description,
+ ImmutableSet.of(PERSON_TYPE),
AtlasTypeUtil.createOptionalAttrDef("orgLevel", "OrgLevel"),
AtlasTypeUtil.createOptionalAttrDef("empCode", "string"),
- new AtlasAttributeDef("department", "Department", false,
+ new AtlasAttributeDef("department", DEPARTMENT_TYPE, false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
false, false,
Collections.<AtlasConstraintDef>emptyList()),
- new AtlasAttributeDef("manager", "Manager", true,
+ new AtlasAttributeDef("manager", MANAGER_TYPE, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
new ArrayList<AtlasConstraintDef>() {{
@@ -256,9 +256,9 @@ public final class TestUtilsV2 {
);
- AtlasEntityDef managerTypeDef =
AtlasTypeUtil.createClassTypeDef("Manager", "Manager"+_description,
- ImmutableSet.of("Employee"),
- new AtlasAttributeDef("subordinates",
String.format("array<%s>", "Employee"), false,
AtlasAttributeDef.Cardinality.SET,
+ AtlasEntityDef managerTypeDef =
AtlasTypeUtil.createClassTypeDef(MANAGER_TYPE, "Manager"+_description,
+ ImmutableSet.of(EMPLOYEE_TYPE),
+ new AtlasAttributeDef("subordinates",
String.format("array<%s>", EMPLOYEE_TYPE), false,
AtlasAttributeDef.Cardinality.SET,
1, 10, false, false,
Collections.<AtlasConstraintDef>emptyList()));
@@ -288,7 +288,7 @@ public final class TestUtilsV2 {
));
AtlasStructDef addressDetails =
- createStructTypeDef("Address", "Address"+_description,
+ createStructTypeDef(ADDRESS_TYPE, "Address"+_description,
AtlasTypeUtil.createRequiredAttrDef("street",
"string"),
AtlasTypeUtil.createRequiredAttrDef("city", "string"),
AtlasTypeUtil.createRequiredAttrDef("zip", "int"));
@@ -297,22 +297,22 @@ public final class TestUtilsV2 {
AtlasTypeUtil.createClassTypeDef(DEPARTMENT_TYPE,
"Department"+_description, ImmutableSet.<String>of(),
AtlasTypeUtil.createRequiredAttrDef("name", "string"),
AtlasTypeUtil.createRequiredAttrDef("dep-code",
"string"),
- new AtlasAttributeDef("employees",
String.format("array<%s>", "Person"), true,
+ new AtlasAttributeDef("employees",
String.format("array<%s>", PERSON_TYPE), true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
new ArrayList<AtlasStructDef.AtlasConstraintDef>()
{{
add(new
AtlasStructDef.AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF));
}}));
- AtlasEntityDef personTypeDef =
AtlasTypeUtil.createClassTypeDef("Person", "Person"+_description,
ImmutableSet.<String>of(),
+ AtlasEntityDef personTypeDef =
AtlasTypeUtil.createClassTypeDef(PERSON_TYPE, "Person"+_description,
ImmutableSet.<String>of(),
AtlasTypeUtil.createRequiredAttrDef("name", "string"),
AtlasTypeUtil.createRequiredAttrDef("emp-code", "string"),
AtlasTypeUtil.createOptionalAttrDef("orgLevel", "OrgLevel"),
- AtlasTypeUtil.createOptionalAttrDef("address", "Address"),
- new AtlasAttributeDef("department", "Department", false,
+ AtlasTypeUtil.createOptionalAttrDef("address", ADDRESS_TYPE),
+ new AtlasAttributeDef("department", DEPARTMENT_TYPE, false,
AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
false, false,
Collections.<AtlasConstraintDef>emptyList()),
- new AtlasAttributeDef("manager", "Manager", true,
+ new AtlasAttributeDef("manager", MANAGER_TYPE, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
new ArrayList<AtlasConstraintDef>() {{
@@ -321,7 +321,7 @@ public final class TestUtilsV2 {
put(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE,
"subordinates");
}}));
}}),
- new AtlasAttributeDef("mentor", "Person", true,
+ new AtlasAttributeDef("mentor", PERSON_TYPE, true,
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
Collections.<AtlasConstraintDef>emptyList()),
@@ -348,6 +348,7 @@ public final class TestUtilsV2 {
}
public static final String DEPARTMENT_TYPE = "Department";
+ public static final String PERSON_TYPE = "Person";
public static final String EMPLOYEE_TYPE = "Employee";
public static final String MANAGER_TYPE = "Manager";
public static final String ADDRESS_TYPE = "Address";
@@ -441,92 +442,6 @@ public final class TestUtilsV2 {
return entitiesWithExtInfo;
}
- public static Map<String, AtlasEntity> createDeptEg1() {
- Map<String, AtlasEntity> deptEmpEntities = new HashMap<>();
-
- AtlasEntity hrDept = new AtlasEntity(DEPARTMENT_TYPE);
- AtlasEntity john = new AtlasEntity(EMPLOYEE_TYPE);
-
- AtlasEntity jane = new AtlasEntity("Manager");
- AtlasEntity johnAddr = new AtlasEntity("Address");
- AtlasEntity janeAddr = new AtlasEntity("Address");
- AtlasEntity julius = new AtlasEntity("Manager");
- AtlasEntity juliusAddr = new AtlasEntity("Address");
- AtlasEntity max = new AtlasEntity(EMPLOYEE_TYPE);
- AtlasEntity maxAddr = new AtlasEntity("Address");
-
- AtlasObjectId deptId = new AtlasObjectId(hrDept.getGuid(),
hrDept.getTypeName());
- hrDept.setAttribute("name", "hr");
- john.setAttribute("name", "John");
- john.setAttribute("department", deptId);
- johnAddr.setAttribute("street", "Stewart Drive");
- johnAddr.setAttribute("city", "Sunnyvale");
- john.setAttribute("address", johnAddr);
-
- john.setAttribute("birthday",new Date(1950, 5, 15));
- john.setAttribute("hasPets", true);
- john.setAttribute("numberOfCars", 1);
- john.setAttribute("houseNumber", 153);
- john.setAttribute("carMileage", 13364);
- john.setAttribute("shares", 15000);
- john.setAttribute("salary", 123345.678);
- john.setAttribute("age", 50);
- john.setAttribute("numberOfStarsEstimate", new
BigInteger("1000000000000000000000"));
- john.setAttribute("approximationOfPi", new
BigDecimal("3.141592653589793238462643383279502884197169399375105820974944592307816406286"));
-
- jane.setAttribute("name", "Jane");
- jane.setAttribute("department", deptId);
- janeAddr.setAttribute("street", "Great America Parkway");
- janeAddr.setAttribute("city", "Santa Clara");
- jane.setAttribute("address", janeAddr);
- janeAddr.setAttribute("street", "Great America Parkway");
-
- julius.setAttribute("name", "Julius");
- julius.setAttribute("department", deptId);
- juliusAddr.setAttribute("street", "Madison Ave");
- juliusAddr.setAttribute("city", "Newtonville");
- julius.setAttribute("address", juliusAddr);
- julius.setAttribute("subordinates", ImmutableList.of());
-
- AtlasObjectId janeId = AtlasTypeUtil.getAtlasObjectId(jane);
- AtlasObjectId johnId = AtlasTypeUtil.getAtlasObjectId(john);
-
- //TODO - Change to MANAGER_TYPE for JULIUS
- AtlasObjectId maxId = new AtlasObjectId(max.getGuid(), EMPLOYEE_TYPE);
- AtlasObjectId juliusId = new AtlasObjectId(julius.getGuid(),
EMPLOYEE_TYPE);
-
- max.setAttribute("name", "Max");
- max.setAttribute("department", deptId);
- maxAddr.setAttribute("street", "Ripley St");
- maxAddr.setAttribute("city", "Newton");
- max.setAttribute("address", maxAddr);
- max.setAttribute("manager", janeId);
- max.setAttribute("mentor", juliusId);
- max.setAttribute("birthday",new Date(1979, 3, 15));
- max.setAttribute("hasPets", true);
- max.setAttribute("age", 36);
- max.setAttribute("numberOfCars", 2);
- max.setAttribute("houseNumber", 17);
- max.setAttribute("carMileage", 13);
- max.setAttribute("shares", Long.MAX_VALUE);
- max.setAttribute("salary", Double.MAX_VALUE);
- max.setAttribute("numberOfStarsEstimate", new
BigInteger("1000000000000000000000000000000"));
- max.setAttribute("approximationOfPi", new
BigDecimal("3.1415926535897932"));
-
- john.setAttribute("manager", janeId);
- john.setAttribute("mentor", maxId);
- hrDept.setAttribute("employees", ImmutableList.of(johnId, janeId,
juliusId, maxId));
-
- jane.setAttribute("subordinates", ImmutableList.of(johnId, maxId));
-
- deptEmpEntities.put(jane.getGuid(), jane);
- deptEmpEntities.put(john.getGuid(), john);
- deptEmpEntities.put(julius.getGuid(), julius);
- deptEmpEntities.put(max.getGuid(), max);
- deptEmpEntities.put(deptId.getGuid(), hrDept);
- return deptEmpEntities;
- }
-
public static final String DATABASE_TYPE = "hive_database";
public static final String DATABASE_NAME = "foo";
public static final String TABLE_TYPE = "hive_table";
@@ -1132,7 +1047,7 @@ public final class TestUtilsV2 {
}
public static List<AtlasEntityDef> getEntityWithValidSuperType() {
- AtlasEntityDef developerTypeDef =
AtlasTypeUtil.createClassTypeDef("Developer", "Developer_description",
ImmutableSet.of("Employee"),
+ AtlasEntityDef developerTypeDef =
AtlasTypeUtil.createClassTypeDef("Developer", "Developer_description",
ImmutableSet.of(EMPLOYEE_TYPE),
new AtlasAttributeDef("language", String.format("array<%s>",
"string"), false, AtlasAttributeDef.Cardinality.SET,
1, 10, false, false,
Collections.<AtlasConstraintDef>emptyList()));
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
index 1d89c83..de328e9 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
@@ -56,6 +56,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import static
org.apache.atlas.model.instance.EntityMutations.EntityOperation.DELETE;
import static
org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE;
@@ -69,14 +70,16 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore
{
private final AtlasTypeRegistry typeRegistry;
private final AtlasEntityChangeNotifier entityChangeNotifier;
private final EntityGraphMapper entityGraphMapper;
+ private final EntityGraphRetriever entityRetriever;
@Inject
public AtlasEntityStoreV1(DeleteHandlerDelegateV1 deleteDelegate,
AtlasTypeRegistry typeRegistry,
- AtlasEntityChangeNotifier entityChangeNotifier,
EntityGraphMapper entityGraphMapper) {
+ AtlasEntityChangeNotifier entityChangeNotifier,
EntityGraphMapper entityGraphMapper, EntityGraphRetriever entityRetriever) {
this.deleteDelegate = deleteDelegate;
this.typeRegistry = typeRegistry;
this.entityChangeNotifier = entityChangeNotifier;
this.entityGraphMapper = entityGraphMapper;
+ this.entityRetriever = entityRetriever;
}
@Override
@@ -92,8 +95,6 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
LOG.debug("==> getById({}, {})", guid, isMinExtInfo);
}
- EntityGraphRetriever entityRetriever = new
EntityGraphRetriever(typeRegistry);
-
AtlasEntityWithExtInfo ret =
entityRetriever.toAtlasEntityWithExtInfo(guid, isMinExtInfo);
if (ret == null) {
@@ -114,8 +115,6 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore
{
LOG.debug("==> getHeaderById({})", guid);
}
- EntityGraphRetriever entityRetriever = new
EntityGraphRetriever(typeRegistry);
-
AtlasEntityHeader ret = entityRetriever.toAtlasEntityHeader(guid);
if (ret == null) {
@@ -142,8 +141,6 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore
{
LOG.debug("==> getByIds({}, {})", guids, isMinExtInfo);
}
- EntityGraphRetriever entityRetriever = new
EntityGraphRetriever(typeRegistry);
-
AtlasEntitiesWithExtInfo ret =
entityRetriever.toAtlasEntitiesWithExtInfo(guids, isMinExtInfo);
if (LOG.isDebugEnabled()) {
@@ -170,8 +167,6 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore
{
AtlasVertex entityVertex =
AtlasGraphUtilsV1.getVertexByUniqueAttributes(entityType, uniqAttributes);
- EntityGraphRetriever entityRetriever = new
EntityGraphRetriever(typeRegistry);
-
AtlasEntityWithExtInfo ret =
entityRetriever.toAtlasEntityWithExtInfo(entityVertex, isMinExtInfo);
if (ret == null) {
@@ -224,6 +219,75 @@ public class AtlasEntityStoreV1 implements
AtlasEntityStore {
// Create/Update entities
EntityMutationContext context = preCreateOrUpdate(entityStream,
entityGraphMapper, isPartialUpdate);
+ // for existing entities, skip update if incoming entity doesn't have
any change
+ if (CollectionUtils.isNotEmpty(context.getUpdatedEntities())) {
+ MetricRecorder checkForUnchangedEntities =
RequestContextV1.get().startMetricRecord("checkForUnchangedEntities");
+
+ List<AtlasEntity> entitiesToSkipUpdate = null;
+
+ for (AtlasEntity entity : context.getUpdatedEntities()) {
+ String guid = entity.getGuid();
+ AtlasVertex vertex = context.getVertex(guid);
+ AtlasEntityType entityType =
typeRegistry.getEntityTypeByName(entity.getTypeName());
+ boolean hasUpdates = false;
+
+ if (!hasUpdates) {
+ hasUpdates = entity.getStatus() ==
AtlasEntity.Status.DELETED; // entity status could be updated during import
+ }
+
+ if (!hasUpdates) {
+ for (AtlasAttribute attribute :
entityType.getAllAttributes().values()) {
+ if
(!entity.getAttributes().containsKey(attribute.getName())) { // if value is
not provided, current value will not be updated
+ continue;
+ }
+
+ Object currVal =
entityRetriever.getEntityAttribute(vertex, attribute);
+
+ Object newVal =
entity.getAttribute(attribute.getName());
+ if
(!attribute.getAttributeType().areEqualValues(currVal, newVal,
context.getGuidAssignments())) {
+ hasUpdates = true;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("found attribute update:
entity(guid={}, typeName={}), attrName={}, currValue={}, newValue={}", guid,
entity.getTypeName(), attribute.getName(), currVal, newVal);
+ }
+
+ break;
+ }
+ }
+ }
+
+ // if classifications are to be replaced, then skip updates
only when no change in classifications
+ if (!hasUpdates && replaceClassifications) {
+ List<AtlasClassification> newVal =
entity.getClassifications();
+ List<AtlasClassification> currVal =
entityRetriever.getClassifications(vertex);
+
+ if (!Objects.equals(currVal, newVal)) {
+ hasUpdates = true;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("found classifications update:
entity(guid={}, typeName={}), currValue={}, newValue={}", guid,
entity.getTypeName(), currVal, newVal);
+ }
+ }
+ }
+
+ if (!hasUpdates) {
+ if (entitiesToSkipUpdate == null) {
+ entitiesToSkipUpdate = new ArrayList<>();
+ }
+
+ LOG.info("skipping unchanged entity: {}", entity);
+
+ entitiesToSkipUpdate.add(entity);
+ }
+ }
+
+ if (entitiesToSkipUpdate != null) {
+ context.getUpdatedEntities().removeAll(entitiesToSkipUpdate);
+ }
+
+ RequestContextV1.get().endMetricRecord(checkForUnchangedEntities);
+ }
+
EntityMutationResponse ret =
entityGraphMapper.mapAttributesAndClassifications(context, isPartialUpdate,
replaceClassifications);
ret.setGuidAssignments(context.getGuidAssignments());
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
index 200f56b..d364556 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
@@ -146,6 +146,18 @@ public final class EntityGraphRetriever {
return toAtlasEntityHeader(entityVertex,
Collections.<String>emptySet());
}
+ public Object getEntityAttribute(AtlasVertex entityVertex, AtlasAttribute
attribute) {
+ Object ret = null;
+
+ try {
+ ret = getVertexAttribute(entityVertex, attribute);
+ } catch (AtlasBaseException excp) {
+ // ignore
+ }
+
+ return ret;
+ }
+
public AtlasVertex getEntityVertex(String guid) throws AtlasBaseException {
AtlasVertex ret = AtlasGraphUtilsV1.findByGuid(guid);
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java
index 524aa1e..eaf4602 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java
@@ -25,10 +25,7 @@ import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.instance.AtlasEntity;
-import org.apache.atlas.repository.store.graph.v1.AtlasEntityChangeNotifier;
-import org.apache.atlas.repository.store.graph.v1.AtlasEntityStoreV1;
-import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegateV1;
-import org.apache.atlas.repository.store.graph.v1.EntityGraphMapper;
+import org.apache.atlas.repository.store.graph.v1.*;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.utils.TestResourceFileUtils;
@@ -62,6 +59,9 @@ public class ExportSkipLineageTest extends
ExportImportTestBase {
private EntityGraphMapper graphMapper;
@Inject
+ private EntityGraphRetriever graphRetriever;
+
+ @Inject
ExportService exportService;
private DeleteHandlerDelegateV1 deleteDelegate =
mock(DeleteHandlerDelegateV1.class);
@@ -73,7 +73,7 @@ public class ExportSkipLineageTest extends
ExportImportTestBase {
loadBaseModel(typeDefStore, typeRegistry);
loadHiveModel(typeDefStore, typeRegistry);
- entityStore = new AtlasEntityStoreV1(deleteDelegate, typeRegistry,
mockChangeNotifier, graphMapper);
+ entityStore = new AtlasEntityStoreV1(deleteDelegate, typeRegistry,
mockChangeNotifier, graphMapper, graphRetriever);
createEntities(entityStore, ENTITIES_SUB_DIR, new String[]{"db",
"table-columns", "table-view", "table-table-lineage"});
final Object[] entityGuids = new Object[]{DB_GUID, TABLE_GUID,
TABLE_TABLE_GUID, TABLE_VIEW_GUID};
verifyCreatedEntities(entityStore, entityGuids, 4);
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
index c4480ad..a2a5f58 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
@@ -122,7 +122,7 @@ public class ZipFileResourceTestUtils {
entry.getKey().contains("Column") ||
entry.getKey().contains("StorageDesc")) continue;
- assertTrue(metricsForCompare.containsKey(entry.getKey()),
entry.getKey());
+ assertTrue(metricsForCompare.containsKey(entry.getKey()), "key '"
+ entry.getKey() + "' missing from import result: " +
metricsForCompare.toString());
assertEquals(entry.getValue(),
metricsForCompare.get(entry.getKey()), entry.getKey());
}
}
diff --git
a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java
b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java
index 8664fa1..f371295 100644
---
a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java
+++
b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java
@@ -59,6 +59,7 @@ import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
+import org.springframework.util.CollectionUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -73,11 +74,12 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import static org.apache.atlas.TestUtils.*;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotEquals;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
+import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME;
+import static org.apache.atlas.TestUtils.COLUMN_TYPE;
+import static org.apache.atlas.TestUtils.DATABASE_TYPE;
+import static org.apache.atlas.TestUtils.NAME;
+import static org.apache.atlas.TestUtils.TABLE_TYPE;
+import static org.testng.Assert.*;
public abstract class AtlasDeleteHandlerV1Test {
@@ -175,7 +177,7 @@ public abstract class AtlasDeleteHandlerV1Test {
//get entity by unique attribute should throw EntityNotFoundException
try {
- metadataService.getEntityDefinition(TestUtils.DATABASE_TYPE,
"name", (String) response.getFirstEntityCreated().getAttribute("name"));
+ metadataService.getEntityDefinition(DATABASE_TYPE, "name",
(String) response.getFirstEntityCreated().getAttribute("name"));
fail("Expected EntityNotFoundException");
} catch(EntityNotFoundException e) {
//expected
@@ -188,7 +190,7 @@ public abstract class AtlasDeleteHandlerV1Test {
assertNotEquals(newCreationResponse.getFirstEntityCreated().getGuid(),
response.getFirstEntityCreated().getGuid());
//get by unique attribute should return the new entity
- ITypedReferenceableInstance instance =
metadataService.getEntityDefinitionReference(TestUtils.DATABASE_TYPE, "name",
(String) dbEntity.getAttribute("name"));
+ ITypedReferenceableInstance instance =
metadataService.getEntityDefinitionReference(DATABASE_TYPE, "name", (String)
dbEntity.getAttribute("name"));
assertEquals(instance.getId()._getId(),
newCreationResponse.getFirstEntityCreated().getGuid());
}
@@ -266,7 +268,7 @@ public abstract class AtlasDeleteHandlerV1Test {
final AtlasEntityHeader column3Created =
tblCreationResponse.getCreatedEntityByTypeNameAndAttribute(COLUMN_TYPE, NAME,
(String) columnEntity3.getAttribute(NAME));
// Retrieve the table entities from the Repository, to get their guids
and the composite column guids.
- ITypedReferenceableInstance tableInstance =
metadataService.getEntityDefinitionReference(TestUtils.TABLE_TYPE, NAME,
(String) tableEntity.getAttribute(NAME));
+ ITypedReferenceableInstance tableInstance =
metadataService.getEntityDefinitionReference(TABLE_TYPE, NAME, (String)
tableEntity.getAttribute(NAME));
List<IReferenceableInstance> columns = (List<IReferenceableInstance>)
tableInstance.get(COLUMNS_ATTR_NAME);
//Delete column
@@ -304,7 +306,7 @@ public abstract class AtlasDeleteHandlerV1Test {
assertEntityDeleted(colId);
// Delete the table entities. The deletion should cascade to their
composite columns.
- tableInstance =
metadataService.getEntityDefinitionReference(TestUtils.TABLE_TYPE, NAME,
(String) tableEntity.getAttribute(NAME));
+ tableInstance =
metadataService.getEntityDefinitionReference(TABLE_TYPE, NAME, (String)
tableEntity.getAttribute(NAME));
init();
EntityMutationResponse tblDeletionResponse =
entityStore.deleteById(tableInstance.getId()._getId());
@@ -346,12 +348,18 @@ public abstract class AtlasDeleteHandlerV1Test {
init();
final EntityMutationResponse hrDeptCreationResponse =
entityStore.createOrUpdate(new AtlasEntityStream(hrDept), false);
- final AtlasEntityHeader deptCreated =
getFirstCreatedOrUpdatedEntityByTyp(hrDeptCreationResponse, DEPARTMENT_TYPE);
+ final AtlasEntityHeader deptCreated =
getFirstCreatedOrUpdatedEntityByTyp(hrDeptCreationResponse,
TestUtilsV2.DEPARTMENT_TYPE);
final AtlasEntityHeader maxEmployeeCreated =
getCreatedOrUpdatedEntityByTypeAndAttribute(hrDeptCreationResponse,
TestUtilsV2.EMPLOYEE_TYPE, NAME, "Max");
final AtlasEntityHeader johnEmployeeCreated =
getCreatedOrUpdatedEntityByTypeAndAttribute(hrDeptCreationResponse,
TestUtilsV2.EMPLOYEE_TYPE, NAME, "John");
final AtlasEntityHeader janeEmployeeCreated =
getCreatedOrUpdatedEntityByTypeAndAttribute(hrDeptCreationResponse,
TestUtilsV2.MANAGER_TYPE, NAME, "Jane");
final AtlasEntityHeader juliusEmployeeCreated =
getCreatedOrUpdatedEntityByTypeAndAttribute(hrDeptCreationResponse,
TestUtilsV2.MANAGER_TYPE, NAME, "Julius");
+ assertNotNull(deptCreated, "entity not found in response: typeName=" +
TestUtilsV2.DEPARTMENT_TYPE + "; response=" + hrDeptCreationResponse);
+ assertNotNull(maxEmployeeCreated, "entity not found in response:
typeName=" + TestUtilsV2.EMPLOYEE_TYPE + ", name=Max" + "; response=" +
hrDeptCreationResponse);
+ assertNotNull(johnEmployeeCreated, "entity not found in response:
typeName=" + TestUtilsV2.EMPLOYEE_TYPE + ", name=John" + "; response=" +
hrDeptCreationResponse);
+ assertNotNull(janeEmployeeCreated, "entity not found in response:
typeName=" + TestUtilsV2.MANAGER_TYPE + ", name=Jane" + "; response=" +
hrDeptCreationResponse);
+ assertNotNull(juliusEmployeeCreated, "entity not found in response:
typeName=" + TestUtilsV2.MANAGER_TYPE + ", name=Julius" + "; response=" +
hrDeptCreationResponse);
+
ITypedReferenceableInstance max =
metadataService.getEntityDefinition(maxEmployeeCreated.getGuid());
String maxGuid = max.getId()._getId();
AtlasVertex vertex =
GraphHelper.getInstance().getVertexForGUID(maxGuid);
@@ -469,7 +477,7 @@ public abstract class AtlasDeleteHandlerV1Test {
init();
final EntityMutationResponse hrDeptCreationResponse =
entityStore.createOrUpdate(new AtlasEntityStream(hrDept), false);
- final AtlasEntityHeader deptCreated =
getFirstCreatedOrUpdatedEntityByTyp(hrDeptCreationResponse, DEPARTMENT_TYPE);
+ final AtlasEntityHeader deptCreated =
getFirstCreatedOrUpdatedEntityByTyp(hrDeptCreationResponse,
TestUtilsV2.DEPARTMENT_TYPE);
final AtlasEntityHeader maxEmployee =
getCreatedOrUpdatedEntityByTypeAndAttribute(hrDeptCreationResponse,
TestUtilsV2.EMPLOYEE_TYPE, NAME, "Max");
final AtlasEntityHeader johnEmployee =
getCreatedOrUpdatedEntityByTypeAndAttribute(hrDeptCreationResponse,
TestUtilsV2.EMPLOYEE_TYPE, NAME, "John");
final AtlasEntityHeader janeEmployee =
getCreatedOrUpdatedEntityByTypeAndAttribute(hrDeptCreationResponse,
TestUtilsV2.MANAGER_TYPE, NAME, "Jane");
@@ -518,6 +526,11 @@ public abstract class AtlasDeleteHandlerV1Test {
final AtlasEntity.AtlasEntityWithExtInfo johnUpdated =
entityStore.getById(johnEmployee.getGuid());
assertJohnForTestDisconnectBidirectionalReferences(johnUpdated,
janeEmployee.getGuid());
+
+ // cleanup - delete entities created by this test
+ entityStore.deleteById(juliusEmployee.getGuid());
+ entityStore.deleteById(johnEmployee.getGuid());
+ entityStore.deleteById(deptCreated.getGuid());
}
protected List<String> extractGuids(final List<AtlasEntityHeader>
updatedEntities) {
@@ -857,7 +870,7 @@ public abstract class AtlasDeleteHandlerV1Test {
final AtlasEntityHeader column3Created =
tblCreationResponse.getCreatedEntityByTypeNameAndAttribute(COLUMN_TYPE, NAME,
(String) columnEntity3.getAttribute(NAME));
// Retrieve the table entities from the Repository, to get their guids
and the composite column guids.
- ITypedReferenceableInstance tableInstance =
metadataService.getEntityDefinitionReference(TestUtils.TABLE_TYPE, NAME,
(String) tableEntity.getAttribute(NAME));
+ ITypedReferenceableInstance tableInstance =
metadataService.getEntityDefinitionReference(TABLE_TYPE, NAME, (String)
tableEntity.getAttribute(NAME));
List<IReferenceableInstance> columns = (List<IReferenceableInstance>)
tableInstance.get(COLUMNS_ATTR_NAME);
//Delete column
@@ -989,11 +1002,28 @@ public abstract class AtlasDeleteHandlerV1Test {
}
}
+ /* | (ownedRef)
+ ---------------------------------------------- |
+ | MapValueReferencerContainer | |
+ |----------------------------------------------| |
----------------------------------
+ | requiredMap: map<string, MapValueReferencer> | -> |
MapValueReferencer |
+ ----------------------------------------------
|----------------------------------| -------------------
+ | refToMapValue:
CompositeMapValue | -> | CompositeMapValue |
+
---------------------------------- |-------------------|
+
->| name: string |
+
| -------------------
+
------------------------------------- |
+ |
CompositeMapOwner | |
+
|-------------------------------------| |
+ | name: string
| |
+ | map: map<string,
CompositeMapValue> |-- (ownedRef)
+
-------------------------------------
+ */
@Test
public void testLowerBoundsIgnoredWhenDeletingCompositeEntitesOwnedByMap()
throws Exception {
// Define MapValueReferencer type with required reference to
CompositeMapValue.
AtlasStructDef.AtlasAttributeDef[] mapValueAttributes = new
AtlasStructDef.AtlasAttributeDef[]{
- new AtlasStructDef.AtlasAttributeDef("refToMapValue",
"CompositeMapValue",
+ new AtlasStructDef.AtlasAttributeDef("refToMapValue",
compositeMapValueType.getTypeName(),
false,
AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
false, false,
@@ -1020,6 +1050,7 @@ public abstract class AtlasDeleteHandlerV1Test {
Arrays.asList(mapContainerAttributes),
Collections.<String>emptySet());
+
AtlasTypesDef typesDef =
AtlasTypeUtil.getTypesDef(ImmutableList.<AtlasEnumDef>of(),
ImmutableList.<AtlasStructDef>of(),
ImmutableList.<AtlasClassificationDef>of(),
@@ -1043,7 +1074,7 @@ public abstract class AtlasDeleteHandlerV1Test {
Assert.assertNotNull(mapValueInstance);
String mapValueGuid = mapValueInstance.getId()._getId();
- // Create instance of MapValueReferencerContainer
+ // Create instance of MapValueReferencer
init();
AtlasEntity mapValueReferencer = new
AtlasEntity(mapValueDef.getName());
mapValueReferencer.setAttribute("refToMapValue", new
AtlasObjectId(mapValueInstance.getId()._getId(),
mapValueInstance.getTypeName()));
@@ -1052,8 +1083,10 @@ public abstract class AtlasDeleteHandlerV1Test {
List<AtlasEntityHeader> createEntitiesResult =
entityStore.createOrUpdate(new AtlasEntityStream(entities),
false).getCreatedEntities();
Assert.assertEquals(createEntitiesResult.size(), 1);
+ String mapValueReferencerGuid = createEntitiesResult.get(0).getGuid();
+ mapValueReferencer.setGuid(mapValueReferencerGuid);
- // Create instance of MapValueReferencer, and update
mapValueReferencerContainer
+ // Create instance of MapValueReferencerContainer, and add
MapValueReferencer in its requiredMap attribute
// to reference it.
AtlasEntity mapValueReferenceContainer = new
AtlasEntity(mapContainerDef.getName());
entities = new AtlasEntity.AtlasEntitiesWithExtInfo();
@@ -1066,11 +1099,9 @@ public abstract class AtlasDeleteHandlerV1Test {
EntityMutationResponse updateEntitiesResult =
entityStore.createOrUpdate(new AtlasEntityStream(entities), false);
String mapValueReferencerContainerGuid =
updateEntitiesResult.getCreatedEntitiesByTypeName("MapValueReferencerContainer").get(0).getGuid();
- String mapValueReferencerGuid =
updateEntitiesResult.getUpdatedEntitiesByTypeName("MapValueReferencer").get(0).getGuid();
Assert.assertEquals(updateEntitiesResult.getCreatedEntities().size(),
1);
- Assert.assertEquals(updateEntitiesResult.getUpdatedEntities().size(),
1);
-
Assert.assertEquals(updateEntitiesResult.getUpdatedEntities().get(0).getGuid(),
mapValueReferencerGuid);
+
Assert.assertTrue(CollectionUtils.isEmpty(updateEntitiesResult.getUpdatedEntities()));
// Delete map owner and map referencer container. A total of 4
entities should be deleted,
diff --git
a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
index 89b0cea..0d94d29 100644
---
a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
+++
b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
@@ -112,6 +112,9 @@ public class AtlasEntityStoreV1Test {
@Inject
private EntityGraphMapper graphMapper;
+ @Inject
+ private EntityGraphRetriever graphRetriever;
+
@BeforeClass
public void setUp() throws Exception {
@@ -147,7 +150,7 @@ public class AtlasEntityStoreV1Test {
@BeforeTest
public void init() throws Exception {
- entityStore = new AtlasEntityStoreV1(deleteDelegate, typeRegistry,
mockChangeNotifier, graphMapper);
+ entityStore = new AtlasEntityStoreV1(deleteDelegate, typeRegistry,
mockChangeNotifier, graphMapper, graphRetriever);
RequestContextV1.clear();
RequestContextV1.get().setUser(TestUtilsV2.TEST_USER);
@@ -398,6 +401,7 @@ public class AtlasEntityStoreV1Test {
//Drop the first key and change the class type as well to col0
columnsMap.clear();
columnsMap.put("col0", AtlasTypeUtil.getAtlasObjectId(col0));
+ tableEntity.setAttribute(TestUtils.COLUMNS_MAP, columnsMap);
init();
response = entityStore.createOrUpdate(new
AtlasEntityStream(entitiesInfo), false);
@@ -430,6 +434,7 @@ public class AtlasEntityStoreV1Test {
//Remove an entry
paramsMap.remove("key1");
+ tableEntity.setAttribute("parametersMap", paramsMap);
init();
response = entityStore.createOrUpdate(new
AtlasEntityStream(entitiesInfo), false);
validateMutationResponse(response,
EntityMutations.EntityOperation.UPDATE, 1);
@@ -456,21 +461,24 @@ public class AtlasEntityStoreV1Test {
//add a new element to array of struct
partitions.add(new AtlasStruct(TestUtils.PARTITION_STRUCT_TYPE,
TestUtilsV2.NAME, "part3"));
+ tableEntity.setAttribute("partitions", partitions);
init();
response = entityStore.createOrUpdate(new
AtlasEntityStream(entitiesInfo), false);
updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE);
validateEntity(entitiesInfo, getEntityFromStore(updatedTable));
//remove one of the struct values
- init();
partitions.remove(1);
+ tableEntity.setAttribute("partitions", partitions);
+ init();
response = entityStore.createOrUpdate(new
AtlasEntityStream(entitiesInfo), false);
updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE);
validateEntity(entitiesInfo, getEntityFromStore(updatedTable));
//Update struct value within array of struct
- init();
partitions.get(0).setAttribute(TestUtilsV2.NAME, "part4");
+ tableEntity.setAttribute("partitions", partitions);
+ init();
response = entityStore.createOrUpdate(new
AtlasEntityStream(entitiesInfo), false);
updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE);
validateEntity(entitiesInfo, getEntityFromStore(updatedTable));
@@ -478,6 +486,7 @@ public class AtlasEntityStoreV1Test {
//add a repeated element to array of struct
partitions.add(new AtlasStruct(TestUtils.PARTITION_STRUCT_TYPE,
TestUtilsV2.NAME, "part4"));
+ tableEntity.setAttribute("partitions", partitions);
init();
response = entityStore.createOrUpdate(new
AtlasEntityStream(entitiesInfo), false);
updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE);
@@ -485,6 +494,7 @@ public class AtlasEntityStoreV1Test {
// Remove all elements. Should set array attribute to null
partitions.clear();
+ tableEntity.setAttribute("partitions", partitions);
init();
response = entityStore.createOrUpdate(new
AtlasEntityStream(entitiesInfo), false);
updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE);
diff --git a/repository/src/test/resources/reporting-v1-full.zip
b/repository/src/test/resources/reporting-v1-full.zip
index c2fc9c2..539dfce 100644
Binary files a/repository/src/test/resources/reporting-v1-full.zip and
b/repository/src/test/resources/reporting-v1-full.zip differ
diff --git
a/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java
b/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java
index d5ff423..9cc5498 100644
---
a/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java
+++
b/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java
@@ -51,7 +51,7 @@ public class AdminExportImportTestIT extends BaseResourceIT {
static final String IMPORT_TRANSFORM_CLEAR_ATTRS =
"{ \"Asset\": { \"*\":[
\"clearAttrValue:replicatedTo,replicatedFrom\" ] } }";
static final String IMPORT_TRANSFORM_SET_DELETED =
- "{ \"Asset\": { \"*\":[ \"setDeleted\" ] } }";
+ "{ \"Referenceable\": { \"*\":[ \"setDeleted\" ] } }";
@Test
public void isActive() throws AtlasServiceException {
