http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeExistsException.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeExistsException.java b/typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeExistsException.java new file mode 100644 index 0000000..8a28e38 --- /dev/null +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeExistsException.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.typesystem.exception; + +import org.apache.atlas.AtlasException; + +public class TypeExistsException extends AtlasException { + public TypeExistsException(String message) { + super(message); + } +}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeNotFoundException.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeNotFoundException.java b/typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeNotFoundException.java new file mode 100644 index 0000000..3654a4b --- /dev/null +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeNotFoundException.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.typesystem.exception; + +import org.apache.atlas.AtlasException; + +/** + * A simple wrapper for 404. + */ +public class TypeNotFoundException extends AtlasException { + public TypeNotFoundException() { + } + + public TypeNotFoundException(String message) { + super(message); + } + + public TypeNotFoundException(String message, Throwable cause) { + super(message, cause); + } + + public TypeNotFoundException(Throwable cause) { + super(cause); + } + + public TypeNotFoundException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/DownCastStructInstance.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/DownCastStructInstance.java b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/DownCastStructInstance.java index deb15b5..d3b9a33 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/DownCastStructInstance.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/DownCastStructInstance.java @@ -52,6 +52,11 @@ public class DownCastStructInstance implements IStruct { fieldMapping.set(this, attrName, val); } + @Override + public void setNull(String attrName) throws AtlasException { + throw new UnsupportedOperationException("unset on attributes are not allowed"); + } + /* * Use only for json serialization * @nonpublic http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java index 641146a..20c2f91 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java @@ -20,12 +20,16 @@ package org.apache.atlas.typesystem.persistence; import com.google.common.collect.ImmutableList; import org.apache.atlas.AtlasException; +import org.apache.atlas.utils.ParamChecker; import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.types.FieldMapping; +import org.apache.atlas.utils.MD5Utils; import java.math.BigDecimal; import java.math.BigInteger; +import java.nio.charset.Charset; +import java.security.MessageDigest; import java.util.Date; import java.util.Map; import java.util.UUID; @@ -37,6 +41,8 @@ public class Id implements ITypedReferenceableInstance { public final int version; public Id(String id, int version, String className) { + ParamChecker.notEmpty(className, "id"); + ParamChecker.notEmpty(className, "className"); this.id = id; this.className = className; this.version = version; @@ -248,4 +254,12 @@ public class Id implements ITypedReferenceableInstance { public void setString(String attrName, String val) throws AtlasException { throw new AtlasException("Get/Set not supported on an Id object"); } + + @Override + public String getSignatureHash(MessageDigest digester) throws AtlasException { + digester.update(id.getBytes(Charset.forName("UTF-8"))); + digester.update(className.getBytes(Charset.forName("UTF-8"))); + byte[] digest = digester.digest(); + return MD5Utils.toString(digest); + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java index 911a5f4..0fa4666 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java @@ -24,10 +24,14 @@ import org.apache.atlas.AtlasException; import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedStruct; +import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.FieldMapping; +import org.apache.atlas.typesystem.types.TypeSystem; +import org.apache.atlas.utils.MD5Utils; import java.math.BigDecimal; import java.math.BigInteger; +import java.security.MessageDigest; import java.util.Date; /* @@ -75,7 +79,7 @@ public class ReferenceableInstance extends StructInstance implements ITypedRefer * @nopub * @param id */ - void replaceWithNewId(Id id) { + public void replaceWithNewId(Id id) { this.id = id; } @@ -92,4 +96,12 @@ public class ReferenceableInstance extends StructInstance implements ITypedRefer throw new RuntimeException(me); } } + + @Override + public String getSignatureHash(MessageDigest digester) throws AtlasException { + ClassType classType = TypeSystem.getInstance().getDataType(ClassType.class, getTypeName()); + classType.updateSignatureHash(digester, this); + byte[] digest = digester.digest(); + return MD5Utils.toString(digest); + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java index ea4a3cb..16c3a24 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java @@ -29,12 +29,15 @@ import org.apache.atlas.typesystem.types.DataTypes; import org.apache.atlas.typesystem.types.EnumType; import org.apache.atlas.typesystem.types.EnumValue; import org.apache.atlas.typesystem.types.FieldMapping; +import org.apache.atlas.typesystem.types.StructType; import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeUtils; import org.apache.atlas.typesystem.types.ValueConversionException; +import org.apache.atlas.utils.MD5Utils; import java.math.BigDecimal; import java.math.BigInteger; +import java.security.MessageDigest; import java.util.Date; import java.util.HashMap; import java.util.Map; @@ -229,6 +232,30 @@ public class StructInstance implements ITypedStruct { } int nullPos = fieldMapping.fieldNullPos.get(attrName); nullFlags[nullPos] = true; + + int pos = fieldMapping.fieldPos.get(attrName); + + if (i.dataType() == DataTypes.BIGINTEGER_TYPE) { + bigIntegers[pos] = null; + } else if (i.dataType() == DataTypes.BIGDECIMAL_TYPE) { + bigDecimals[pos] = null; + } else if (i.dataType() == DataTypes.DATE_TYPE) { + dates[pos] = null; + } else if (i.dataType() == DataTypes.STRING_TYPE) { + strings[pos] = null; + } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) { + arrays[pos] = null; + } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) { + maps[pos] = null; + } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.STRUCT + || i.dataType().getTypeCategory() == DataTypes.TypeCategory.TRAIT) { + structs[pos] = null; + } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) { + ids[pos] = null; + referenceables[pos] = null; + } else { + throw new AtlasException(String.format("Unknown datatype %s", i.dataType())); + } } /* @@ -729,4 +756,12 @@ public class StructInstance implements ITypedStruct { throw new RuntimeException(me); } } + + @Override + public String getSignatureHash(MessageDigest digester) throws AtlasException { + StructType structType = TypeSystem.getInstance().getDataType(StructType.class, getTypeName()); + structType.updateSignatureHash(digester, this); + byte[] digest = digester.digest(); + return MD5Utils.toString(digest); + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java index 2fea1be..29c3450 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java @@ -18,7 +18,7 @@ package org.apache.atlas.typesystem.types; -import org.apache.atlas.ParamChecker; +import org.apache.atlas.utils.ParamChecker; public final class AttributeDefinition { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java index cdfbf07..ac758fa 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java @@ -34,6 +34,8 @@ import org.apache.atlas.typesystem.persistence.StructInstance; import java.math.BigDecimal; import java.math.BigInteger; +import java.nio.charset.Charset; +import java.security.MessageDigest; import java.util.Date; import java.util.List; import java.util.Map; @@ -123,9 +125,9 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc r != null ? createInstanceWithTraits(id, r, r.getTraits().toArray(new String[0])) : createInstance(id); - if (id != null && id.isAssigned()) { - return tr; - } +// if (id != null && id.isAssigned()) { +// return tr; +// } for (Map.Entry<String, AttributeInfo> e : fieldMapping.fields.entrySet()) { String attrKey = e.getKey(); @@ -214,4 +216,24 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc public List<String> getNames(AttributeInfo info) { return infoToNameMap.get(info); } + + @Override + public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException { + if( !(val instanceof ITypedReferenceableInstance)) { + throw new IllegalArgumentException("Unexpected value type " + val.getClass().getSimpleName() + ". Expected instance of ITypedStruct"); + } + digester.update(getName().getBytes(Charset.forName("UTF-8"))); + + if(fieldMapping.fields != null && val != null) { + IReferenceableInstance typedValue = (IReferenceableInstance) val; + if(fieldMapping.fields.values() != null) { + for (AttributeInfo aInfo : fieldMapping.fields.values()) { + Object attrVal = typedValue.get(aInfo.name); + if (attrVal != null) { + aInfo.dataType().updateSignatureHash(digester, attrVal); + } + } + } + } + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java index 4c55ce7..afd1682 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java @@ -29,10 +29,13 @@ import org.apache.commons.lang3.StringUtils; import java.math.BigDecimal; import java.math.BigInteger; +import java.nio.charset.Charset; +import java.security.MessageDigest; import java.text.ParseException; import java.util.Collection; import java.util.Date; import java.util.Iterator; +import java.util.List; import java.util.Map; public class DataTypes { @@ -95,6 +98,14 @@ public class DataTypes { return nullValue(); } + + @Override + public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException { + if ( val != null ) { + digester.update(val.toString().getBytes(Charset.forName("UTF-8"))); + } + } + } public static class BooleanType extends PrimitiveType<Boolean> { @@ -161,6 +172,13 @@ public class DataTypes { public Byte nullValue() { return 0; } + + @Override + public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException { + if ( val != null ) { + digester.update(((Byte) val).byteValue()); + } + } } public static class ShortType extends PrimitiveType<Short> { @@ -508,6 +526,7 @@ public class DataTypes { } else if (val instanceof Iterator) { it = (Iterator) val; } + if (it != null) { ImmutableCollection.Builder b = m.isUnique ? ImmutableSet.builder() : ImmutableList.builder(); while (it.hasNext()) { @@ -557,6 +576,15 @@ public class DataTypes { public TypeCategory getTypeCategory() { return TypeCategory.ARRAY; } + + @Override + public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException { + IDataType elemType = getElemType(); + List vals = (List) val; + for (Object listElem : vals) { + elemType.updateSignatureHash(digester, listElem); + } + } } public static class MapType extends AbstractDataType<ImmutableMap<?, ?>> { @@ -586,7 +614,7 @@ public class DataTypes { } protected void setValueType(IDataType valueType) { - this.keyType = valueType; + this.valueType = valueType; } @Override @@ -605,7 +633,8 @@ public class DataTypes { Map.Entry e = it.next(); b.put(keyType.convert(e.getKey(), TypeSystem.getInstance().allowNullsInCollections() ? Multiplicity.OPTIONAL : - Multiplicity.REQUIRED), valueType.convert(e.getValue(), + Multiplicity.REQUIRED), + valueType.convert(e.getValue(), TypeSystem.getInstance().allowNullsInCollections() ? Multiplicity.OPTIONAL : Multiplicity.REQUIRED)); } @@ -657,6 +686,17 @@ public class DataTypes { public TypeCategory getTypeCategory() { return TypeCategory.MAP; } + + @Override + public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException { + IDataType keyType = getKeyType(); + IDataType valueType = getValueType(); + Map vals = (Map) val; + for (Object key : vals.keySet()) { + keyType.updateSignatureHash(digester, key); + valueType.updateSignatureHash(digester, vals.get(key)); + } + } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java index 1439303..b751307 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java @@ -23,6 +23,9 @@ import com.google.common.collect.ImmutableMap; import org.apache.atlas.AtlasException; import scala.math.BigInt; +import java.nio.charset.Charset; +import java.security.MessageDigest; + public class EnumType extends AbstractDataType<EnumValue> { public final TypeSystem typeSystem; @@ -80,7 +83,7 @@ public class EnumType extends AbstractDataType<EnumValue> { public void validateUpdate(IDataType newType) throws TypeUpdateException { super.validateUpdate(newType); - EnumType enumType = (EnumType)newType; + EnumType enumType = (EnumType) newType; for (EnumValue enumValue : values()) { //The old enum value should be part of new enum definition as well if (!enumType.valueMap.containsKey(enumValue.value)) { @@ -96,6 +99,12 @@ public class EnumType extends AbstractDataType<EnumValue> { } } + public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException { + if (val != null) { + digester.update(fromValue((String) val).toString().getBytes(Charset.forName("UTF-8"))); + } + } + public EnumValue fromOrdinal(int o) { return ordinalMap.get(o); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java index 7afc52e..aca1a41 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java @@ -18,7 +18,7 @@ package org.apache.atlas.typesystem.types; -import org.apache.atlas.ParamChecker; +import org.apache.atlas.utils.ParamChecker; import java.util.Arrays; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumValue.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumValue.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumValue.java index d22e7e0..d75259b 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumValue.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumValue.java @@ -18,7 +18,7 @@ package org.apache.atlas.typesystem.types; -import org.apache.atlas.ParamChecker; +import org.apache.atlas.utils.ParamChecker; public class EnumValue { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java index d9b1b34..293014e 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java @@ -20,6 +20,8 @@ package org.apache.atlas.typesystem.types; import org.apache.atlas.AtlasException; +import java.security.MessageDigest; + public interface IDataType<T> { String getName(); @@ -30,5 +32,7 @@ public interface IDataType<T> { void output(T val, Appendable buf, String prefix) throws AtlasException; void validateUpdate(IDataType newType) throws TypeUpdateException; + + void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/Multiplicity.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/Multiplicity.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/Multiplicity.java index a54dabc..74d7f7c 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/Multiplicity.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/Multiplicity.java @@ -25,8 +25,8 @@ public final class Multiplicity { public static final Multiplicity OPTIONAL = new Multiplicity(0, 1, false); public static final Multiplicity REQUIRED = new Multiplicity(1, 1, false); - public static final Multiplicity COLLECTION = new Multiplicity(1, Integer.MAX_VALUE, false); - public static final Multiplicity SET = new Multiplicity(1, Integer.MAX_VALUE, true); + public static final Multiplicity COLLECTION = new Multiplicity(0, Integer.MAX_VALUE, false); + public static final Multiplicity SET = new Multiplicity(0, Integer.MAX_VALUE, true); public final int lower; public final int upper; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java index fb4f79f..db87cf9 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java @@ -214,5 +214,11 @@ public class ObjectGraphWalker { this.aInfo = aInfo; this.value = value; } + + @Override + public String toString(){ + StringBuilder string = new StringBuilder().append(instance).append(aInfo).append(value); + return string.toString(); + } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java index 1a40484..ba053d2 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java @@ -22,6 +22,8 @@ import org.apache.atlas.AtlasException; import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.ITypedStruct; +import java.nio.charset.Charset; +import java.security.MessageDigest; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -193,6 +195,24 @@ public class StructType extends AbstractDataType<IStruct> implements IConstructa handler.output(s, buf, prefix); } + @Override + public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException { + if( !(val instanceof ITypedStruct)) { + throw new IllegalArgumentException("Unexpected value type " + val.getClass().getSimpleName() + ". Expected instance of ITypedStruct"); + } + digester.update(getName().getBytes(Charset.forName("UTF-8"))); + + if(fieldMapping.fields != null && val != null) { + IStruct typedValue = (IStruct) val; + for (AttributeInfo aInfo : fieldMapping.fields.values()) { + Object attrVal = typedValue.get(aInfo.name); + if(attrVal != null) { + aInfo.dataType().updateSignatureHash(digester, attrVal); + } + } + } + } + public List<String> getNames(AttributeInfo info) { return infoToNameMap.get(info); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java index e4bd28d..e47c4e5 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java @@ -18,7 +18,7 @@ package org.apache.atlas.typesystem.types; -import org.apache.atlas.ParamChecker; +import org.apache.atlas.utils.ParamChecker; import java.util.Arrays; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java index d23d247..f5a3875 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java @@ -23,6 +23,8 @@ import org.apache.atlas.AtlasException; import org.apache.atlas.typesystem.IStruct; import org.apache.atlas.typesystem.ITypedStruct; +import java.nio.charset.Charset; +import java.security.MessageDigest; import java.util.List; import java.util.Map; @@ -68,6 +70,24 @@ public class TraitType extends HierarchicalType<TraitType, IStruct> } @Override + public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException { + if( !(val instanceof ITypedStruct)) { + throw new IllegalArgumentException("Unexpected value type " + val.getClass().getSimpleName() + ". Expected instance of ITypedStruct"); + } + digester.update(getName().getBytes(Charset.forName("UTF-8"))); + + if(fieldMapping.fields != null && val != null) { + IStruct typedValue = (IStruct) val; + for (AttributeInfo aInfo : fieldMapping.fields.values()) { + Object attrVal = typedValue.get(aInfo.name); + if(attrVal != null) { + aInfo.dataType().updateSignatureHash(digester, attrVal); + } + } + } + } + + @Override public List<String> getNames(AttributeInfo info) { return infoToNameMap.get(info); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java index d9b1edb..c0b0698 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java @@ -22,10 +22,10 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.Multimap; import org.apache.atlas.AtlasException; -import org.apache.atlas.TypeExistsException; -import org.apache.atlas.TypeNotFoundException; import org.apache.atlas.classification.InterfaceAudience; import org.apache.atlas.typesystem.TypesDef; +import org.apache.atlas.typesystem.exception.TypeExistsException; +import org.apache.atlas.typesystem.exception.TypeNotFoundException; import javax.inject.Singleton; import java.lang.reflect.Constructor; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/scala/org/apache/atlas/typesystem/json/Serialization.scala ---------------------------------------------------------------------- diff --git a/typesystem/src/main/scala/org/apache/atlas/typesystem/json/Serialization.scala b/typesystem/src/main/scala/org/apache/atlas/typesystem/json/Serialization.scala index e38772d..c5aa6e8 100755 --- a/typesystem/src/main/scala/org/apache/atlas/typesystem/json/Serialization.scala +++ b/typesystem/src/main/scala/org/apache/atlas/typesystem/json/Serialization.scala @@ -29,18 +29,22 @@ import org.json4s.native.Serialization._ import scala.collection.JavaConversions._ import scala.collection.JavaConverters._ -class BigDecimalSerializer extends CustomSerializer[java.math.BigDecimal](format => ( { - case JDecimal(e) => e.bigDecimal -}, { - case e: java.math.BigDecimal => JDecimal(new BigDecimal(e)) -} +class BigDecimalSerializer extends CustomSerializer[java.math.BigDecimal](format => ( + { + case JDecimal(e) => e.bigDecimal + }, + { + case e: java.math.BigDecimal => JDecimal(new BigDecimal(e)) + } )) -class BigIntegerSerializer extends CustomSerializer[java.math.BigInteger](format => ( { - case JInt(e) => e.bigInteger -}, { - case e: java.math.BigInteger => JInt(new BigInt(e)) -} +class BigIntegerSerializer extends CustomSerializer[java.math.BigInteger](format => ( + { + case JInt(e) => e.bigInteger + }, + { + case e: java.math.BigInteger => JInt(new BigInt(e)) + } )) class IdSerializer extends CustomSerializer[Id](format => ( { @@ -292,12 +296,19 @@ object Serialization { read[ReferenceableInstance](jsonStr) } - def traitFromJson(jsonStr: String): ITypedInstance = { - implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer + - new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer + def traitFromJson(jsonStr: String): ITypedInstance = { + implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer + + new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer + + read[StructInstance](jsonStr) + } - read[StructInstance](jsonStr) - } + def arrayFromJson(jsonStr: String): ITypedInstance = { + implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer + + new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer + + read[StructInstance](jsonStr) + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java index 703f4ee..c735ecd 100755 --- a/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java +++ b/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java @@ -86,7 +86,7 @@ public class AuditFilter implements Filter { LOG.debug("Audit: {}/{} performed request {} {} ({}) at time {}", who, fromAddress, whatRequest, whatURL, whatAddrs, whenISO9601); - audit(who, fromAddress, fromHost, whatURL, whatAddrs, whenISO9601); + audit(who, fromAddress, whatRequest, fromHost, whatURL, whatAddrs, whenISO9601); } private String getUserFromRequest(HttpServletRequest httpRequest) { @@ -95,9 +95,9 @@ public class AuditFilter implements Filter { return userFromRequest == null ? "UNKNOWN" : userFromRequest; } - private void audit(String who, String fromAddress, String fromHost, String whatURL, String whatAddrs, + private void audit(String who, String fromAddress, String whatRequest, String fromHost, String whatURL, String whatAddrs, String whenISO9601) { - AUDIT_LOG.info("Audit: {}/{}-{} performed request {} ({}) at time {}", who, fromAddress, fromHost, whatURL, + AUDIT_LOG.info("Audit: {}/{}-{} performed request {} {} ({}) at time {}", who, fromAddress, fromHost, whatRequest, whatURL, whatAddrs, whenISO9601); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java index 21ea05f..2ee0027 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java @@ -21,11 +21,13 @@ package org.apache.atlas.web.resources; import com.google.common.base.Preconditions; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; -import org.apache.atlas.ParamChecker; -import org.apache.atlas.TypeNotFoundException; -import org.apache.atlas.repository.EntityExistsException; -import org.apache.atlas.repository.EntityNotFoundException; +import org.apache.atlas.typesystem.exception.EntityExistsException; +import org.apache.atlas.typesystem.exception.EntityNotFoundException; +import org.apache.atlas.typesystem.exception.TypeNotFoundException; +import org.apache.atlas.utils.ParamChecker; import org.apache.atlas.services.MetadataService; +import org.apache.atlas.typesystem.Referenceable; +import org.apache.atlas.typesystem.json.InstanceSerialization; import org.apache.atlas.typesystem.types.ValueConversionException; import org.apache.atlas.web.util.Servlets; import org.apache.commons.lang.StringUtils; @@ -85,7 +87,6 @@ public class EntityResource { this.metadataService = metadataService; } - /** * Submits the entity definitions (instances). * The body contains the JSONArray of entity json. The service takes care of de-duping the entities based on any @@ -138,6 +139,175 @@ public class EntityResource { } /** + * Complete update of a set of entities - the values not specified will be replaced with null/removed + * Adds/Updates given entities identified by its GUID or unique attribute + * @return response payload as json + */ + @PUT + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response updateEntities(@Context HttpServletRequest request) { + try { + final String entities = Servlets.getRequestPayload(request); + LOG.debug("updating entities {} ", AtlasClient.toString(new JSONArray(entities))); + + final String guids = metadataService.updateEntities(entities); + + JSONObject response = new JSONObject(); + response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId()); + response.put(AtlasClient.GUID, new JSONArray(guids)); + response.put(AtlasClient.DEFINITION, metadataService.getEntityDefinition(new JSONArray(guids).getString(0))); + + return Response.ok(response).build(); + } catch(EntityExistsException e) { + LOG.error("Unique constraint violation", e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT)); + } catch (ValueConversionException ve) { + LOG.error("Unable to persist entity instance due to a desrialization error ", ve); + throw new WebApplicationException(Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST)); + } catch (AtlasException | IllegalArgumentException e) { + LOG.error("Unable to persist entity instance", e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable e) { + LOG.error("Unable to persist entity instance", e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + /** + * Adds/Updates given entity identified by its unique attribute( entityType, attributeName and value) + * Updates support only partial update of an entity - Adds/updates any new values specified + * Updates do not support removal of attribute values + * + * @param entityType the entity type + * @param attribute the unique attribute used to identify the entity + * @param value the unique attributes value + * @param request The updated entity json + * @return response payload as json + * The body contains the JSONArray of entity json. The service takes care of de-duping the entities based on any + * unique attribute for the give type. + */ + @POST + @Path("qualifiedName") + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response updateByUniqueAttribute(@QueryParam("type") String entityType, + @QueryParam("property") String attribute, + @QueryParam("value") String value, @Context HttpServletRequest request) { + try { + String entities = Servlets.getRequestPayload(request); + + LOG.debug("Partially updating entity by unique attribute {} {} {} {} ", entityType, attribute, value, entities); + + Referenceable updatedEntity = + InstanceSerialization.fromJsonReferenceable(entities, true); + final String guid = metadataService.updateEntityByUniqueAttribute(entityType, attribute, value, updatedEntity); + + JSONObject response = new JSONObject(); + response.put(AtlasClient.REQUEST_ID, Thread.currentThread().getName()); + response.put(AtlasClient.GUID, guid); + return Response.ok(response).build(); + } catch (ValueConversionException ve) { + LOG.error("Unable to persist entity instance due to a desrialization error ", ve); + throw new WebApplicationException(Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST)); + } catch(EntityExistsException e) { + LOG.error("Unique constraint violation", e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT)); + } catch (EntityNotFoundException e) { + LOG.error("An entity with type={} and qualifiedName={} does not exist", entityType, value, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND)); + } catch (AtlasException | IllegalArgumentException e) { + LOG.error("Unable to create/update entity {}" + entityType + ":" + attribute + "." + value, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable e) { + LOG.error("Unable to update entity {}" + entityType + ":" + attribute + "." + value, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + /** + * Updates entity identified by its GUID + * Support Partial update of an entity - Adds/updates any new values specified + * Does not support removal of attribute values + * + * @param guid + * @param request The updated entity json + * @return + */ + @POST + @Path("{guid}") + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response updateEntityByGuid(@PathParam("guid") String guid, @QueryParam("property") String attribute, + @Context HttpServletRequest request) { + if (StringUtils.isEmpty(attribute)) { + return updateEntityPartialByGuid(guid, request); + } else { + return updateEntityAttributeByGuid(guid, attribute, request); + } + } + + private Response updateEntityPartialByGuid(String guid, HttpServletRequest request) { + try { + ParamChecker.notEmpty(guid, "Guid property cannot be null"); + final String entityJson = Servlets.getRequestPayload(request); + LOG.debug("partially updating entity for guid {} : {} ", guid, entityJson); + + Referenceable updatedEntity = + InstanceSerialization.fromJsonReferenceable(entityJson, true); + metadataService.updateEntityPartialByGuid(guid, updatedEntity); + + JSONObject response = new JSONObject(); + response.put(AtlasClient.REQUEST_ID, Thread.currentThread().getName()); + return Response.ok(response).build(); + } catch (EntityNotFoundException e) { + LOG.error("An entity with GUID={} does not exist", guid, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND)); + } catch (AtlasException | IllegalArgumentException e) { + LOG.error("Unable to update entity {}", guid, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable e) { + LOG.error("Unable to update entity {}", guid, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + /** + * Supports Partial updates + * Adds/Updates given entity specified by its GUID + * Supports updation of only simple primitive attributes like strings, ints, floats, enums, class references and + * does not support updation of complex types like arrays, maps + * @param guid entity id + * @param property property to add + * @postbody property's value + * @return response payload as json + */ + private Response updateEntityAttributeByGuid(String guid, String property, HttpServletRequest request) { + try { + Preconditions.checkNotNull(property, "Entity property cannot be null"); + String value = Servlets.getRequestPayload(request); + Preconditions.checkNotNull(value, "Entity value cannot be null"); + + metadataService.updateEntityAttributeByGuid(guid, property, value); + + JSONObject response = new JSONObject(); + response.put(AtlasClient.REQUEST_ID, Thread.currentThread().getName()); + response.put(AtlasClient.GUID, guid); + + return Response.ok(response).build(); + } catch (EntityNotFoundException e) { + LOG.error("An entity with GUID={} does not exist", guid, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND)); + } catch (AtlasException | IllegalArgumentException e) { + LOG.error("Unable to add property {} to entity id {}", property, guid, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable e) { + LOG.error("Unable to add property {} to entity id {}", property, guid, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + /** * Fetch the complete definition of an entity given its GUID. * * @param guid GUID for the entity @@ -265,39 +435,6 @@ public class EntityResource { } } - /** - * Adds property to the given entity id - * @param guid entity id - * @param property property to add - * @param value property's value - * @return response payload as json - */ - @PUT - @Path("{guid}") - @Consumes(Servlets.JSON_MEDIA_TYPE) - @Produces(Servlets.JSON_MEDIA_TYPE) - public Response update(@PathParam("guid") String guid, @QueryParam("property") String property, - @QueryParam("value") String value) { - try { - Preconditions.checkNotNull(property, "Entity property cannot be null"); - Preconditions.checkNotNull(value, "Entity value cannot be null"); - - metadataService.updateEntity(guid, property, value); - - JSONObject response = new JSONObject(); - response.put(AtlasClient.REQUEST_ID, Thread.currentThread().getName()); - return Response.ok(response).build(); - } catch (EntityNotFoundException e) { - LOG.error("An entity with GUID={} does not exist", guid, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND)); - } catch (AtlasException | IllegalArgumentException e) { - LOG.error("Unable to add property {} to entity id {}", property, guid, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); - } catch (Throwable e) { - LOG.error("Unable to add property {} to entity id {}", property, guid, e); - throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); - } - } // Trait management functions http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java index 7bcaf6b..9b3fbc9 100644 --- a/webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java @@ -19,10 +19,10 @@ package org.apache.atlas.web.resources; import org.apache.atlas.AtlasClient; -import org.apache.atlas.ParamChecker; +import org.apache.atlas.typesystem.exception.EntityNotFoundException; +import org.apache.atlas.utils.ParamChecker; import org.apache.atlas.discovery.DiscoveryException; import org.apache.atlas.discovery.LineageService; -import org.apache.atlas.repository.EntityNotFoundException; import org.apache.atlas.web.util.Servlets; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java index 35f171f..45502b7 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java @@ -20,7 +20,7 @@ package org.apache.atlas.web.resources; import com.google.common.base.Preconditions; import org.apache.atlas.AtlasClient; -import org.apache.atlas.ParamChecker; +import org.apache.atlas.utils.ParamChecker; import org.apache.atlas.discovery.DiscoveryException; import org.apache.atlas.discovery.DiscoveryService; import org.apache.atlas.web.util.Servlets; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java index 1f0b98a..3b90248 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java @@ -21,8 +21,8 @@ package org.apache.atlas.web.resources; import com.sun.jersey.api.client.ClientResponse; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; -import org.apache.atlas.TypeExistsException; import org.apache.atlas.services.MetadataService; +import org.apache.atlas.typesystem.exception.TypeExistsException; import org.apache.atlas.typesystem.types.DataTypes; import org.apache.atlas.web.util.Servlets; import org.codehaus.jettison.json.JSONArray; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/main/java/org/apache/atlas/web/util/Servlets.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/util/Servlets.java b/webapp/src/main/java/org/apache/atlas/web/util/Servlets.java index 3d4c715..8c6b616 100755 --- a/webapp/src/main/java/org/apache/atlas/web/util/Servlets.java +++ b/webapp/src/main/java/org/apache/atlas/web/util/Servlets.java @@ -19,7 +19,7 @@ package org.apache.atlas.web.util; import org.apache.atlas.AtlasClient; -import org.apache.atlas.ParamChecker; +import org.apache.atlas.utils.ParamChecker; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.commons.lang3.StringUtils; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java b/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java index 6929961..4d2cce7 100644 --- a/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java +++ b/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java @@ -118,7 +118,7 @@ public class EntityNotificationIT extends BaseResourceIT { final String guid = tableId._getId(); - serviceClient.updateEntity(guid, property, newValue); + serviceClient.updateEntityAttribute(guid, property, newValue); waitForNotification(MAX_WAIT_TIME); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java b/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java index e03f618..e5af26c 100644 --- a/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java +++ b/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java @@ -30,7 +30,7 @@ import org.testng.annotations.Guice; import org.testng.annotations.Test; @Guice(modules = NotificationModule.class) -public class NotificationHookConsumerIT extends BaseResourceIT{ +public class NotificationHookConsumerIT extends BaseResourceIT { @Inject private NotificationInterface kafka; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java index 291ef48..361cece 100755 --- a/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java @@ -42,6 +42,7 @@ import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.utils.TypesUtil; +import org.apache.atlas.utils.ParamChecker; import org.apache.atlas.web.util.Servlets; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.RandomStringUtils; @@ -83,7 +84,11 @@ public abstract class BaseResourceIT { protected void createType(TypesDef typesDef) throws Exception { HierarchicalTypeDefinition<ClassType> sampleType = typesDef.classTypesAsJavaList().get(0); - if (serviceClient.getType(sampleType.typeName) == null) { + try { + serviceClient.getType(sampleType.typeName); + LOG.info("Types already exist. Skipping type creation"); + } catch(AtlasServiceException ase) { + //Expected if type doesnt exist String typesAsJSON = TypesSerialization.toJson(typesDef); createType(typesAsJSON); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java index 380f280..7337eaf 100755 --- a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java @@ -52,7 +52,10 @@ import org.testng.annotations.Test; import javax.ws.rs.HttpMethod; import javax.ws.rs.core.Response; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; /** @@ -174,7 +177,6 @@ public class EntityJerseyResourceIT extends BaseResourceIT { @Test public void testSubmitEntityWithBadDateFormat() throws Exception { - try { Referenceable tableInstance = createHiveTableInstance("db" + randomString(), "table" + randomString()); tableInstance.set("lastAccessTime", "2014-07-11"); @@ -191,8 +193,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT { final String guid = tableId._getId(); //add property String description = "bar table - new desc"; - ClientResponse clientResponse = addProperty(guid, "description", description); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + addProperty(guid, "description", description); String entityRef = getEntityDefinition(getEntityDefinition(guid)); Assert.assertNotNull(entityRef); @@ -200,13 +201,16 @@ public class EntityJerseyResourceIT extends BaseResourceIT { tableInstance.set("description", description); //invalid property for the type - clientResponse = addProperty(guid, "invalid_property", "bar table"); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode()); + try { + addProperty(guid, "invalid_property", "bar table"); + Assert.fail("Expected AtlasServiceException"); + } catch (AtlasServiceException e) { + Assert.assertEquals(e.getStatus().getStatusCode(), Response.Status.BAD_REQUEST.getStatusCode()); + } //non-string property, update String currentTime = String.valueOf(System.currentTimeMillis()); - clientResponse = addProperty(guid, "createTime", currentTime); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + addProperty(guid, "createTime", currentTime); entityRef = getEntityDefinition(getEntityDefinition(guid)); Assert.assertNotNull(entityRef); @@ -222,12 +226,16 @@ public class EntityJerseyResourceIT extends BaseResourceIT { Assert.fail(); } - @Test(dependsOnMethods = "testSubmitEntity", expectedExceptions = IllegalArgumentException.class) + @Test(dependsOnMethods = "testSubmitEntity") public void testAddNullPropertyValue() throws Exception { final String guid = tableId._getId(); //add property - addProperty(guid, "description", null); - Assert.fail(); + try { + addProperty(guid, "description", null); + Assert.fail("Expected AtlasServiceException"); + } catch(AtlasServiceException e) { + Assert.assertEquals(e.getStatus().getStatusCode(), Response.Status.BAD_REQUEST.getStatusCode()); + } } @Test(dependsOnMethods = "testSubmitEntity") @@ -242,8 +250,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT { //Add reference property final String guid = tableId._getId(); - ClientResponse clientResponse = addProperty(guid, "db", dbId); - Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + addProperty(guid, "db", dbId); } @Test(dependsOnMethods = "testSubmitEntity") @@ -264,11 +271,8 @@ public class EntityJerseyResourceIT extends BaseResourceIT { InstanceSerialization.fromJsonReferenceable(definition, true); } - private ClientResponse addProperty(String guid, String property, String value) { - WebResource resource = service.path(ENTITIES).path(guid); - - return resource.queryParam("property", property).queryParam("value", value).accept(Servlets.JSON_MEDIA_TYPE) - .type(Servlets.JSON_MEDIA_TYPE).method(HttpMethod.PUT, ClientResponse.class); + private void addProperty(String guid, String property, String value) throws AtlasServiceException { + serviceClient.updateEntityAttribute(guid, property, value); } private ClientResponse getEntityDefinition(String guid) { @@ -547,4 +551,82 @@ public class EntityJerseyResourceIT extends BaseResourceIT { Referenceable getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true); Assert.assertEquals(getReferenceable.get(attrName), attrValue); } + + @Test(dependsOnMethods = "testSubmitEntity") + public void testPartialUpdate() throws Exception { + final List<Referenceable> columns = new ArrayList<>(); + Map<String, Object> values = new HashMap<>(); + values.put("name", "col1"); + values.put("dataType", "string"); + values.put("comment", "col1 comment"); + + Referenceable ref = new Referenceable(BaseResourceIT.COLUMN_TYPE, values); + columns.add(ref); + Referenceable tableUpdated = new Referenceable(BaseResourceIT.HIVE_TABLE_TYPE, new HashMap<String, Object>() {{ + put("columns", columns); + }}); + + LOG.debug("Updating entity= " + tableUpdated); + serviceClient.updateEntity(tableId._getId(), tableUpdated); + + ClientResponse response = getEntityDefinition(tableId._getId()); + String definition = getEntityDefinition(response); + Referenceable getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true); + List<Referenceable> refs = (List<Referenceable>) getReferenceable.get("columns"); + + Assert.assertTrue(refs.get(0).equalsContents(columns.get(0))); + + //Update by unique attribute + values.put("dataType", "int"); + ref = new Referenceable(BaseResourceIT.COLUMN_TYPE, values); + columns.set(0, ref); + tableUpdated = new Referenceable(BaseResourceIT.HIVE_TABLE_TYPE, new HashMap<String, Object>() {{ + put("columns", columns); + }}); + + LOG.debug("Updating entity= " + tableUpdated); + serviceClient.updateEntity(BaseResourceIT.HIVE_TABLE_TYPE, "name", (String) tableInstance.get("name"), + tableUpdated); + + response = getEntityDefinition(tableId._getId()); + definition = getEntityDefinition(response); + getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true); + refs = (List<Referenceable>) getReferenceable.get("columns"); + + Assert.assertTrue(refs.get(0).equalsContents(columns.get(0))); + Assert.assertEquals(refs.get(0).get("dataType"), "int"); + + } + + @Test(dependsOnMethods = "testSubmitEntity") + public void testCompleteUpdate() throws Exception { + final List<Referenceable> columns = new ArrayList<>(); + Map<String, Object> values1 = new HashMap<>(); + values1.put("name", "col3"); + values1.put("dataType", "string"); + values1.put("comment", "col3 comment"); + + Map<String, Object> values2 = new HashMap<>(); + values2.put("name", "col4"); + values2.put("dataType", "string"); + values2.put("comment", "col4 comment"); + + Referenceable ref1 = new Referenceable(BaseResourceIT.COLUMN_TYPE, values1); + Referenceable ref2 = new Referenceable(BaseResourceIT.COLUMN_TYPE, values2); + columns.add(ref1); + columns.add(ref2); + tableInstance.set("columns", columns); + + LOG.debug("Replacing entity= " + tableInstance); + serviceClient.updateEntities(tableInstance); + + ClientResponse response = getEntityDefinition(tableId._getId()); + String definition = getEntityDefinition(response); + Referenceable getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true); + List<Referenceable> refs = (List<Referenceable>) getReferenceable.get("columns"); + Assert.assertEquals(refs.size(), 2); + + Assert.assertTrue(refs.get(0).equalsContents(columns.get(0))); + Assert.assertTrue(refs.get(1).equalsContents(columns.get(1))); + } }
