Repository: incubator-atlas Updated Branches: refs/heads/master 915595780 -> bfd5f5caa
ATLAS-738 Add query ability on system properties like guid, state, createdtime etc (shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/bfd5f5ca Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/bfd5f5ca Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/bfd5f5ca Branch: refs/heads/master Commit: bfd5f5caa609b9084ec816deb4c6bf11eec18bab Parents: 9155957 Author: Shwetha GS <[email protected]> Authored: Thu May 12 12:02:46 2016 +0530 Committer: Shwetha GS <[email protected]> Committed: Thu May 12 12:02:46 2016 +0530 ---------------------------------------------------------------------- release-log.txt | 1 + .../org/apache/atlas/repository/Constants.java | 16 ++++ .../graph/GraphBackedMetadataRepository.java | 3 + .../org/apache/atlas/query/TypeUtils.scala | 6 ++ .../GraphBackedDiscoveryServiceTest.java | 98 +++++++++++++------- .../atlas/typesystem/types/AttributeInfo.java | 2 +- .../atlas/typesystem/types/utils/TypesUtil.java | 14 +++ 7 files changed, 106 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 7e3149c..66064d6 100644 --- a/release-log.txt +++ b/release-log.txt @@ -20,6 +20,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-738 Add query ability on system properties like guid, state, createdtime etc (shwethags) ATLAS-692 Create abstraction layer for graph databases (jnhagelb via yhemanth) ATLAS-689 Migrate Atlas-Storm integration to use Storm 1.0 dependencies. (svimal2106 via yhemanth) ATLAS-754 InstanceSerialization does not serialize Reference in the values array of Reference.(harishjp via sumasai) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/repository/src/main/java/org/apache/atlas/repository/Constants.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/Constants.java b/repository/src/main/java/org/apache/atlas/repository/Constants.java index 0832c81..893f1b6 100755 --- a/repository/src/main/java/org/apache/atlas/repository/Constants.java +++ b/repository/src/main/java/org/apache/atlas/repository/Constants.java @@ -18,6 +18,10 @@ package org.apache.atlas.repository; +import org.apache.atlas.typesystem.types.AttributeInfo; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.utils.TypesUtil; + public final class Constants { /** @@ -62,6 +66,18 @@ public final class Constants { public static final String TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "timestamp"; public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "modificationTimestamp"; + public static AttributeInfo getAttributeInfoForSystemAttributes(String field) { + switch (field) { + case STATE_PROPERTY_KEY: + case GUID_PROPERTY_KEY: + return TypesUtil.newAttributeInfo(field, DataTypes.STRING_TYPE); + + case TIMESTAMP_PROPERTY_KEY: + case MODIFICATION_TIMESTAMP_PROPERTY_KEY: + return TypesUtil.newAttributeInfo(field, DataTypes.LONG_TYPE); + } + return null; + } /** * search backing index name. http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java index de29e86..3604277 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java @@ -105,6 +105,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository { @Override public String getFieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo) throws AtlasException { + if (aInfo.name.startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX)) { + return aInfo.name; + } return GraphHelper.getQualifiedFieldName(dataType, aInfo.name); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala b/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala index 552cc32..5a64c53 100755 --- a/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala +++ b/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala @@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicInteger import org.apache.atlas.AtlasException import org.apache.atlas.query.Expressions.{PathExpression, SelectExpression} +import org.apache.atlas.repository.Constants import org.apache.atlas.typesystem.types.DataTypes.{ArrayType, PrimitiveType, TypeCategory} import org.apache.atlas.typesystem.types._ @@ -203,6 +204,11 @@ object TypeUtils { return Some(FieldInfo(typ,fMap.get.fields.get(id))) } + val systemField = Constants.getAttributeInfoForSystemAttributes(id) + if (systemField != null) { + return Some(FieldInfo(systemField.dataType(), systemField)) + } + try { val FIELD_QUALIFIER(clsNm, rest) = id val idTyp = typSystem.getDataType(classOf[IDataType[_]], clsNm) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java index 9cc501f..3b50dfb 100755 --- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java @@ -56,6 +56,8 @@ import java.util.Map; import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef; import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef; import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; @Guice(modules = RepositoryMetadataModule.class) public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest { @@ -94,32 +96,62 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest { } @Test + public void testSearchBySystemProperties() throws Exception { + //system property in select + String dslQuery = "from Department select __guid"; + + String jsonResults = discoveryService.searchByDSL(dslQuery); + assertNotNull(jsonResults); + + JSONObject results = new JSONObject(jsonResults); + assertEquals(results.length(), 3); + + JSONArray rows = results.getJSONArray("rows"); + assertNotNull(rows); + assertEquals(rows.length(), 1); + assertNotNull(rows.getJSONObject(0).getString("__guid")); + + //system property in where clause + String guid = rows.getJSONObject(0).getString("__guid"); + dslQuery = "Department where __guid = '" + guid + "' and __state = 'ACTIVE'"; + jsonResults = discoveryService.searchByDSL(dslQuery); + assertNotNull(jsonResults); + + results = new JSONObject(jsonResults); + assertEquals(results.length(), 3); + + rows = results.getJSONArray("rows"); + assertNotNull(rows); + assertEquals(rows.length(), 1); + } + + @Test public void testSearchByDSLReturnsEntity() throws Exception { String dslQuery = "from Department"; String jsonResults = discoveryService.searchByDSL(dslQuery); - Assert.assertNotNull(jsonResults); + assertNotNull(jsonResults); JSONObject results = new JSONObject(jsonResults); - Assert.assertEquals(results.length(), 3); + assertEquals(results.length(), 3); System.out.println("results = " + results); Object query = results.get("query"); - Assert.assertNotNull(query); + assertNotNull(query); JSONObject dataType = results.getJSONObject("dataType"); - Assert.assertNotNull(dataType); + assertNotNull(dataType); String typeName = dataType.getString("typeName"); - Assert.assertNotNull(typeName); - Assert.assertEquals(typeName, "Department"); + assertNotNull(typeName); + assertEquals(typeName, "Department"); JSONArray rows = results.getJSONArray("rows"); - Assert.assertNotNull(rows); - Assert.assertEquals(rows.length(), 1); + assertNotNull(rows); + assertEquals(rows.length(), 1); //Assert that entity state is set in the result entities String entityState = rows.getJSONObject(0).getJSONObject("$id$").getString("state"); - Assert.assertEquals(entityState, Id.EntityState.ACTIVE.name()); + assertEquals(entityState, Id.EntityState.ACTIVE.name()); } @Test(expectedExceptions = Throwable.class) @@ -167,11 +199,11 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest { Assert.assertTrue(resultList.size() > 0); for (Map<String, Object> vertexProps : resultList) { Object object = vertexProps.get(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY); - Assert.assertNotNull(object); + assertNotNull(object); Long timestampAsLong = Long.valueOf((String)object); Assert.assertTrue(timestampAsLong > 1420070400000L); object = vertexProps.get(Constants.TIMESTAMP_PROPERTY_KEY); - Assert.assertNotNull(object); + assertNotNull(object); } } @@ -490,23 +522,23 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest { public void testSearchByDSLQueriesWithOrderBy(String dslQuery, Integer expectedNumRows, String orderBy, boolean ascending) throws Exception { System.out.println("Executing dslQuery = " + dslQuery); String jsonResults = discoveryService.searchByDSL(dslQuery); - Assert.assertNotNull(jsonResults); + assertNotNull(jsonResults); JSONObject results = new JSONObject(jsonResults); - Assert.assertEquals(results.length(), 3); + assertEquals(results.length(), 3); Object query = results.get("query"); - Assert.assertNotNull(query); + assertNotNull(query); JSONObject dataType = results.getJSONObject("dataType"); - Assert.assertNotNull(dataType); + assertNotNull(dataType); String typeName = dataType.getString("typeName"); - Assert.assertNotNull(typeName); + assertNotNull(typeName); JSONArray rows = results.getJSONArray("rows"); - Assert.assertNotNull(rows); - Assert.assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not have any results + assertNotNull(rows); + assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not have any results List<String> returnedList = new ArrayList<String> (); for (int i = 0; i < rows.length(); i++) { JSONObject row = rows.getJSONObject(i); @@ -546,23 +578,23 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest { public void testSearchByDSLQueries(String dslQuery, Integer expectedNumRows) throws Exception { System.out.println("Executing dslQuery = " + dslQuery); String jsonResults = discoveryService.searchByDSL(dslQuery); - Assert.assertNotNull(jsonResults); + assertNotNull(jsonResults); JSONObject results = new JSONObject(jsonResults); - Assert.assertEquals(results.length(), 3); + assertEquals(results.length(), 3); System.out.println("results = " + results); Object query = results.get("query"); - Assert.assertNotNull(query); + assertNotNull(query); JSONObject dataType = results.getJSONObject("dataType"); - Assert.assertNotNull(dataType); + assertNotNull(dataType); String typeName = dataType.getString("typeName"); - Assert.assertNotNull(typeName); + assertNotNull(typeName); JSONArray rows = results.getJSONArray("rows"); - Assert.assertNotNull(rows); - Assert.assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not have any results + assertNotNull(rows); + assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not have any results System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows"); } @@ -570,23 +602,23 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest { public void testSearchByDSLQueriesWithLimit(String dslQuery, Integer expectedNumRows) throws Exception { System.out.println("Executing dslQuery = " + dslQuery); String jsonResults = discoveryService.searchByDSL(dslQuery); - Assert.assertNotNull(jsonResults); + assertNotNull(jsonResults); JSONObject results = new JSONObject(jsonResults); - Assert.assertEquals(results.length(), 3); + assertEquals(results.length(), 3); System.out.println("results = " + results); Object query = results.get("query"); - Assert.assertNotNull(query); + assertNotNull(query); JSONObject dataType = results.getJSONObject("dataType"); - Assert.assertNotNull(dataType); + assertNotNull(dataType); String typeName = dataType.getString("typeName"); - Assert.assertNotNull(typeName); + assertNotNull(typeName); JSONArray rows = results.getJSONArray("rows"); - Assert.assertNotNull(rows); - Assert.assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not have any results + assertNotNull(rows); + assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not have any results System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows"); } @@ -609,7 +641,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest { String dslQuery = "from D where a = 1"; String jsonResults = discoveryService.searchByDSL(dslQuery); - Assert.assertNotNull(jsonResults); + assertNotNull(jsonResults); JSONObject results = new JSONObject(jsonResults); System.out.println("results = " + results); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java index 330d1cb..e748579 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java @@ -38,7 +38,7 @@ public class AttributeInfo { public final String reverseAttributeName; private IDataType dataType; - AttributeInfo(TypeSystem t, AttributeDefinition def, Map<String, IDataType> tempTypes) throws AtlasException { + public AttributeInfo(TypeSystem t, AttributeDefinition def, Map<String, IDataType> tempTypes) throws AtlasException { this.name = def.name; this.dataType = (tempTypes != null && tempTypes.containsKey(def.dataTypeName)) ? tempTypes.get(def.dataTypeName) : http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java index 2ea90a3..ef8448d 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java @@ -21,8 +21,10 @@ package org.apache.atlas.typesystem.types.utils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import org.apache.atlas.AtlasException; import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.types.AttributeDefinition; +import org.apache.atlas.typesystem.types.AttributeInfo; import org.apache.atlas.typesystem.types.ClassType; import org.apache.atlas.typesystem.types.EnumTypeDefinition; import org.apache.atlas.typesystem.types.EnumValue; @@ -32,6 +34,7 @@ import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.TypeSystem; import scala.collection.JavaConversions; /** @@ -100,4 +103,15 @@ public class TypesUtil { return new TypesDef(JavaConversions.asScalaBuffer(enums), JavaConversions.asScalaBuffer(structs), JavaConversions.asScalaBuffer(traits), JavaConversions.asScalaBuffer(classes)); } + + private static final TypeSystem ts = TypeSystem.getInstance(); + + public static AttributeInfo newAttributeInfo(String attribute, IDataType type) { + try { + return new AttributeInfo(ts, new AttributeDefinition(attribute, type.getName(), Multiplicity.REQUIRED, + false, null), null); + } catch (AtlasException e) { + throw new RuntimeException(e); + } + } }
