[ https://issues.apache.org/jira/browse/ATLAS-4152?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17350702#comment-17350702 ]
ASF subversion and git services commented on ATLAS-4152: -------------------------------------------------------- Commit f3afbdb3953e8ddbfcab6177083227b6cb89ecba in atlas's branch refs/heads/master from Ashutosh Mestry [ https://gitbox.apache.org/repos/asf?p=atlas.git;h=f3afbdb ] ATLAS-4152: Entity correlation for deleted entities. Part #2. Unit test fix. > [Atlas: Spooling] Multiple entries are created for same table when the table > is dropped while kafka is down > ----------------------------------------------------------------------------------------------------------- > > Key: ATLAS-4152 > URL: https://issues.apache.org/jira/browse/ATLAS-4152 > Project: Atlas > Issue Type: Bug > Components: atlas-core > Reporter: Dharshana M Krishnamoorthy > Assignee: Ashutosh Mestry > Priority: Major > > A single table has multiple duplicate entries when the table is dropped while > kafka is down (Spooling scenario) > Steps to re-produce: > * Stop kafka broker > * Create a few (20) tables save the prefix (abc_table_1, abc_table_2, ... > abc_table_n) > * Make sure the data is spooled > * Start kafka and create a few more tables (xyz_table_1, xyz_table_2, ... > xyz_table_n) > * Wait for 5 mins for the tables to reflect in atlas > * Fire a basic search with abc_* prefix and verify the tables are created > * Collect the createdTime of all the table and verify the order > Here tables were created with prefix btwxb_table. > *btwxb_table_5, btwxb_table_10, btwxb_table_15 and btwxb_table_20* are > dropped when kafka is down > Each of those tables are having a total of 3 entries per table_name. All the > tetails are same except the guid > {code:java} > { > "searchParameters":{ > "includeSubTypes":true, > "excludeDeletedEntities":false, > "includeSubClassifications":true, > "typeName":"hive_table", > "limit":40, > "offset":0, > "includeClassificationAttributes":false, > "query":"btwxb*" > }, > "queryText":"btwxb*", > "approximateCount":28, > "queryType":"BASIC", > "entities":[ > { > "status":"DELETED", > "isIncomplete":false, > "guid":"9c348843-ebf0-4a0f-a909-4cdbf75ea39d", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_15", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_15@cm", > "createTime":1612959528000, > "name":"btwxb_table_15" > }, > "classificationNames":[ > > ] > }, > { > "status":"DELETED", > "isIncomplete":false, > "guid":"a25fbfc3-f6fb-4f9f-bce4-1da3f2de5921", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_5", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_5@cm", > "createTime":1612959525000, > "name":"btwxb_table_5" > }, > "classificationNames":[ > > ] > }, > { > "status":"ACTIVE", > "isIncomplete":false, > "guid":"bf6de3f6-48f2-4941-b5f2-a1ef3756ffa2", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_8", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_8@cm", > "createTime":1612959527000, > "name":"btwxb_table_8" > }, > "classificationNames":[ > > ] > }, > { > "status":"ACTIVE", > "isIncomplete":false, > "guid":"977adf75-9336-484f-91ed-7976817fb729", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_7", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_7@cm", > "createTime":1612959527000, > "name":"btwxb_table_7" > }, > "classificationNames":[ > > ] > }, > { > "status":"ACTIVE", > "isIncomplete":false, > "guid":"d43fb189-2dd5-4682-8cc1-d2b8566be7fb", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_9", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_9@cm", > "createTime":1612959527000, > "name":"btwxb_table_9" > }, > "classificationNames":[ > > ] > }, > { > "status":"ACTIVE", > "isIncomplete":false, > "guid":"a55ca87c-1122-4ed4-9812-5047cc8a7968", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_6", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_6@cm", > "createTime":1612959527000, > "name":"btwxb_table_6" > }, > "classificationNames":[ > > ] > }, > { > "status":"DELETED", > "isIncomplete":false, > "guid":"ac786189-fbf6-4503-9aad-6547c1cc63ff", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_15", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_15@cm", > "createTime":1612959528000, > "name":"btwxb_table_15" > }, > "classificationNames":[ > > ] > }, > { > "status":"ACTIVE", > "isIncomplete":false, > "guid":"83503968-7095-48af-ada4-03e71217e64b", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_16", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_16@cm", > "createTime":1612959529000, > "name":"btwxb_table_16" > }, > "classificationNames":[ > > ] > }, > { > "status":"ACTIVE", > "isIncomplete":false, > "guid":"93b49b64-64aa-4bf3-9827-3571278a8c32", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_18", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_18@cm", > "createTime":1612959529000, > "name":"btwxb_table_18" > }, > "classificationNames":[ > > ] > }, > { > "status":"ACTIVE", > "isIncomplete":false, > "guid":"126a33a6-5eca-49e2-aa7a-1eb00d996da9", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_19", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_19@cm", > "createTime":1612959530000, > "name":"btwxb_table_19" > }, > "classificationNames":[ > > ] > }, > { > "status":"ACTIVE", > "isIncomplete":false, > "guid":"8edcc1c2-efb9-449d-87cc-820ebc015bed", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_17", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_17@cm", > "createTime":1612959529000, > "name":"btwxb_table_17" > }, > "classificationNames":[ > > ] > }, > { > "status":"DELETED", > "isIncomplete":false, > "guid":"dd00cd2f-cea4-4253-8f2a-11be0ba5384c", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_10", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_10@cm", > "createTime":1612959527000, > "name":"btwxb_table_10" > }, > "classificationNames":[ > > ] > }, > { > "status":"DELETED", > "isIncomplete":false, > "guid":"f775d77e-384c-481b-99cf-a0ad14048d67", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_15", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_15@cm", > "createTime":1612959528000, > "name":"btwxb_table_15" > }, > "classificationNames":[ > > ] > }, > { > "status":"DELETED", > "isIncomplete":false, > "guid":"495ffb98-4e7d-428b-8eb3-404573cd5255", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_5", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_5@cm", > "createTime":1612959525000, > "name":"btwxb_table_5" > }, > "classificationNames":[ > > ] > }, > { > "status":"DELETED", > "isIncomplete":false, > "guid":"50df8787-f4fc-464e-86ba-4f40d4b7c9fc", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_5", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_5@cm", > "createTime":1612959525000, > "name":"btwxb_table_5" > }, > "classificationNames":[ > > ] > }, > { > "status":"DELETED", > "isIncomplete":false, > "guid":"08d8a9ef-d44e-434e-8be9-a83d4748969f", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_10", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_10@cm", > "createTime":1612959527000, > "name":"btwxb_table_10" > }, > "classificationNames":[ > > ] > }, > { > "status":"ACTIVE", > "isIncomplete":false, > "guid":"fc9274b6-9aef-43ba-8237-4387bc46a16a", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_3", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_3@cm", > "createTime":1612959524000, > "name":"btwxb_table_3" > }, > "classificationNames":[ > > ] > }, > { > "status":"ACTIVE", > "isIncomplete":false, > "guid":"08e0bcd4-5945-4581-9451-9dc47872f409", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_2", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_2@cm", > "createTime":1612959524000, > "name":"btwxb_table_2" > }, > "classificationNames":[ > > ] > }, > { > "status":"ACTIVE", > "isIncomplete":false, > "guid":"cff905a7-829e-4545-880a-bae33887b524", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_4", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_4@cm", > "createTime":1612959524000, > "name":"btwxb_table_4" > }, > "classificationNames":[ > > ] > }, > { > "status":"DELETED", > "isIncomplete":false, > "guid":"b5108bcf-2310-41e5-8fb1-ada0226b1ff9", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_20", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_20@cm", > "createTime":1612959531000, > "name":"btwxb_table_20" > }, > "classificationNames":[ > > ] > }, > { > "status":"DELETED", > "isIncomplete":false, > "guid":"367ce1eb-c624-479d-ac55-d093507cabd7", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_10", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_10@cm", > "createTime":1612959527000, > "name":"btwxb_table_10" > }, > "classificationNames":[ > > ] > }, > { > "status":"ACTIVE", > "isIncomplete":false, > "guid":"d6586202-1aa6-419f-83c6-ba4e008334d1", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_13", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_13@cm", > "createTime":1612959528000, > "name":"btwxb_table_13" > }, > "classificationNames":[ > > ] > }, > { > "status":"ACTIVE", > "isIncomplete":false, > "guid":"38772c4b-211b-410c-962b-41018337f8a0", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_11", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_11@cm", > "createTime":1612959528000, > "name":"btwxb_table_11" > }, > "classificationNames":[ > > ] > }, > { > "status":"ACTIVE", > "isIncomplete":false, > "guid":"c2f0aa63-df72-4836-968f-7b87c673cb12", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_12", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_12@cm", > "createTime":1612959528000, > "name":"btwxb_table_12" > }, > "classificationNames":[ > > ] > }, > { > "status":"ACTIVE", > "isIncomplete":false, > "guid":"07408cc7-8828-4e20-87ca-ce76ad72b443", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_14", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_14@cm", > "createTime":1612959528000, > "name":"btwxb_table_14" > }, > "classificationNames":[ > > ] > }, > { > "status":"ACTIVE", > "isIncomplete":false, > "guid":"70738a7e-d29f-4234-82e7-adbdc4feaed6", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_1", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_1@cm", > "createTime":1612959523000, > "name":"btwxb_table_1" > }, > "classificationNames":[ > > ] > }, > { > "status":"DELETED", > "isIncomplete":false, > "guid":"49151c4e-e8a0-482c-9360-0ff4eec99692", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_20", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_20@cm", > "createTime":1612959531000, > "name":"btwxb_table_20" > }, > "classificationNames":[ > > ] > }, > { > "status":"DELETED", > "isIncomplete":false, > "guid":"6b4bed96-816a-4c33-a61f-44a6a66559a4", > "meanings":[ > > ], > "labels":[ > > ], > "typeName":"hive_table", > "meaningNames":[ > > ], > "displayText":"btwxb_table_20", > "attributes":{ > "owner":"hrt_qa", > "qualifiedName":"default.btwxb_table_20@cm", > "createTime":1612959531000, > "name":"btwxb_table_20" > }, > "classificationNames":[ > > ] > } > ] > } {code} -- This message was sent by Atlassian Jira (v8.3.4#803005)