----------------------------------------------------------- This is an automatically generated e-mail. To reply, visit: https://reviews.apache.org/r/73169/#review222548 -----------------------------------------------------------
Ship it! Ship It! - Sarath Subramanian On Feb. 2, 2021, 5:52 p.m., Sidharth Mishra wrote: > > ----------------------------------------------------------- > This is an automatically generated e-mail. To reply, visit: > https://reviews.apache.org/r/73169/ > ----------------------------------------------------------- > > (Updated Feb. 2, 2021, 5:52 p.m.) > > > Review request for atlas, Ashutosh Mestry, Deep Singh, Madhan Neethiraj, > Radhika Kundam, and Sarath Subramanian. > > > Bugs: ATLAS-4123 > https://issues.apache.org/jira/browse/ATLAS-4123 > > > Repository: atlas > > > Description > ------- > > ATLAS-4123: Lineage missing in case of CREATE MATERIALIZED VIEW query at Hive > > > Diffs > ----- > > addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java > e48967d63 > > > Diff: https://reviews.apache.org/r/73169/diff/1/ > > > Testing > ------- > > Atlas hook topic message from the hive hook before fix: > ======================================================= > > { > "version": { > "version": "1.0.0", > "versionParts": [ > 1 > ] > }, > "msgCompressionKind": "NONE", > "msgSplitIdx": 1, > "msgSplitCount": 1, > "msgSourceIP": "172.27.10.2", > "msgCreatedBy": "hive", > "msgCreationTime": 1612311707304, > "message": { > "type": "ENTITY_CREATE_V2", > "user": "hive", > "entities": { > "referredEntities": { > "-73285185733726514": { > "typeName": "hive_storagedesc", > "attributes": { > "qualifiedName": "default.test_mv_ctas_2@cm_storage", > "storedAsSubDirectories": false, > "location": > "hdfs://sarath-secure-2.sarath-secure.root.hwx.site:8020/warehouse/tablespace/managed/hive/test_mv_ctas_2", > "compressed": false, > "inputFormat": > "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat", > "parameters": {}, > "outputFormat": > "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat", > "serdeInfo": { > "typeName": "hive_serde", > "attributes": { > "serializationLib": > "org.apache.hadoop.hive.ql.io.orc.OrcSerde", > "name": null, > "parameters": {} > } > }, > "numBuckets": -1 > }, > "guid": "-73285185733726514", > "isIncomplete": false, > "provenanceType": 0, > "version": 0, > "relationshipAttributes": { > "table": { > "guid": "-73285185733726513", > "typeName": "hive_table", > "uniqueAttributes": { > "qualifiedName": "default.test_mv_ctas_2@cm" > }, > "relationshipType": "hive_table_storagedesc" > } > }, > "proxy": false > }, > "-73285185733726515": { > "typeName": "hive_column", > "attributes": { > "owner": "admin", > "qualifiedName": "default.test_mv_ctas_2.id@cm", > "name": "id", > "comment": null, > "position": 0, > "type": "int" > }, > "guid": "-73285185733726515", > "isIncomplete": false, > "provenanceType": 0, > "version": 0, > "relationshipAttributes": { > "table": { > "guid": "-73285185733726513", > "typeName": "hive_table", > "uniqueAttributes": { > "qualifiedName": "default.test_mv_ctas_2@cm" > }, > "relationshipType": "hive_table_columns" > } > }, > "proxy": false > } > }, > "entities": [ > { > "typeName": "hive_table", > "attributes": { > "owner": "admin", > "viewExpandedText": "select `test_1`.`id` from > `default`.`test_1`", > "tableType": "MATERIALIZED_VIEW", > "temporary": false, > "lastAccessTime": 1612311707000, > "createTime": 1612311707000, > "qualifiedName": "default.test_mv_ctas_2@cm", > "name": "test_mv_ctas_2", > "comment": null, > "parameters": { > "transient_lastDdlTime": "1612311707", > "bucketing_version": "2", > "transactional_properties": "default", > "transactional": "true" > }, > "retention": 0, > "viewOriginalText": "select * from test_1" > }, > "guid": "-73285185733726513", > "isIncomplete": false, > "provenanceType": 0, > "version": 0, > "relationshipAttributes": { > "sd": { > "guid": "-73285185733726514", > "typeName": "hive_storagedesc", > "uniqueAttributes": { > "qualifiedName": > "default.test_mv_ctas_2@cm_storage" > }, > "relationshipType": "hive_table_storagedesc" > }, > "columns": [ > { > "guid": "-73285185733726515", > "typeName": "hive_column", > "uniqueAttributes": { > "qualifiedName": > "default.test_mv_ctas_2.id@cm" > }, > "relationshipType": "hive_table_columns" > } > ], > "partitionKeys": [], > "db": { > "typeName": "hive_db", > "uniqueAttributes": { > "qualifiedName": "default@cm" > }, > "relationshipType": "hive_table_db" > } > }, > "proxy": false > } > ] > } > } > } > > Atlas hook topic message from the hive hook after fix: > ======================================================= > { > "version": { > "version": "1.0.0", > "versionParts": [ > 1 > ] > }, > "msgCompressionKind": "NONE", > "msgSplitIdx": 1, > "msgSplitCount": 1, > "msgSourceIP": "172.27.10.2", > "msgCreatedBy": "hive", > "msgCreationTime": 1612317050425, > "message": { > "type": "ENTITY_CREATE_V2", > "user": "hive", > "entities": { > "referredEntities": { > "-73293806469890554": { > "typeName": "hive_column", > "attributes": { > "owner": "admin", > "qualifiedName": "default.test_mv_ctas_1.id@cm", > "name": "id", > "comment": null, > "position": 0, > "type": "int" > }, > "guid": "-73293806469890554", > "isIncomplete": false, > "provenanceType": 0, > "version": 0, > "relationshipAttributes": { > "table": { > "guid": "-73293806469890552", > "typeName": "hive_table", > "uniqueAttributes": { > "qualifiedName": "default.test_mv_ctas_1@cm" > }, > "relationshipType": "hive_table_columns" > } > }, > "proxy": false > }, > "-73293806469890553": { > "typeName": "hive_storagedesc", > "attributes": { > "qualifiedName": "default.test_mv_ctas_1@cm_storage", > "storedAsSubDirectories": false, > "location": > "hdfs://sarath-secure-2.sarath-secure.root.hwx.site:8020/warehouse/tablespace/managed/hive/test_mv_ctas_1", > "compressed": false, > "inputFormat": > "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat", > "parameters": {}, > "outputFormat": > "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat", > "serdeInfo": { > "typeName": "hive_serde", > "attributes": { > "serializationLib": > "org.apache.hadoop.hive.ql.io.orc.OrcSerde", > "name": null, > "parameters": {} > } > }, > "numBuckets": -1 > }, > "guid": "-73293806469890553", > "isIncomplete": false, > "provenanceType": 0, > "version": 0, > "relationshipAttributes": { > "table": { > "guid": "-73293806469890552", > "typeName": "hive_table", > "uniqueAttributes": { > "qualifiedName": "default.test_mv_ctas_1@cm" > }, > "relationshipType": "hive_table_storagedesc" > } > }, > "proxy": false > }, > "-73293806469890551": { > "typeName": "hive_db", > "attributes": { > "owner": "public", > "ownerType": "ROLE", > "managedLocation": null, > "qualifiedName": "default@cm", > "clusterName": "cm", > "name": "default", > "description": "Default Hive database", > "location": > "hdfs://sarath-secure-2.sarath-secure.root.hwx.site:8020/warehouse/tablespace/external/hive", > "parameters": {} > }, > "guid": "-73293806469890551", > "isIncomplete": false, > "provenanceType": 0, > "version": 0, > "proxy": false > } > }, > "entities": [ > { > "typeName": "hive_table", > "attributes": { > "owner": "admin", > "viewExpandedText": "select `test_1`.`id` from > `default`.`test_1`", > "tableType": "MATERIALIZED_VIEW", > "temporary": false, > "lastAccessTime": 1612317049000, > "createTime": 1612317049000, > "qualifiedName": "default.test_mv_ctas_1@cm", > "name": "test_mv_ctas_1", > "comment": null, > "parameters": { > "transient_lastDdlTime": "1612317049", > "bucketing_version": "2", > "transactional_properties": "default", > "transactional": "true" > }, > "retention": 0, > "viewOriginalText": "select * from test_1" > }, > "guid": "-73293806469890552", > "isIncomplete": false, > "provenanceType": 0, > "version": 0, > "relationshipAttributes": { > "sd": { > "guid": "-73293806469890553", > "typeName": "hive_storagedesc", > "uniqueAttributes": { > "qualifiedName": > "default.test_mv_ctas_1@cm_storage" > }, > "relationshipType": "hive_table_storagedesc" > }, > "columns": [ > { > "guid": "-73293806469890554", > "typeName": "hive_column", > "uniqueAttributes": { > "qualifiedName": > "default.test_mv_ctas_1.id@cm" > }, > "relationshipType": "hive_table_columns" > } > ], > "partitionKeys": [], > "db": { > "guid": "-73293806469890551", > "typeName": "hive_db", > "uniqueAttributes": { > "qualifiedName": "default@cm" > }, > "relationshipType": "hive_table_db" > } > }, > "proxy": false > } > ] > } > } > } > { > "version": { > "version": "1.0.0", > "versionParts": [ > 1 > ] > }, > "msgCompressionKind": "NONE", > "msgSplitIdx": 1, > "msgSplitCount": 1, > "msgSourceIP": "172.27.10.2", > "msgCreatedBy": "hive", > "msgCreationTime": 1612317051395, > "message": { > "type": "ENTITY_CREATE_V2", > "user": "admin", > "entities": { > "referredEntities": { > "-73293807317085531": { > "typeName": "hive_column", > "attributes": { > "owner": "admin", > "qualifiedName": "default.test_mv_ctas_1.id@cm", > "name": "id", > "comment": null, > "position": 0, > "type": "int" > }, > "guid": "-73293807317085531", > "isIncomplete": false, > "provenanceType": 0, > "version": 0, > "relationshipAttributes": { > "table": { > "guid": "-73293807317085529", > "typeName": "hive_table", > "uniqueAttributes": { > "qualifiedName": "default.test_mv_ctas_1@cm" > }, > "relationshipType": "hive_table_columns" > } > }, > "proxy": false > }, > "-73293807317085530": { > "typeName": "hive_storagedesc", > "attributes": { > "qualifiedName": "default.test_mv_ctas_1@cm_storage", > "storedAsSubDirectories": false, > "location": > "hdfs://sarath-secure-2.sarath-secure.root.hwx.site:8020/warehouse/tablespace/managed/hive/test_mv_ctas_1", > "compressed": false, > "inputFormat": > "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat", > "parameters": {}, > "outputFormat": > "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat", > "serdeInfo": { > "typeName": "hive_serde", > "attributes": { > "serializationLib": > "org.apache.hadoop.hive.ql.io.orc.OrcSerde", > "name": null, > "parameters": {} > } > }, > "numBuckets": -1 > }, > "guid": "-73293807317085530", > "isIncomplete": false, > "provenanceType": 0, > "version": 0, > "relationshipAttributes": { > "table": { > "guid": "-73293807317085529", > "typeName": "hive_table", > "uniqueAttributes": { > "qualifiedName": "default.test_mv_ctas_1@cm" > }, > "relationshipType": "hive_table_storagedesc" > } > }, > "proxy": false > }, > "-73293807317085529": { > "typeName": "hive_table", > "attributes": { > "owner": "admin", > "viewExpandedText": "select `test_1`.`id` from > `default`.`test_1`", > "tableType": "MATERIALIZED_VIEW", > "temporary": false, > "lastAccessTime": 1612317049000, > "createTime": 1612317049000, > "qualifiedName": "default.test_mv_ctas_1@cm", > "name": "test_mv_ctas_1", > "comment": null, > "parameters": { > "totalSize": "0", > "rawDataSize": "0", > "numRows": "0", > "transactional_properties": "default", > "COLUMN_STATS_ACCURATE": > "{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\"}}", > "numFiles": "0", > "transient_lastDdlTime": "1612317050", > "bucketing_version": "2", > "numFilesErasureCoded": "0", > "transactional": "true" > }, > "retention": 0, > "viewOriginalText": "select * from test_1" > }, > "guid": "-73293807317085529", > "isIncomplete": false, > "provenanceType": 0, > "version": 0, > "relationshipAttributes": { > "sd": { > "guid": "-73293807317085530", > "typeName": "hive_storagedesc", > "uniqueAttributes": { > "qualifiedName": > "default.test_mv_ctas_1@cm_storage" > }, > "relationshipType": "hive_table_storagedesc" > }, > "columns": [ > { > "guid": "-73293807317085531", > "typeName": "hive_column", > "uniqueAttributes": { > "qualifiedName": > "default.test_mv_ctas_1.id@cm" > }, > "relationshipType": "hive_table_columns" > } > ], > "partitionKeys": [], > "db": { > "guid": "-73293807317085525", > "typeName": "hive_db", > "uniqueAttributes": { > "qualifiedName": "default@cm" > }, > "relationshipType": "hive_table_db" > } > }, > "proxy": false > }, > "-73293807317085528": { > "typeName": "hive_column", > "attributes": { > "owner": "admin", > "qualifiedName": "default.test_1.id@cm", > "name": "id", > "comment": null, > "position": 0, > "type": "int" > }, > "guid": "-73293807317085528", > "isIncomplete": false, > "provenanceType": 0, > "version": 0, > "relationshipAttributes": { > "table": { > "guid": "-73293807317085526", > "typeName": "hive_table", > "uniqueAttributes": { > "qualifiedName": "default.test_1@cm" > }, > "relationshipType": "hive_table_columns" > } > }, > "proxy": false > }, > "-73293807317085527": { > "typeName": "hive_storagedesc", > "attributes": { > "qualifiedName": "default.test_1@cm_storage", > "storedAsSubDirectories": false, > "location": > "hdfs://sarath-secure-2.sarath-secure.root.hwx.site:8020/warehouse/tablespace/managed/hive/test_1", > "compressed": false, > "inputFormat": > "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat", > "parameters": {}, > "outputFormat": > "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat", > "serdeInfo": { > "typeName": "hive_serde", > "attributes": { > "serializationLib": > "org.apache.hadoop.hive.ql.io.orc.OrcSerde", > "name": null, > "parameters": {} > } > }, > "numBuckets": -1 > }, > "guid": "-73293807317085527", > "isIncomplete": false, > "provenanceType": 0, > "version": 0, > "relationshipAttributes": { > "table": { > "guid": "-73293807317085526", > "typeName": "hive_table", > "uniqueAttributes": { > "qualifiedName": "default.test_1@cm" > }, > "relationshipType": "hive_table_storagedesc" > } > }, > "proxy": false > }, > "-73293807317085526": { > "typeName": "hive_table", > "attributes": { > "owner": "admin", > "tableType": "MANAGED_TABLE", > "temporary": false, > "lastAccessTime": 1612314520000, > "createTime": 1612314520000, > "qualifiedName": "default.test_1@cm", > "name": "test_1", > "comment": null, > "parameters": { > "totalSize": "0", > "rawDataSize": "0", > "numRows": "0", > "transactional_properties": "default", > "COLUMN_STATS_ACCURATE": > "{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\"}}", > "numFiles": "0", > "transient_lastDdlTime": "1612314520", > "bucketing_version": "2", > "numFilesErasureCoded": "0", > "transactional": "true" > }, > "retention": 0 > }, > "guid": "-73293807317085526", > "isIncomplete": false, > "provenanceType": 0, > "version": 0, > "relationshipAttributes": { > "sd": { > "guid": "-73293807317085527", > "typeName": "hive_storagedesc", > "uniqueAttributes": { > "qualifiedName": "default.test_1@cm_storage" > }, > "relationshipType": "hive_table_storagedesc" > }, > "columns": [ > { > "guid": "-73293807317085528", > "typeName": "hive_column", > "uniqueAttributes": { > "qualifiedName": "default.test_1.id@cm" > }, > "relationshipType": "hive_table_columns" > } > ], > "partitionKeys": [], > "db": { > "guid": "-73293807317085525", > "typeName": "hive_db", > "uniqueAttributes": { > "qualifiedName": "default@cm" > }, > "relationshipType": "hive_table_db" > } > }, > "proxy": false > }, > "-73293807317085525": { > "typeName": "hive_db", > "attributes": { > "owner": "public", > "ownerType": "ROLE", > "managedLocation": > "hdfs://sarath-secure-2.sarath-secure.root.hwx.site:8020/warehouse/tablespace/managed/hive", > "qualifiedName": "default@cm", > "clusterName": "cm", > "name": "default", > "description": "Default Hive database", > "location": > "hdfs://sarath-secure-2.sarath-secure.root.hwx.site:8020/warehouse/tablespace/external/hive", > "parameters": {} > }, > "guid": "-73293807317085525", > "isIncomplete": false, > "provenanceType": 0, > "version": 0, > "proxy": false > } > }, > "entities": [ > { > "typeName": "hive_process", > "attributes": { > "recentQueries": [ > "create materialized view test_mv_ctas_1 as > select * from test_1" > ], > "qualifiedName": > "CREATE_MATERIALIZED_VIEW:default.test_1@cm:1612314520000->:default@cm:default.test_mv_ctas_1@cm:1612317049000", > "clusterName": "cm", > "name": > "CREATE_MATERIALIZED_VIEW:default.test_1@cm:1612314520000->:default@cm:default.test_mv_ctas_1@cm:1612317049000", > "queryText": "", > "operationType": "CREATE_MATERIALIZED_VIEW", > "startTime": 1612317051380, > "queryPlan": "Not Supported", > "endTime": 1612317051380, > "userName": "", > "queryId": "" > }, > "guid": "-73293807317085532", > "isIncomplete": false, > "provenanceType": 0, > "version": 0, > "relationshipAttributes": { > "outputs": [ > { > "guid": "-73293807317085529", > "typeName": "hive_table", > "uniqueAttributes": { > "qualifiedName": > "default.test_mv_ctas_1@cm" > }, > "relationshipType": "process_dataset_outputs" > } > ], > "inputs": [ > { > "guid": "-73293807317085526", > "typeName": "hive_table", > "uniqueAttributes": { > "qualifiedName": "default.test_1@cm" > }, > "relationshipType": "dataset_process_inputs" > } > ] > }, > "proxy": false > }, > { > "typeName": "hive_process_execution", > "attributes": { > "hostName": > "sarath-secure-2.sarath-secure.root.hwx.site", > "qualifiedName": > "CREATE_MATERIALIZED_VIEW:default.test_1@cm:1612314520000->:default@cm:default.test_mv_ctas_1@cm:1612317049000:1612317032474:1612317051380", > "name": > "CREATE_MATERIALIZED_VIEW:default.test_1@cm:1612314520000->:default@cm:default.test_mv_ctas_1@cm:1612317049000:1612317032474:1612317051380", > "queryText": "create materialized view test_mv_ctas_1 > as select * from test_1", > "startTime": 1612317032474, > "queryPlan": "Not Supported", > "endTime": 1612317051380, > "userName": "admin", > "queryId": > "hive_20210203015032_7bef0f08-bddf-4992-b761-97aa261be104" > }, > "guid": "-73293807317085533", > "isIncomplete": false, > "provenanceType": 0, > "version": 0, > "relationshipAttributes": { > "process": { > "guid": "-73293807317085532", > "typeName": "hive_process", > "relationshipType": > "hive_process_process_executions" > } > }, > "proxy": false > } > ] > } > } > } > > > Thanks, > > Sidharth Mishra > >
