HIVE-11279 : Hive should emit lineage information in json compact format (Lenni Kuff via Szehon)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2b1f03e3 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2b1f03e3 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2b1f03e3 Branch: refs/heads/hbase-metastore Commit: 2b1f03e33d2bbfc6916e80caff963475c2740f82 Parents: 7788968 Author: Szehon Ho <sze...@cloudera.com> Authored: Fri Jul 17 11:19:25 2015 -0700 Committer: Szehon Ho <sze...@cloudera.com> Committed: Fri Jul 17 11:19:55 2015 -0700 ---------------------------------------------------------------------- .../hadoop/hive/ql/hooks/LineageLogger.java | 4 +- .../test/results/clientpositive/lineage2.q.out | 2296 +----------------- .../test/results/clientpositive/lineage3.q.out | 2235 +---------------- 3 files changed, 58 insertions(+), 4477 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2b1f03e3/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java index fc32af7..d615372 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java @@ -137,9 +137,7 @@ public class LineageLogger implements ExecuteWithHookContext { try { StringBuilderWriter out = new StringBuilderWriter(1024); JsonWriter writer = new JsonWriter(out); - writer.setIndent(" "); - out.append("POSTHOOK: LINEAGE: "); String queryStr = plan.getQueryStr().trim(); writer.beginObject(); writer.name("version").value(FORMAT_VERSION); @@ -182,7 +180,7 @@ public class LineageLogger implements ExecuteWithHookContext { // Log to console log(lineage); } else { - // In none test mode, emit to a log file, + // In non-test mode, emit to a log file, // which can be different from the normal hive.log. // For example, using NoDeleteRollingFileAppender to // log to some file with different rolling policy. http://git-wip-us.apache.org/repos/asf/hive/blob/2b1f03e3/ql/src/test/results/clientpositive/lineage2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/lineage2.q.out b/ql/src/test/results/clientpositive/lineage2.q.out index 669be97..9b227c6 100644 --- a/ql/src/test/results/clientpositive/lineage2.q.out +++ b/ql/src/test/results/clientpositive/lineage2.q.out @@ -5,118 +5,12 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src1 PREHOOK: Output: database:default PREHOOK: Output: default@src2 -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "3a39d46286e4c2cd2139c9bb248f7b4f", - "queryText": "create table src2 as select key key2, value value2 from src1", - "edges": [ - { - "sources": [ - 2 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 3 - ], - "targets": [ - 1 - ], - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "default.src2.key2" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.src2.value2" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.src1.key" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.src1.value" - } - ] -} +{"version":"1.0","engine":"mr","hash":"3a39d46286e4c2cd2139c9bb248f7b4f","queryText":"create table src2 as select key key2, value value2 from src1","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.src2.key2"},{"id":1,"vertexType":"COLUMN","vertexId":"default.src2.value2"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"}]} PREHOOK: query: select * from src1 where key is not null and value is not null limit 3 PREHOOK: type: QUERY PREHOOK: Input: default@src1 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "b5b224847b2333e790a2c229434a04c8", - "queryText": "select * from src1 where key is not null and value is not null limit 3", - "edges": [ - { - "sources": [ - 2 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 3 - ], - "targets": [ - 1 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 2, - 3 - ], - "targets": [ - 0, - 1 - ], - "expression": "(src1.key is not null and src1.value is not null)", - "edgeType": "PREDICATE" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "src1.key" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "src1.value" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.src1.key" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.src1.value" - } - ] -} +{"version":"1.0","engine":"mr","hash":"b5b224847b2333e790a2c229434a04c8","queryText":"select * from src1 where key is not null and value is not null limit 3","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2,3],"targets":[0,1],"expression":"(src1.key is not null and src1.value is not null)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"src1.key"},{"id":1,"vertexType":"COLUMN","vertexId":"src1.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"}]} 238 val_238 311 val_311 @@ -124,66 +18,7 @@ PREHOOK: query: select * from src1 where key > 10 and value > 'val' order by key PREHOOK: type: QUERY PREHOOK: Input: default@src1 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "773d9d0ea92e797eae292ae1eeea11ab", - "queryText": "select * from src1 where key > 10 and value > 'val' order by key limit 5", - "edges": [ - { - "sources": [ - 2 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 3 - ], - "targets": [ - 1 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 2, - 3 - ], - "targets": [ - 0, - 1 - ], - "expression": "((UDFToDouble(src1.key) > UDFToDouble(10)) and (src1.value > 'val'))", - "edgeType": "PREDICATE" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "src1.key" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "src1.value" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.src1.key" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.src1.value" - } - ] -} +{"version":"1.0","engine":"mr","hash":"773d9d0ea92e797eae292ae1eeea11ab","queryText":"select * from src1 where key > 10 and value > 'val' order by key limit 5","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2,3],"targets":[0,1],"expression":"((UDFToDouble(src1.key) > UDFToDouble(10)) and (src1.value > 'val'))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"src1.key"},{"id":1,"vertexType":"COLUMN","vertexId":"src1.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"}]} 146 val_146 150 val_150 213 val_213 @@ -196,158 +31,17 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src1 PREHOOK: Output: database:default PREHOOK: Output: default@dest1 -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "712fe958c357bcfc978b95c43eb19084", - "queryText": "create table dest1 as select * from src1", - "edges": [ - { - "sources": [ - 2 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 3 - ], - "targets": [ - 1 - ], - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "default.dest1.key" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.dest1.value" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.src1.key" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.src1.value" - } - ] -} +{"version":"1.0","engine":"mr","hash":"712fe958c357bcfc978b95c43eb19084","queryText":"create table dest1 as select * from src1","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest1.key"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest1.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"}]} PREHOOK: query: insert into table dest1 select * from src2 PREHOOK: type: QUERY PREHOOK: Input: default@src2 PREHOOK: Output: default@dest1 -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "ecc718a966d8887b18084a55dd96f0bc", - "queryText": "insert into table dest1 select * from src2", - "edges": [ - { - "sources": [ - 2 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 3 - ], - "targets": [ - 1 - ], - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "default.dest1.key" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.dest1.value" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.src2.key2" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.src2.value2" - } - ] -} +{"version":"1.0","engine":"mr","hash":"ecc718a966d8887b18084a55dd96f0bc","queryText":"insert into table dest1 select * from src2","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest1.key"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest1.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src2.key2"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src2.value2"}]} PREHOOK: query: select key k, dest1.value from dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "416b6f4cd63edd4f9d8213d2d7819d21", - "queryText": "select key k, dest1.value from dest1", - "edges": [ - { - "sources": [ - 2 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 3 - ], - "targets": [ - 1 - ], - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "k" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "dest1.value" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.dest1.key" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.dest1.value" - } - ] -} +{"version":"1.0","engine":"mr","hash":"416b6f4cd63edd4f9d8213d2d7819d21","queryText":"select key k, dest1.value from dest1","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"k"},{"id":1,"vertexType":"COLUMN","vertexId":"dest1.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest1.value"}]} 238 val_238 311 val_311 @@ -403,42 +97,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src1 PREHOOK: Input: default@src2 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "50fa3d1074b3fda37ce11dc6ec92ebf3", - "queryText": "select key from src1 union select key2 from src2 order by key", - "edges": [ - { - "sources": [ - 1, - 2 - ], - "targets": [ - 0 - ], - "expression": "key", - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "u2.key" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.src1.key" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.src2.key2" - } - ] -} +{"version":"1.0","engine":"mr","hash":"50fa3d1074b3fda37ce11dc6ec92ebf3","queryText":"select key from src1 union select key2 from src2 order by key","edges":[{"sources":[1,2],"targets":[0],"expression":"key","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"u2.key"},{"id":1,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src2.key2"}]} 128 146 @@ -460,42 +119,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src1 PREHOOK: Input: default@src2 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "a739460bd79c8c91ec35e22c97329769", - "queryText": "select key k from src1 union select key2 from src2 order by k", - "edges": [ - { - "sources": [ - 1, - 2 - ], - "targets": [ - 0 - ], - "expression": "key", - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "u2.k" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.src1.key" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.src2.key2" - } - ] -} +{"version":"1.0","engine":"mr","hash":"a739460bd79c8c91ec35e22c97329769","queryText":"select key k from src1 union select key2 from src2 order by k","edges":[{"sources":[1,2],"targets":[0],"expression":"key","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"u2.k"},{"id":1,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src2.key2"}]} 128 146 @@ -516,55 +140,7 @@ PREHOOK: query: select key, count(1) a from dest1 group by key PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "3901b5e3a164064736b3234355046340", - "queryText": "select key, count(1) a from dest1 group by key", - "edges": [ - { - "sources": [ - 2 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 3 - ], - "targets": [ - 1 - ], - "expression": "count(1)", - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "key" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "a" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.dest1.key" - }, - { - "id": 3, - "vertexType": "TABLE", - "vertexId": "default.dest1" - } - ] -} +{"version":"1.0","engine":"mr","hash":"3901b5e3a164064736b3234355046340","queryText":"select key, count(1) a from dest1 group by key","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"count(1)","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"key"},{"id":1,"vertexType":"COLUMN","vertexId":"a"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest1.key"},{"id":3,"vertexType":"TABLE","vertexId":"default.dest1"}]} 20 128 2 146 2 @@ -585,55 +161,7 @@ PREHOOK: query: select key k, count(*) from dest1 group by key PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "0d5a212f10847aeaab31e8c31121e6d4", - "queryText": "select key k, count(*) from dest1 group by key", - "edges": [ - { - "sources": [ - 2 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 3 - ], - "targets": [ - 1 - ], - "expression": "count(*)", - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "k" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "c1" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.dest1.key" - }, - { - "id": 3, - "vertexType": "TABLE", - "vertexId": "default.dest1" - } - ] -} +{"version":"1.0","engine":"mr","hash":"0d5a212f10847aeaab31e8c31121e6d4","queryText":"select key k, count(*) from dest1 group by key","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"count(*)","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"k"},{"id":1,"vertexType":"COLUMN","vertexId":"c1"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest1.key"},{"id":3,"vertexType":"TABLE","vertexId":"default.dest1"}]} 20 128 2 146 2 @@ -654,55 +182,7 @@ PREHOOK: query: select key k, count(value) from dest1 group by key PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "56429eccb04ded722f5bd9d9d8cf7260", - "queryText": "select key k, count(value) from dest1 group by key", - "edges": [ - { - "sources": [ - 2 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 3 - ], - "targets": [ - 1 - ], - "expression": "count(default.dest1.value)", - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "k" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "c1" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.dest1.key" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.dest1.value" - } - ] -} +{"version":"1.0","engine":"mr","hash":"56429eccb04ded722f5bd9d9d8cf7260","queryText":"select key k, count(value) from dest1 group by key","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"count(default.dest1.value)","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"k"},{"id":1,"vertexType":"COLUMN","vertexId":"c1"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest1.value"}]} 20 128 2 146 2 @@ -723,55 +203,7 @@ PREHOOK: query: select value, max(length(key)) from dest1 group by value PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "7e1cfc3dece85b41b6f7c46365580cde", - "queryText": "select value, max(length(key)) from dest1 group by value", - "edges": [ - { - "sources": [ - 2 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 3 - ], - "targets": [ - 1 - ], - "expression": "max(length(dest1.key))", - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "value" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "c1" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.dest1.value" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.dest1.key" - } - ] -} +{"version":"1.0","engine":"mr","hash":"7e1cfc3dece85b41b6f7c46365580cde","queryText":"select value, max(length(key)) from dest1 group by value","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"max(length(dest1.key))","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"value"},{"id":1,"vertexType":"COLUMN","vertexId":"c1"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest1.value"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest1.key"}]} 3 val_146 3 val_150 3 @@ -795,55 +227,7 @@ PREHOOK: query: select value, max(length(key)) from dest1 group by value order b PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "c6578ce1dd72498c4af33f20f164e483", - "queryText": "select value, max(length(key)) from dest1 group by value order by value limit 5", - "edges": [ - { - "sources": [ - 2 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 3 - ], - "targets": [ - 1 - ], - "expression": "max(length(dest1.key))", - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "value" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "c1" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.dest1.value" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.dest1.key" - } - ] -} +{"version":"1.0","engine":"mr","hash":"c6578ce1dd72498c4af33f20f164e483","queryText":"select value, max(length(key)) from dest1 group by value order by value limit 5","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"max(length(dest1.key))","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"value"},{"id":1,"vertexType":"COLUMN","vertexId":"c1"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest1.value"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest1.key"}]} 3 val_146 3 val_150 3 @@ -853,55 +237,7 @@ PREHOOK: query: select key, length(value) from dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "91fbcea5cb34362071555cd93e8d0abe", - "queryText": "select key, length(value) from dest1", - "edges": [ - { - "sources": [ - 2 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 3 - ], - "targets": [ - 1 - ], - "expression": "length(dest1.value)", - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "key" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "c1" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.dest1.key" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.dest1.value" - } - ] -} +{"version":"1.0","engine":"mr","hash":"91fbcea5cb34362071555cd93e8d0abe","queryText":"select key, length(value) from dest1","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"length(dest1.value)","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"key"},{"id":1,"vertexType":"COLUMN","vertexId":"c1"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest1.value"}]} 238 7 0 311 7 @@ -956,36 +292,7 @@ PREHOOK: query: select length(value) + 3 from dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "3d8a347cc9052111cb328938d37b9b03", - "queryText": "select length(value) + 3 from dest1", - "edges": [ - { - "sources": [ - 1 - ], - "targets": [ - 0 - ], - "expression": "(length(dest1.value) + 3)", - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "c0" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.dest1.value" - } - ] -} +{"version":"1.0","engine":"mr","hash":"3d8a347cc9052111cb328938d37b9b03","queryText":"select length(value) + 3 from dest1","edges":[{"sources":[1],"targets":[0],"expression":"(length(dest1.value) + 3)","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"c0"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest1.value"}]} 10 3 10 @@ -1040,29 +347,7 @@ PREHOOK: query: select 5 from dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "bae960bf4376ec00e37258469b17360d", - "queryText": "select 5 from dest1", - "edges": [ - { - "sources": [], - "targets": [ - 0 - ], - "expression": "5", - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "c0" - } - ] -} +{"version":"1.0","engine":"mr","hash":"bae960bf4376ec00e37258469b17360d","queryText":"select 5 from dest1","edges":[{"sources":[],"targets":[0],"expression":"5","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"c0"}]} 5 5 5 @@ -1117,29 +402,7 @@ PREHOOK: query: select 3 * 5 from dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "753abad4d55afd3df34fdc73abfcd44d", - "queryText": "select 3 * 5 from dest1", - "edges": [ - { - "sources": [], - "targets": [ - 0 - ], - "expression": "(3 * 5)", - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "c0" - } - ] -} +{"version":"1.0","engine":"mr","hash":"753abad4d55afd3df34fdc73abfcd44d","queryText":"select 3 * 5 from dest1","edges":[{"sources":[],"targets":[0],"expression":"(3 * 5)","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"c0"}]} 15 15 15 @@ -1198,485 +461,31 @@ PREHOOK: Input: default@src1 PREHOOK: Input: default@src2 PREHOOK: Output: database:default PREHOOK: Output: default@dest2 -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "386791c174a4999fc916e300b5e76bf2", - "queryText": "create table dest2 as select * from src1 JOIN src2 ON src1.key = src2.key2", - "edges": [ - { - "sources": [ - 4 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 5 - ], - "targets": [ - 1 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 6 - ], - "targets": [ - 2 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 7 - ], - "targets": [ - 3 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 4, - 6 - ], - "targets": [ - 0, - 1, - 2, - 3 - ], - "expression": "(src1.key = src2.key2)", - "edgeType": "PREDICATE" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "default.dest2.key" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.dest2.value" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.dest2.key2" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.dest2.value2" - }, - { - "id": 4, - "vertexType": "COLUMN", - "vertexId": "default.src1.key" - }, - { - "id": 5, - "vertexType": "COLUMN", - "vertexId": "default.src1.value" - }, - { - "id": 6, - "vertexType": "COLUMN", - "vertexId": "default.src2.key2" - }, - { - "id": 7, - "vertexType": "COLUMN", - "vertexId": "default.src2.value2" - } - ] -} +{"version":"1.0","engine":"mr","hash":"386791c174a4999fc916e300b5e76bf2","queryText":"create table dest2 as select * from src1 JOIN src2 ON src1.key = src2.key2","edges":[{"sources":[4],"targets":[0],"edgeType":"PROJECTION"},{"sources":[5],"targets":[1],"edgeType":"PROJECTION"},{"sources":[6],"targets":[2],"edgeType":"PROJECTION"},{"sources":[7],"targets":[3],"edgeType":"PROJECTION"},{"sources":[4,6],"targets":[0,1,2,3],"expression":"(src1.key = src2.key2)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest2.key"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest2.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest2.key2"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest2.value2"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":5,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":6,"vertexType":"COLUMN","vertexId":"default.src2.key2"},{"id":7,"vertexType":"COLUMN","vertexId":"default.src2.val ue2"}]} PREHOOK: query: insert overwrite table dest2 select * from src1 JOIN src2 ON src1.key = src2.key2 PREHOOK: type: QUERY PREHOOK: Input: default@src1 PREHOOK: Input: default@src2 PREHOOK: Output: default@dest2 -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "e494b771d94800dc3430bf5d0810cd9f", - "queryText": "insert overwrite table dest2 select * from src1 JOIN src2 ON src1.key = src2.key2", - "edges": [ - { - "sources": [ - 4 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 5 - ], - "targets": [ - 1 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 6 - ], - "targets": [ - 2 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 7 - ], - "targets": [ - 3 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 4, - 6 - ], - "targets": [ - 0, - 1, - 2, - 3 - ], - "expression": "(src1.key = src2.key2)", - "edgeType": "PREDICATE" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "default.dest2.key" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.dest2.value" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.dest2.key2" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.dest2.value2" - }, - { - "id": 4, - "vertexType": "COLUMN", - "vertexId": "default.src1.key" - }, - { - "id": 5, - "vertexType": "COLUMN", - "vertexId": "default.src1.value" - }, - { - "id": 6, - "vertexType": "COLUMN", - "vertexId": "default.src2.key2" - }, - { - "id": 7, - "vertexType": "COLUMN", - "vertexId": "default.src2.value2" - } - ] -} +{"version":"1.0","engine":"mr","hash":"e494b771d94800dc3430bf5d0810cd9f","queryText":"insert overwrite table dest2 select * from src1 JOIN src2 ON src1.key = src2.key2","edges":[{"sources":[4],"targets":[0],"edgeType":"PROJECTION"},{"sources":[5],"targets":[1],"edgeType":"PROJECTION"},{"sources":[6],"targets":[2],"edgeType":"PROJECTION"},{"sources":[7],"targets":[3],"edgeType":"PROJECTION"},{"sources":[4,6],"targets":[0,1,2,3],"expression":"(src1.key = src2.key2)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest2.key"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest2.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest2.key2"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest2.value2"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":5,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":6,"vertexType":"COLUMN","vertexId":"default.src2.key2"},{"id":7,"vertexType":"COLUMN","vertexId":"default.s rc2.value2"}]} PREHOOK: query: insert into table dest2 select * from src1 JOIN src2 ON src1.key = src2.key2 PREHOOK: type: QUERY PREHOOK: Input: default@src1 PREHOOK: Input: default@src2 PREHOOK: Output: default@dest2 -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "efeaddd0d36105b1013b414627850dc2", - "queryText": "insert into table dest2 select * from src1 JOIN src2 ON src1.key = src2.key2", - "edges": [ - { - "sources": [ - 4 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 5 - ], - "targets": [ - 1 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 6 - ], - "targets": [ - 2 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 7 - ], - "targets": [ - 3 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 4, - 6 - ], - "targets": [ - 0, - 1, - 2, - 3 - ], - "expression": "(src1.key = src2.key2)", - "edgeType": "PREDICATE" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "default.dest2.key" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.dest2.value" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.dest2.key2" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.dest2.value2" - }, - { - "id": 4, - "vertexType": "COLUMN", - "vertexId": "default.src1.key" - }, - { - "id": 5, - "vertexType": "COLUMN", - "vertexId": "default.src1.value" - }, - { - "id": 6, - "vertexType": "COLUMN", - "vertexId": "default.src2.key2" - }, - { - "id": 7, - "vertexType": "COLUMN", - "vertexId": "default.src2.value2" - } - ] -} +{"version":"1.0","engine":"mr","hash":"efeaddd0d36105b1013b414627850dc2","queryText":"insert into table dest2 select * from src1 JOIN src2 ON src1.key = src2.key2","edges":[{"sources":[4],"targets":[0],"edgeType":"PROJECTION"},{"sources":[5],"targets":[1],"edgeType":"PROJECTION"},{"sources":[6],"targets":[2],"edgeType":"PROJECTION"},{"sources":[7],"targets":[3],"edgeType":"PROJECTION"},{"sources":[4,6],"targets":[0,1,2,3],"expression":"(src1.key = src2.key2)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest2.key"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest2.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest2.key2"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest2.value2"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":5,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":6,"vertexType":"COLUMN","vertexId":"default.src2.key2"},{"id":7,"vertexType":"COLUMN","vertexId":"default.src2.v alue2"}]} PREHOOK: query: insert into table dest2 select * from src1 JOIN src2 ON length(src1.value) = length(src2.value2) + 1 PREHOOK: type: QUERY PREHOOK: Input: default@src1 PREHOOK: Input: default@src2 PREHOOK: Output: default@dest2 -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "e9450a56b3d103642e06bef0e4f0d482", - "queryText": "insert into table dest2\n select * from src1 JOIN src2 ON length(src1.value) = length(src2.value2) + 1", - "edges": [ - { - "sources": [ - 4 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 5 - ], - "targets": [ - 1 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 6 - ], - "targets": [ - 2 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 7 - ], - "targets": [ - 3 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 5, - 7 - ], - "targets": [ - 0, - 1, - 2, - 3 - ], - "expression": "(length(src1.value) = (length(src2.value2) + 1))", - "edgeType": "PREDICATE" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "default.dest2.key" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.dest2.value" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.dest2.key2" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.dest2.value2" - }, - { - "id": 4, - "vertexType": "COLUMN", - "vertexId": "default.src1.key" - }, - { - "id": 5, - "vertexType": "COLUMN", - "vertexId": "default.src1.value" - }, - { - "id": 6, - "vertexType": "COLUMN", - "vertexId": "default.src2.key2" - }, - { - "id": 7, - "vertexType": "COLUMN", - "vertexId": "default.src2.value2" - } - ] -} +{"version":"1.0","engine":"mr","hash":"e9450a56b3d103642e06bef0e4f0d482","queryText":"insert into table dest2\n select * from src1 JOIN src2 ON length(src1.value) = length(src2.value2) + 1","edges":[{"sources":[4],"targets":[0],"edgeType":"PROJECTION"},{"sources":[5],"targets":[1],"edgeType":"PROJECTION"},{"sources":[6],"targets":[2],"edgeType":"PROJECTION"},{"sources":[7],"targets":[3],"edgeType":"PROJECTION"},{"sources":[5,7],"targets":[0,1,2,3],"expression":"(length(src1.value) = (length(src2.value2) + 1))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest2.key"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest2.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest2.key2"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest2.value2"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":5,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":6,"vertexType":"COLUMN","vertexId":"default.src2.key2"},{"i d":7,"vertexType":"COLUMN","vertexId":"default.src2.value2"}]} PREHOOK: query: select * from src1 where length(key) > 2 PREHOOK: type: QUERY PREHOOK: Input: default@src1 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "4028c94d222d5dd221f651d414386972", - "queryText": "select * from src1 where length(key) > 2", - "edges": [ - { - "sources": [ - 2 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 3 - ], - "targets": [ - 1 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 2 - ], - "targets": [ - 0, - 1 - ], - "expression": "(length(src1.key) > 2)", - "edgeType": "PREDICATE" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "src1.key" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "src1.value" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.src1.key" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.src1.value" - } - ] -} +{"version":"1.0","engine":"mr","hash":"4028c94d222d5dd221f651d414386972","queryText":"select * from src1 where length(key) > 2","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(length(src1.key) > 2)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"src1.key"},{"id":1,"vertexType":"COLUMN","vertexId":"src1.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"}]} 238 val_238 311 val_311 255 val_255 @@ -1694,66 +503,7 @@ PREHOOK: query: select * from src1 where length(key) > 2 and value > 'a' PREHOOK: type: QUERY PREHOOK: Input: default@src1 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "5727531f7743cfcd60d634d8c835515f", - "queryText": "select * from src1 where length(key) > 2 and value > 'a'", - "edges": [ - { - "sources": [ - 2 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 3 - ], - "targets": [ - 1 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 2, - 3 - ], - "targets": [ - 0, - 1 - ], - "expression": "((length(src1.key) > 2) and (src1.value > 'a'))", - "edgeType": "PREDICATE" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "src1.key" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "src1.value" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.src1.key" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.src1.value" - } - ] -} +{"version":"1.0","engine":"mr","hash":"5727531f7743cfcd60d634d8c835515f","queryText":"select * from src1 where length(key) > 2 and value > 'a'","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2,3],"targets":[0,1],"expression":"((length(src1.key) > 2) and (src1.value > 'a'))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"src1.key"},{"id":1,"vertexType":"COLUMN","vertexId":"src1.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"}]} 238 val_238 311 val_311 255 val_255 @@ -1773,238 +523,14 @@ PREHOOK: Input: default@src1 PREHOOK: Input: default@src2 PREHOOK: Output: database:default PREHOOK: Output: default@dest3 -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "a2c4e9a3ec678039814f5d84b1e38ce4", - "queryText": "create table dest3 as\n select * from src1 JOIN src2 ON src1.key = src2.key2 WHERE length(key) > 1", - "edges": [ - { - "sources": [ - 4 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 5 - ], - "targets": [ - 1 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 6 - ], - "targets": [ - 2 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 7 - ], - "targets": [ - 3 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 4 - ], - "targets": [ - 0, - 1, - 2, - 3 - ], - "expression": "(length(src1.key) > 1)", - "edgeType": "PREDICATE" - }, - { - "sources": [ - 4, - 6 - ], - "targets": [ - 0, - 1, - 2, - 3 - ], - "expression": "(src1.key = src2.key2)", - "edgeType": "PREDICATE" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "default.dest3.key" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.dest3.value" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.dest3.key2" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.dest3.value2" - }, - { - "id": 4, - "vertexType": "COLUMN", - "vertexId": "default.src1.key" - }, - { - "id": 5, - "vertexType": "COLUMN", - "vertexId": "default.src1.value" - }, - { - "id": 6, - "vertexType": "COLUMN", - "vertexId": "default.src2.key2" - }, - { - "id": 7, - "vertexType": "COLUMN", - "vertexId": "default.src2.value2" - } - ] -} +{"version":"1.0","engine":"mr","hash":"a2c4e9a3ec678039814f5d84b1e38ce4","queryText":"create table dest3 as\n select * from src1 JOIN src2 ON src1.key = src2.key2 WHERE length(key) > 1","edges":[{"sources":[4],"targets":[0],"edgeType":"PROJECTION"},{"sources":[5],"targets":[1],"edgeType":"PROJECTION"},{"sources":[6],"targets":[2],"edgeType":"PROJECTION"},{"sources":[7],"targets":[3],"edgeType":"PROJECTION"},{"sources":[4],"targets":[0,1,2,3],"expression":"(length(src1.key) > 1)","edgeType":"PREDICATE"},{"sources":[4,6],"targets":[0,1,2,3],"expression":"(src1.key = src2.key2)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest3.key"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest3.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest3.key2"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest3.value2"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":5,"vertexType":"COLUMN","vertexId":"default.src1.value" },{"id":6,"vertexType":"COLUMN","vertexId":"default.src2.key2"},{"id":7,"vertexType":"COLUMN","vertexId":"default.src2.value2"}]} PREHOOK: query: insert overwrite table dest2 select * from src1 JOIN src2 ON src1.key = src2.key2 WHERE length(key) > 3 PREHOOK: type: QUERY PREHOOK: Input: default@src1 PREHOOK: Input: default@src2 PREHOOK: Output: default@dest2 -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "76d84512204ddc576ad4d93f252e4358", - "queryText": "insert overwrite table dest2\n select * from src1 JOIN src2 ON src1.key = src2.key2 WHERE length(key) > 3", - "edges": [ - { - "sources": [ - 4 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 5 - ], - "targets": [ - 1 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 6 - ], - "targets": [ - 2 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 7 - ], - "targets": [ - 3 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 4 - ], - "targets": [ - 0, - 1, - 2, - 3 - ], - "expression": "(length(src1.key) > 3)", - "edgeType": "PREDICATE" - }, - { - "sources": [ - 4, - 6 - ], - "targets": [ - 0, - 1, - 2, - 3 - ], - "expression": "(src1.key = src2.key2)", - "edgeType": "PREDICATE" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "default.dest2.key" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.dest2.value" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.dest2.key2" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.dest2.value2" - }, - { - "id": 4, - "vertexType": "COLUMN", - "vertexId": "default.src1.key" - }, - { - "id": 5, - "vertexType": "COLUMN", - "vertexId": "default.src1.value" - }, - { - "id": 6, - "vertexType": "COLUMN", - "vertexId": "default.src2.key2" - }, - { - "id": 7, - "vertexType": "COLUMN", - "vertexId": "default.src2.value2" - } - ] -} +{"version":"1.0","engine":"mr","hash":"76d84512204ddc576ad4d93f252e4358","queryText":"insert overwrite table dest2\n select * from src1 JOIN src2 ON src1.key = src2.key2 WHERE length(key) > 3","edges":[{"sources":[4],"targets":[0],"edgeType":"PROJECTION"},{"sources":[5],"targets":[1],"edgeType":"PROJECTION"},{"sources":[6],"targets":[2],"edgeType":"PROJECTION"},{"sources":[7],"targets":[3],"edgeType":"PROJECTION"},{"sources":[4],"targets":[0,1,2,3],"expression":"(length(src1.key) > 3)","edgeType":"PREDICATE"},{"sources":[4,6],"targets":[0,1,2,3],"expression":"(src1.key = src2.key2)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest2.key"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest2.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest2.key2"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest2.value2"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":5,"vertexType":"COLUMN","vertexId":"default.src1 .value"},{"id":6,"vertexType":"COLUMN","vertexId":"default.src2.key2"},{"id":7,"vertexType":"COLUMN","vertexId":"default.src2.value2"}]} PREHOOK: query: drop table if exists dest_l1 PREHOOK: type: DROPTABLE PREHOOK: query: CREATE TABLE dest_l1(key INT, value STRING) STORED AS TEXTFILE @@ -2026,73 +552,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@src1 PREHOOK: Output: default@dest_l1 -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "60b589744e2527dd235a6c8168d6a653", - "queryText": "INSERT OVERWRITE TABLE dest_l1\nSELECT j.*\nFROM (SELECT t1.key, p1.value\n FROM src1 t1\n LEFT OUTER JOIN src p1\n ON (t1.key = p1.key)\n UNION ALL\n SELECT t2.key, p2.value\n FROM src1 t2\n LEFT OUTER JOIN src p2\n ON (t2.key = p2.key)) j", - "edges": [ - { - "sources": [ - 2 - ], - "targets": [ - 0 - ], - "expression": "UDFToInteger(j.key)", - "edgeType": "PROJECTION" - }, - { - "sources": [ - 3 - ], - "targets": [ - 1 - ], - "expression": "j.value", - "edgeType": "PROJECTION" - }, - { - "sources": [ - 4, - 2 - ], - "targets": [ - 0, - 1 - ], - "expression": "(p1.key = t1.key)", - "edgeType": "PREDICATE" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "default.dest_l1.key" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.dest_l1.value" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.src1.key" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.src.value" - }, - { - "id": 4, - "vertexType": "COLUMN", - "vertexId": "default.src.key" - } - ] -} +{"version":"1.0","engine":"mr","hash":"60b589744e2527dd235a6c8168d6a653","queryText":"INSERT OVERWRITE TABLE dest_l1\nSELECT j.*\nFROM (SELECT t1.key, p1.value\n FROM src1 t1\n LEFT OUTER JOIN src p1\n ON (t1.key = p1.key)\n UNION ALL\n SELECT t2.key, p2.value\n FROM src1 t2\n LEFT OUTER JOIN src p2\n ON (t2.key = p2.key)) j","edges":[{"sources":[2],"targets":[0],"expression":"UDFToInteger(j.key)","edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"j.value","edgeType":"PROJECTION"},{"sources":[4,2],"targets":[0,1],"expression":"(p1.key = t1.key)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_l1.key"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_l1.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src.key"}]} PREHOOK: query: drop table if exists emp PREHOOK: type: DROPTABLE PREHOOK: query: drop table if exists dept @@ -2133,185 +593,7 @@ PREHOOK: Input: default@dept PREHOOK: Input: default@emp PREHOOK: Input: default@project PREHOOK: Output: default@tgt -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "f59797e0422d2e51515063374dfac361", - "queryText": "INSERT INTO TABLE tgt\nSELECT emd.dept_name, emd.name, emd.emp_id, emd.mgr_id, p.project_id, p.project_name\nFROM (\n SELECT d.dept_name, em.name, em.emp_id, em.mgr_id, em.dept_id\n FROM (\n SELECT e.name, e.dept_id, e.emp_id emp_id, m.emp_id mgr_id\n FROM emp e JOIN emp m ON e.emp_id = m.emp_id\n ) em\n JOIN dept d ON d.dept_id = em.dept_id\n ) emd JOIN project p ON emd.dept_id = p.project_id", - "edges": [ - { - "sources": [ - 6 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 7 - ], - "targets": [ - 1 - ], - "expression": "emd.name", - "edgeType": "PROJECTION" - }, - { - "sources": [ - 8 - ], - "targets": [ - 2 - ], - "expression": "emd.emp_id", - "edgeType": "PROJECTION" - }, - { - "sources": [ - 8 - ], - "targets": [ - 3 - ], - "expression": "emd.mgr_id", - "edgeType": "PROJECTION" - }, - { - "sources": [ - 9 - ], - "targets": [ - 4 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 10 - ], - "targets": [ - 5 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 8 - ], - "targets": [ - 0, - 1, - 2, - 3, - 4, - 5 - ], - "expression": "(e.emp_id = m.emp_id)", - "edgeType": "PREDICATE" - }, - { - "sources": [ - 11, - 12 - ], - "targets": [ - 0, - 1, - 2, - 3, - 4, - 5 - ], - "expression": "(em._col1 = d.dept_id)", - "edgeType": "PREDICATE" - }, - { - "sources": [ - 11, - 9 - ], - "targets": [ - 0, - 1, - 2, - 3, - 4, - 5 - ], - "expression": "(emd._col4 = p.project_id)", - "edgeType": "PREDICATE" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "default.tgt.dept_name" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.tgt.name" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.tgt.emp_id" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.tgt.mgr_id" - }, - { - "id": 4, - "vertexType": "COLUMN", - "vertexId": "default.tgt.proj_id" - }, - { - "id": 5, - "vertexType": "COLUMN", - "vertexId": "default.tgt.proj_name" - }, - { - "id": 6, - "vertexType": "COLUMN", - "vertexId": "default.dept.dept_name" - }, - { - "id": 7, - "vertexType": "COLUMN", - "vertexId": "default.emp.name" - }, - { - "id": 8, - "vertexType": "COLUMN", - "vertexId": "default.emp.emp_id" - }, - { - "id": 9, - "vertexType": "COLUMN", - "vertexId": "default.project.project_id" - }, - { - "id": 10, - "vertexType": "COLUMN", - "vertexId": "default.project.project_name" - }, - { - "id": 11, - "vertexType": "COLUMN", - "vertexId": "default.emp.dept_id" - }, - { - "id": 12, - "vertexType": "COLUMN", - "vertexId": "default.dept.dept_id" - } - ] -} +{"version":"1.0","engine":"mr","hash":"f59797e0422d2e51515063374dfac361","queryText":"INSERT INTO TABLE tgt\nSELECT emd.dept_name, emd.name, emd.emp_id, emd.mgr_id, p.project_id, p.project_name\nFROM (\n SELECT d.dept_name, em.name, em.emp_id, em.mgr_id, em.dept_id\n FROM (\n SELECT e.name, e.dept_id, e.emp_id emp_id, m.emp_id mgr_id\n FROM emp e JOIN emp m ON e.emp_id = m.emp_id\n ) em\n JOIN dept d ON d.dept_id = em.dept_id\n ) emd JOIN project p ON emd.dept_id = p.project_id","edges":[{"sources":[6],"targets":[0],"edgeType":"PROJECTION"},{"sources":[7],"targets":[1],"expression":"emd.name","edgeType":"PROJECTION"},{"sources":[8],"targets":[2],"expression":"emd.emp_id","edgeType":"PROJECTION"},{"sources":[8],"targets":[3],"expression":"emd.mgr_id","edgeType":"PROJECTION"},{"sources":[9],"targets":[4],"edgeType":"PROJECTION"},{"sources":[10],"targets":[5],"edgeType":"PROJECTION"},{"sources":[8],"targets":[0,1,2,3,4,5],"expression":"(e.emp_id = m.emp_id)","edgeType":"PR EDICATE"},{"sources":[11,12],"targets":[0,1,2,3,4,5],"expression":"(em._col1 = d.dept_id)","edgeType":"PREDICATE"},{"sources":[11,9],"targets":[0,1,2,3,4,5],"expression":"(emd._col4 = p.project_id)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.tgt.dept_name"},{"id":1,"vertexType":"COLUMN","vertexId":"default.tgt.name"},{"id":2,"vertexType":"COLUMN","vertexId":"default.tgt.emp_id"},{"id":3,"vertexType":"COLUMN","vertexId":"default.tgt.mgr_id"},{"id":4,"vertexType":"COLUMN","vertexId":"default.tgt.proj_id"},{"id":5,"vertexType":"COLUMN","vertexId":"default.tgt.proj_name"},{"id":6,"vertexType":"COLUMN","vertexId":"default.dept.dept_name"},{"id":7,"vertexType":"COLUMN","vertexId":"default.emp.name"},{"id":8,"vertexType":"COLUMN","vertexId":"default.emp.emp_id"},{"id":9,"vertexType":"COLUMN","vertexId":"default.project.project_id"},{"id":10,"vertexType":"COLUMN","vertexId":"default.project.project_name"},{"id":11,"vertexType":"COLUMN","vertexId": "default.emp.dept_id"},{"id":12,"vertexType":"COLUMN","vertexId":"default.dept.dept_id"}]} PREHOOK: query: drop table if exists dest_l2 PREHOOK: type: DROPTABLE PREHOOK: query: create table dest_l2 (id int, c1 tinyint, c2 int, c3 bigint) stored as textfile @@ -2322,68 +604,7 @@ PREHOOK: query: insert into dest_l2 values(0, 1, 100, 10000) PREHOOK: type: QUERY PREHOOK: Input: default@values__tmp__table__1 PREHOOK: Output: default@dest_l2 -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "e001334e3f8384806b0f25a7c303045f", - "queryText": "insert into dest_l2 values(0, 1, 100, 10000)", - "edges": [ - { - "sources": [], - "targets": [ - 0 - ], - "expression": "UDFToInteger(values__tmp__table__1.tmp_values_col1)", - "edgeType": "PROJECTION" - }, - { - "sources": [], - "targets": [ - 1 - ], - "expression": "UDFToByte(values__tmp__table__1.tmp_values_col2)", - "edgeType": "PROJECTION" - }, - { - "sources": [], - "targets": [ - 2 - ], - "expression": "UDFToInteger(values__tmp__table__1.tmp_values_col3)", - "edgeType": "PROJECTION" - }, - { - "sources": [], - "targets": [ - 3 - ], - "expression": "UDFToLong(values__tmp__table__1.tmp_values_col4)", - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "default.dest_l2.id" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.dest_l2.c1" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.dest_l2.c2" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.dest_l2.c3" - } - ] -} +{"version":"1.0","engine":"mr","hash":"e001334e3f8384806b0f25a7c303045f","queryText":"insert into dest_l2 values(0, 1, 100, 10000)","edges":[{"sources":[],"targets":[0],"expression":"UDFToInteger(values__tmp__table__1.tmp_values_col1)","edgeType":"PROJECTION"},{"sources":[],"targets":[1],"expression":"UDFToByte(values__tmp__table__1.tmp_values_col2)","edgeType":"PROJECTION"},{"sources":[],"targets":[2],"expression":"UDFToInteger(values__tmp__table__1.tmp_values_col3)","edgeType":"PROJECTION"},{"sources":[],"targets":[3],"expression":"UDFToLong(values__tmp__table__1.tmp_values_col4)","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_l2.id"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_l2.c1"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest_l2.c2"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest_l2.c3"}]} PREHOOK: query: select * from ( select c1 + c2 x from dest_l2 union all @@ -2391,48 +612,7 @@ PREHOOK: query: select * from ( PREHOOK: type: QUERY PREHOOK: Input: default@dest_l2 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "a2c96a96be9d315ede966be5b45ef20e", - "queryText": "select * from (\n select c1 + c2 x from dest_l2\n union all\n select sum(c3) y from (select c3 from dest_l2) v1) v2 order by x", - "edges": [ - { - "sources": [ - 1, - 2, - 3 - ], - "targets": [ - 0 - ], - "expression": "v2.x", - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "v2.x" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.dest_l2.c1" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.dest_l2.c2" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.dest_l2.c3" - } - ] -} +{"version":"1.0","engine":"mr","hash":"a2c96a96be9d315ede966be5b45ef20e","queryText":"select * from (\n select c1 + c2 x from dest_l2\n union all\n select sum(c3) y from (select c3 from dest_l2) v1) v2 order by x","edges":[{"sources":[1,2,3],"targets":[0],"expression":"v2.x","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"v2.x"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_l2.c1"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest_l2.c2"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest_l2.c3"}]} 101 10000 PREHOOK: query: drop table if exists dest_l3 @@ -2445,60 +625,7 @@ PREHOOK: query: insert into dest_l3 values(0, "s1", "s2", 15) PREHOOK: type: QUERY PREHOOK: Input: default@values__tmp__table__2 PREHOOK: Output: default@dest_l3 -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "09df51ba6ba2d07f2304523ee505f094", - "queryText": "insert into dest_l3 values(0, \"s1\", \"s2\", 15)", - "edges": [ - { - "sources": [], - "targets": [ - 0 - ], - "expression": "UDFToInteger(values__tmp__table__2.tmp_values_col1)", - "edgeType": "PROJECTION" - }, - { - "sources": [], - "targets": [ - 1, - 2 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [], - "targets": [ - 3 - ], - "expression": "UDFToInteger(values__tmp__table__2.tmp_values_col4)", - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "default.dest_l3.id" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.dest_l3.c1" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.dest_l3.c2" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.dest_l3.c3" - } - ] -} +{"version":"1.0","engine":"mr","hash":"09df51ba6ba2d07f2304523ee505f094","queryText":"insert into dest_l3 values(0, \"s1\", \"s2\", 15)","edges":[{"sources":[],"targets":[0],"expression":"UDFToInteger(values__tmp__table__2.tmp_values_col1)","edgeType":"PROJECTION"},{"sources":[],"targets":[1,2],"edgeType":"PROJECTION"},{"sources":[],"targets":[3],"expression":"UDFToInteger(values__tmp__table__2.tmp_values_col4)","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_l3.id"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_l3.c1"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest_l3.c2"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest_l3.c3"}]} PREHOOK: query: select sum(a.c1) over (partition by a.c1 order by a.id) from dest_l2 a where a.c2 != 10 @@ -2507,68 +634,7 @@ having count(a.c2) > 0 PREHOOK: type: QUERY PREHOOK: Input: default@dest_l2 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "0ae7aa4a0cbd1283210fa79e8a19104a", - "queryText": "select sum(a.c1) over (partition by a.c1 order by a.id)\nfrom dest_l2 a\nwhere a.c2 != 10\ngroup by a.c1, a.c2, a.id\nhaving count(a.c2) > 0", - "edges": [ - { - "sources": [ - 1, - 2, - 3 - ], - "targets": [ - 0 - ], - "expression": "(tok_function sum (. (tok_table_or_col $hdt$_0) $f0) (tok_windowspec (tok_partitioningspec (tok_distributeby (. (tok_table_or_col $hdt$_0) $f0)) (tok_orderby (tok_tabsortcolnameasc (. (tok_table_or_col $hdt$_0) $f2)))) (tok_windowvalues (preceding 2147483647) current)))", - "edgeType": "PROJECTION" - }, - { - "sources": [ - 2 - ], - "targets": [ - 0 - ], - "expression": "(a.c2 <> 10)", - "edgeType": "PREDICATE" - }, - { - "sources": [ - 2 - ], - "targets": [ - 0 - ], - "expression": "(count(default.dest_l2.c2) > 0)", - "edgeType": "PREDICATE" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "c0" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.dest_l2.c1" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.dest_l2.c2" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.dest_l2.id" - } - ] -} +{"version":"1.0","engine":"mr","hash":"0ae7aa4a0cbd1283210fa79e8a19104a","queryText":"select sum(a.c1) over (partition by a.c1 order by a.id)\nfrom dest_l2 a\nwhere a.c2 != 10\ngroup by a.c1, a.c2, a.id\nhaving count(a.c2) > 0","edges":[{"sources":[1,2,3],"targets":[0],"expression":"(tok_function sum (. (tok_table_or_col $hdt$_0) $f0) (tok_windowspec (tok_partitioningspec (tok_distributeby (. (tok_table_or_col $hdt$_0) $f0)) (tok_orderby (tok_tabsortcolnameasc (. (tok_table_or_col $hdt$_0) $f2)))) (tok_windowvalues (preceding 2147483647) current)))","edgeType":"PROJECTION"},{"sources":[2],"targets":[0],"expression":"(a.c2 <> 10)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0],"expression":"(count(default.dest_l2.c2) > 0)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"c0"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_l2.c1"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest_l2.c2"},{"id":3,"vertexType":"COLUMN","vertexId":"default .dest_l2.id"}]} 1 PREHOOK: query: select sum(a.c1), count(b.c1), b.c2, b.c3 from dest_l2 a join dest_l3 b on (a.id = b.id) @@ -2580,150 +646,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@dest_l2 PREHOOK: Input: default@dest_l3 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "01879c619517509d9f5b6ead998bb4bb", - "queryText": "select sum(a.c1), count(b.c1), b.c2, b.c3\nfrom dest_l2 a join dest_l3 b on (a.id = b.id)\nwhere a.c2 != 10 and b.c3 > 0\ngroup by a.c1, a.c2, a.id, b.c1, b.c2, b.c3\nhaving count(a.c2) > 0\norder by b.c3 limit 5", - "edges": [ - { - "sources": [ - 4 - ], - "targets": [ - 0 - ], - "expression": "sum(default.dest_l2.c1)", - "edgeType": "PROJECTION" - }, - { - "sources": [ - 5 - ], - "targets": [ - 1 - ], - "expression": "count(default.dest_l3.c1)", - "edgeType": "PROJECTION" - }, - { - "sources": [ - 6 - ], - "targets": [ - 2 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 7 - ], - "targets": [ - 3 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 8, - 7 - ], - "targets": [ - 0, - 1, - 2, - 3 - ], - "expression": "((a.c2 <> 10) and (b.c3 > 0))", - "edgeType": "PREDICATE" - }, - { - "sources": [ - 8 - ], - "targets": [ - 0, - 1, - 2, - 3 - ], - "expression": "(count(default.dest_l2.c2) > 0)", - "edgeType": "PREDICATE" - }, - { - "sources": [ - 9, - 10 - ], - "targets": [ - 0, - 1, - 2, - 3 - ], - "expression": "(a.id = b.id)", - "edgeType": "PREDICATE" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "_c0" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "_c1" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "b.c2" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "b.c3" - }, - { - "id": 4, - "vertexType": "COLUMN", - "vertexId": "default.dest_l2.c1" - }, - { - "id": 5, - "vertexType": "COLUMN", - "vertexId": "default.dest_l3.c1" - }, - { - "id": 6, - "vertexType": "COLUMN", - "vertexId": "default.dest_l3.c2" - }, - { - "id": 7, - "vertexType": "COLUMN", - "vertexId": "default.dest_l3.c3" - }, - { - "id": 8, - "vertexType": "COLUMN", - "vertexId": "default.dest_l2.c2" - }, - { - "id": 9, - "vertexType": "COLUMN", - "vertexId": "default.dest_l2.id" - }, - { - "id": 10, - "vertexType": "COLUMN", - "vertexId": "default.dest_l3.id" - } - ] -} +{"version":"1.0","engine":"mr","hash":"01879c619517509d9f5b6ead998bb4bb","queryText":"select sum(a.c1), count(b.c1), b.c2, b.c3\nfrom dest_l2 a join dest_l3 b on (a.id = b.id)\nwhere a.c2 != 10 and b.c3 > 0\ngroup by a.c1, a.c2, a.id, b.c1, b.c2, b.c3\nhaving count(a.c2) > 0\norder by b.c3 limit 5","edges":[{"sources":[4],"targets":[0],"expression":"sum(default.dest_l2.c1)","edgeType":"PROJECTION"},{"sources":[5],"targets":[1],"expression":"count(default.dest_l3.c1)","edgeType":"PROJECTION"},{"sources":[6],"targets":[2],"edgeType":"PROJECTION"},{"sources":[7],"targets":[3],"edgeType":"PROJECTION"},{"sources":[8,7],"targets":[0,1,2,3],"expression":"((a.c2 <> 10) and (b.c3 > 0))","edgeType":"PREDICATE"},{"sources":[8],"targets":[0,1,2,3],"expression":"(count(default.dest_l2.c2) > 0)","edgeType":"PREDICATE"},{"sources":[9,10],"targets":[0,1,2,3],"expression":"(a.id = b.id)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"_c0"},{"id":1,"vertexType":"COLUMN ","vertexId":"_c1"},{"id":2,"vertexType":"COLUMN","vertexId":"b.c2"},{"id":3,"vertexType":"COLUMN","vertexId":"b.c3"},{"id":4,"vertexType":"COLUMN","vertexId":"default.dest_l2.c1"},{"id":5,"vertexType":"COLUMN","vertexId":"default.dest_l3.c1"},{"id":6,"vertexType":"COLUMN","vertexId":"default.dest_l3.c2"},{"id":7,"vertexType":"COLUMN","vertexId":"default.dest_l3.c3"},{"id":8,"vertexType":"COLUMN","vertexId":"default.dest_l2.c2"},{"id":9,"vertexType":"COLUMN","vertexId":"default.dest_l2.id"},{"id":10,"vertexType":"COLUMN","vertexId":"default.dest_l3.id"}]} 1 1 s2 15 PREHOOK: query: drop table if exists t PREHOOK: type: DROPTABLE @@ -2736,93 +659,7 @@ PREHOOK: Input: default@dest_l2 PREHOOK: Input: default@dest_l3 PREHOOK: Output: database:default PREHOOK: Output: default@t -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "0d2f15b494111ffe236d5be42a76fa28", - "queryText": "create table t as\nselect distinct a.c2, a.c3 from dest_l2 a\ninner join dest_l3 b on (a.id = b.id)\nwhere a.id > 0 and b.c3 = 15", - "edges": [ - { - "sources": [ - 2 - ], - "targets": [ - 0 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 3 - ], - "targets": [ - 1 - ], - "edgeType": "PROJECTION" - }, - { - "sources": [ - 4, - 5 - ], - "targets": [ - 0, - 1 - ], - "expression": "((a.id > 0) and (b.c3 = 15))", - "edgeType": "PREDICATE" - }, - { - "sources": [ - 4, - 6 - ], - "targets": [ - 0, - 1 - ], - "expression": "(a.id = b.id)", - "edgeType": "PREDICATE" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "default.t.c2" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "default.t.c3" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "default.dest_l2.c2" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.dest_l2.c3" - }, - { - "id": 4, - "vertexType": "COLUMN", - "vertexId": "default.dest_l2.id" - }, - { - "id": 5, - "vertexType": "COLUMN", - "vertexId": "default.dest_l3.c3" - }, - { - "id": 6, - "vertexType": "COLUMN", - "vertexId": "default.dest_l3.id" - } - ] -} +{"version":"1.0","engine":"mr","hash":"0d2f15b494111ffe236d5be42a76fa28","queryText":"create table t as\nselect distinct a.c2, a.c3 from dest_l2 a\ninner join dest_l3 b on (a.id = b.id)\nwhere a.id > 0 and b.c3 = 15","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[4,5],"targets":[0,1],"expression":"((a.id > 0) and (b.c3 = 15))","edgeType":"PREDICATE"},{"sources":[4,6],"targets":[0,1],"expression":"(a.id = b.id)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.t.c2"},{"id":1,"vertexType":"COLUMN","vertexId":"default.t.c3"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest_l2.c2"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest_l2.c3"},{"id":4,"vertexType":"COLUMN","vertexId":"default.dest_l2.id"},{"id":5,"vertexType":"COLUMN","vertexId":"default.dest_l3.c3"},{"id":6,"vertexType":"COLUMN","vertexId":"default.dest_l3.id"}]} PREHOOK: query: SELECT substr(src1.key,1,1), count(DISTINCT substr(src1.value,5)), concat(substr(src1.key,1,1),sum(substr(src1.value,5))) from src1 @@ -2830,72 +667,7 @@ GROUP BY substr(src1.key,1,1) PREHOOK: type: QUERY PREHOOK: Input: default@src1 #### A masked pattern was here #### -POSTHOOK: LINEAGE: { - "version": "1.0", - "engine": "mr", - "hash": "5b1022708124ee2b80f9e2e8a0dcb15c", - "queryText": "SELECT substr(src1.key,1,1), count(DISTINCT substr(src1.value,5)),\nconcat(substr(src1.key,1,1),sum(substr(src1.value,5)))\nfrom src1\nGROUP BY substr(src1.key,1,1)", - "edges": [ - { - "sources": [ - 3 - ], - "targets": [ - 0 - ], - "expression": "substr(src1.key, 1, 1)", - "edgeType": "PROJECTION" - }, - { - "sources": [ - 4 - ], - "targets": [ - 1 - ], - "expression": "count(DISTINCT substr(src1.value, 5))", - "edgeType": "PROJECTION" - }, - { - "sources": [ - 3, - 4 - ], - "targets": [ - 2 - ], - "expression": "concat(substr(src1.key, 1, 1), sum(substr(src1.value, 5)))", - "edgeType": "PROJECTION" - } - ], - "vertices": [ - { - "id": 0, - "vertexType": "COLUMN", - "vertexId": "c0" - }, - { - "id": 1, - "vertexType": "COLUMN", - "vertexId": "c1" - }, - { - "id": 2, - "vertexType": "COLUMN", - "vertexId": "c2" - }, - { - "id": 3, - "vertexType": "COLUMN", - "vertexId": "default.src1.key" - }, - { - "id": 4, - "vertexType": "COLUMN", - "vertexId": "default.src1.value" - } - ] -} +{"version":"1.0","engine":"mr","hash":"5b1022708124ee2b80f9e2e8a0dcb15c","queryText":"SELECT substr(src1.key,1,1), count(DISTINCT substr(src1.value,5)),\nconcat(substr(src1.key,1,1),sum(substr(src1.value,5)))\nfrom src1\nGROUP BY substr(src1.key,1,1)","edges":[{"sources":[3],"targets":[0],"expression":"substr(src1.key, 1, 1)","edgeType":"PROJECTION"},{"sources":[4],"targets":[1],"expression":"count(DISTINCT substr(src1.value, 5))","edgeType":"PROJECTION"},{"sources":[3,4],"targets":[2],"expression":"concat(substr(src1.key, 1, 1), sum(substr(src1.value, 5)))","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"c0"},{"id":1,"vertexType":"COLUMN","vertexId":"c1"},{"id":2,"vertexType":"COLUMN","vertexId":"c2"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src1.value"}]} 7 1543.0 1 3 1296.0 2 6 21257.0