| ==== |
| ---- QUERY |
| drop database if exists lineage_test_db cascade |
| ==== |
| ---- QUERY |
| create database lineage_test_db |
| ==== |
| ---- QUERY |
| create table lineage_test_db.alltypes like functional_hbase.alltypes |
| ==== |
| ---- QUERY |
| create table lineage_test_db.alltypesnopart like functional.alltypesnopart |
| ==== |
| ---- QUERY |
| create table lineage_test_db.alltypessmall like functional.alltypessmall |
| ==== |
| ---- QUERY |
| create table lineage_test_db.alltypesinsert like functional.alltypesinsert |
| ==== |
| ---- QUERY |
| create table lineage_test_db.iceberg_partitioned like functional_parquet.iceberg_partitioned |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "d4c603c171ccad77991279cc4d643625", |
| "timestamp": 1687533491, |
| "vertices": [], |
| "edges": [], |
| "queryId": "7547f4882e03cdde:941c4bad00000000", |
| "user": "anurag", |
| "queryText": "create table lineage_test_db.foo (id int)", |
| "endTime": 1687533491 |
| } |
| ---- QUERY |
| # Test lineage is created for DDLs. |
| create table lineage_test_db.foo (id int) |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "ce2d65d4f720533b497fad12de070058", |
| "timestamp": 1687533496, |
| "vertices": [], |
| "edges": [], |
| "queryId": "554879b21cb9ae72:5e564c9200000000", |
| "user": "anurag", |
| "queryText": "create external table lineage_test_db.ext_tbl_loc (id int) location '/test-warehouse/lineage_test_db.db/'", |
| "endTime": 1687533496, |
| "tableLocation": "hdfs://localhost:20500/test-warehouse/lineage_test_db.db" |
| } |
| ---- QUERY |
| # Test location is populated for create external table with location specified. |
| create external table lineage_test_db.ext_tbl_loc (id int) location |
| '/test-warehouse/lineage_test_db.db/' |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "8a07e4a2c8adb0ec4b11276bc0914848", |
| "timestamp": 1687533501, |
| "vertices": [], |
| "edges": [], |
| "queryId": "1948dbf64fdfe0c2:2cbbe10700000000", |
| "user": "anurag", |
| "queryText": "create external table lineage_test_db.ext_tbl (id int)", |
| "endTime": 1687533501, |
| "tableLocation": "hdfs://localhost:20500/test-warehouse/lineage_test_db.db/ext_tbl" |
| } |
| ---- QUERY |
| # Test location is populated for create external table when location is not specified. |
| create external table lineage_test_db.ext_tbl (id int) |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "43a8b8b9355ae5b918c880af6f94bb3e", |
| "timestamp": 1687533506, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687533507, |
| "tableName": "lineage_test_db.ext_ctas", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "lineage_test_db.alltypes.id", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "bigint_col", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687533507, |
| "tableName": "lineage_test_db.ext_ctas", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "lineage_test_db.alltypes.bigint_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "bool_col", |
| "id": 4, |
| "metadata": { |
| "tableCreateTime": 1687533507, |
| "tableName": "lineage_test_db.ext_ctas", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "lineage_test_db.alltypes.bool_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "date_string_col", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687533507, |
| "tableName": "lineage_test_db.ext_ctas", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "lineage_test_db.alltypes.date_string_col", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "double_col", |
| "id": 8, |
| "metadata": { |
| "tableCreateTime": 1687533507, |
| "tableName": "lineage_test_db.ext_ctas", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "lineage_test_db.alltypes.double_col", |
| "id": 9, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "float_col", |
| "id": 10, |
| "metadata": { |
| "tableCreateTime": 1687533507, |
| "tableName": "lineage_test_db.ext_ctas", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "lineage_test_db.alltypes.float_col", |
| "id": 11, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "int_col", |
| "id": 12, |
| "metadata": { |
| "tableCreateTime": 1687533507, |
| "tableName": "lineage_test_db.ext_ctas", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "lineage_test_db.alltypes.int_col", |
| "id": 13, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "month", |
| "id": 14, |
| "metadata": { |
| "tableCreateTime": 1687533507, |
| "tableName": "lineage_test_db.ext_ctas", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "lineage_test_db.alltypes.month", |
| "id": 15, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "smallint_col", |
| "id": 16, |
| "metadata": { |
| "tableCreateTime": 1687533507, |
| "tableName": "lineage_test_db.ext_ctas", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "lineage_test_db.alltypes.smallint_col", |
| "id": 17, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "string_col", |
| "id": 18, |
| "metadata": { |
| "tableCreateTime": 1687533507, |
| "tableName": "lineage_test_db.ext_ctas", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "lineage_test_db.alltypes.string_col", |
| "id": 19, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "timestamp_col", |
| "id": 20, |
| "metadata": { |
| "tableCreateTime": 1687533507, |
| "tableName": "lineage_test_db.ext_ctas", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "lineage_test_db.alltypes.timestamp_col", |
| "id": 21, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "tinyint_col", |
| "id": 22, |
| "metadata": { |
| "tableCreateTime": 1687533507, |
| "tableName": "lineage_test_db.ext_ctas", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "lineage_test_db.alltypes.tinyint_col", |
| "id": 23, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "year", |
| "id": 24, |
| "metadata": { |
| "tableCreateTime": 1687533507, |
| "tableName": "lineage_test_db.ext_ctas", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "lineage_test_db.alltypes.year", |
| "id": 25, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 7 |
| ], |
| "targets": [ |
| 6 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 9 |
| ], |
| "targets": [ |
| 8 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 11 |
| ], |
| "targets": [ |
| 10 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 13 |
| ], |
| "targets": [ |
| 12 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 15 |
| ], |
| "targets": [ |
| 14 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 17 |
| ], |
| "targets": [ |
| 16 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 19 |
| ], |
| "targets": [ |
| 18 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 21 |
| ], |
| "targets": [ |
| 20 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 23 |
| ], |
| "targets": [ |
| 22 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 25 |
| ], |
| "targets": [ |
| 24 |
| ], |
| "edgeType": "PROJECTION" |
| } |
| ], |
| "queryId": "c0450321c0d12cbe:30419c8200000000", |
| "user": "anurag", |
| "queryText": "create external table lineage_test_db.ext_ctas as select * from lineage_test_db.alltypes", |
| "endTime": 1687533508, |
| "tableLocation": "hdfs://localhost:20500/test-warehouse/lineage_test_db.db/ext_ctas" |
| } |
| ---- QUERY |
| # Test lineage is populate for create external table as select. |
| create external table lineage_test_db.ext_ctas as select * from lineage_test_db.alltypes |
| ==== |
| ---- QUERY |
| create view lineage_test_db.alltypes_view as select * from lineage_test_db.alltypes |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "3968bd65781e9e856eaca799f4501513", |
| "timestamp": 1687533513, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "x", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.bigint_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.int_col", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.tinyint_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1, |
| 2, |
| 3 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| } |
| ], |
| "queryId": "8e4d008c58bff1cc:1eb556a200000000", |
| "user": "anurag", |
| "queryText": "select * from ( select tinyint_col + int_col x from functional.alltypes union all select sum(bigint_col) y from (select bigint_col from functional.alltypes) v1) v2", |
| "endTime": 1687533513 |
| } |
| ---- QUERY |
| # Simple query to introduce projection dependencies. Shows resolution |
| # through inline views, unions, and materialization points. |
| select * from ( |
| select tinyint_col + int_col x from functional.alltypes |
| union all |
| select sum(bigint_col) y from (select bigint_col from functional.alltypes) v1) v2 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "0158d41c97236fb9d6b56f5f6fe3d2e6", |
| "timestamp": 1687533518, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "sum(a.tinyint_col) OVER(...)", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.tinyint_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.id", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.smallint_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "count(b.string_col)", |
| "id": 4 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.string_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "timestamp_col", |
| "id": 6 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.timestamp_col", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.int_col", |
| "id": 8, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.year", |
| "id": 9, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.bigint_col", |
| "id": 10, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.float_col", |
| "id": 11, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.id", |
| "id": 12, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 2, |
| 3 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PREDICATE" |
| }, |
| { |
| "sources": [ |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 7 |
| ], |
| "targets": [ |
| 6 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1, |
| 2, |
| 3, |
| 5, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12 |
| ], |
| "targets": [ |
| 0, |
| 4, |
| 6 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "374d7d4cfcda6f00:11963b1000000000", |
| "user": "anurag", |
| "queryText": "select sum(a.tinyint_col) over (partition by a.smallint_col order by a.id), count(b.string_col), b.timestamp_col from functional.alltypes a join functional.alltypessmall b on (a.id = b.id) where a.year = 2010 and b.float_col > 0 group by a.tinyint_col, a.smallint_col, a.id, b.string_col, b.timestamp_col, b.bigint_col having count(a.int_col) > 10 order by b.bigint_col limit 10", |
| "endTime": 1687533519 |
| } |
| ---- QUERY |
| # Simple query to introduce predicate dependencies. Shows conjuncts in WHERE, ON, and |
| # HAVING clause, as well dependencies introduced by GROUP BY and ORDER BY clauses and |
| # analytic functions. |
| select sum(a.tinyint_col) over (partition by a.smallint_col order by a.id), |
| count(b.string_col), b.timestamp_col |
| from functional.alltypes a join functional.alltypessmall b on (a.id = b.id) |
| where a.year = 2010 and b.float_col > 0 |
| group by a.tinyint_col, a.smallint_col, a.id, b.string_col, b.timestamp_col, b.bigint_col |
| having count(a.int_col) > 10 |
| order by b.bigint_col limit 10 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "ad70ea47a7772f9d59c60ebd4e9c1ac8", |
| "timestamp": 1687533524, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "int_col", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687533524, |
| "tableName": "lineage_test_db.lineage_test_tbl1", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.int_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "tinyint_col", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687533524, |
| "tableName": "lineage_test_db.lineage_test_tbl1", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.tinyint_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| } |
| ], |
| "queryId": "5341037bc766b0c5:5b9b5c6a00000000", |
| "user": "anurag", |
| "queryText": "create table lineage_test_db.lineage_test_tbl1 as select int_col, tinyint_col from functional.alltypes", |
| "endTime": 1687533524 |
| } |
| ---- QUERY |
| # CTAS queries |
| create table lineage_test_db.lineage_test_tbl1 as select int_col, tinyint_col from functional.alltypes |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "3d7db666c62b0c19ad8e0dd87fa628b0", |
| "timestamp": 1687533529, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "int_col", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687533529, |
| "tableName": "lineage_test_db.lineage_test_tbl2", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.int_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "string_col", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687533529, |
| "tableName": "lineage_test_db.lineage_test_tbl2", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.string_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.id", |
| "id": 4, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.year", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.id", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.month", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 4, |
| 5, |
| 6, |
| 7 |
| ], |
| "targets": [ |
| 0, |
| 2 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "2545984dd298803d:532b0b6100000000", |
| "user": "anurag", |
| "queryText": "create table lineage_test_db.lineage_test_tbl2 as select distinct a.int_col, a.string_col from functional.alltypes a inner join functional.alltypessmall b on (a.id = b.id) where a.year = 2009 and b.month = 2", |
| "endTime": 1687533530 |
| } |
| ---- QUERY |
| create table lineage_test_db.lineage_test_tbl2 as |
| select distinct a.int_col, a.string_col from functional.alltypes a |
| inner join functional.alltypessmall b on (a.id = b.id) |
| where a.year = 2009 and b.month = 2 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "04f29ab8b0aec034def6173b0c36c588", |
| "timestamp": 1687533535, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "int_col", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687533535, |
| "tableName": "lineage_test_db.lineage_test_tbl3", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypestiny.int_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531270, |
| "tableName": "functional.alltypestiny", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| } |
| ], |
| "queryId": "e741f71c0e666916:abefa06a00000000", |
| "user": "anurag", |
| "queryText": "create table lineage_test_db.lineage_test_tbl3 as select * from (select * from (select int_col from functional.alltypestiny limit 1) v1 ) v2", |
| "endTime": 1687533535 |
| } |
| ---- QUERY |
| create table lineage_test_db.lineage_test_tbl3 as |
| select * from |
| (select * from |
| (select int_col from functional.alltypestiny limit 1) v1 ) v2 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "07232504cc67447abf639eed1c6c2411", |
| "timestamp": 1687533540, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687533540, |
| "tableName": "lineage_test_db.lineage_test_tbl4", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypes.id", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531612, |
| "tableName": "functional_hbase.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "bigint_col", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687533540, |
| "tableName": "lineage_test_db.lineage_test_tbl4", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypes.bigint_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531612, |
| "tableName": "functional_hbase.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "bool_col", |
| "id": 4, |
| "metadata": { |
| "tableCreateTime": 1687533540, |
| "tableName": "lineage_test_db.lineage_test_tbl4", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypes.bool_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531612, |
| "tableName": "functional_hbase.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "date_string_col", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687533540, |
| "tableName": "lineage_test_db.lineage_test_tbl4", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypes.date_string_col", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687531612, |
| "tableName": "functional_hbase.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "double_col", |
| "id": 8, |
| "metadata": { |
| "tableCreateTime": 1687533540, |
| "tableName": "lineage_test_db.lineage_test_tbl4", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypes.double_col", |
| "id": 9, |
| "metadata": { |
| "tableCreateTime": 1687531612, |
| "tableName": "functional_hbase.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "float_col", |
| "id": 10, |
| "metadata": { |
| "tableCreateTime": 1687533540, |
| "tableName": "lineage_test_db.lineage_test_tbl4", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypes.float_col", |
| "id": 11, |
| "metadata": { |
| "tableCreateTime": 1687531612, |
| "tableName": "functional_hbase.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "int_col", |
| "id": 12, |
| "metadata": { |
| "tableCreateTime": 1687533540, |
| "tableName": "lineage_test_db.lineage_test_tbl4", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypes.int_col", |
| "id": 13, |
| "metadata": { |
| "tableCreateTime": 1687531612, |
| "tableName": "functional_hbase.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "month", |
| "id": 14, |
| "metadata": { |
| "tableCreateTime": 1687533540, |
| "tableName": "lineage_test_db.lineage_test_tbl4", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypes.month", |
| "id": 15, |
| "metadata": { |
| "tableCreateTime": 1687531612, |
| "tableName": "functional_hbase.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "smallint_col", |
| "id": 16, |
| "metadata": { |
| "tableCreateTime": 1687533540, |
| "tableName": "lineage_test_db.lineage_test_tbl4", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypes.smallint_col", |
| "id": 17, |
| "metadata": { |
| "tableCreateTime": 1687531612, |
| "tableName": "functional_hbase.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "string_col", |
| "id": 18, |
| "metadata": { |
| "tableCreateTime": 1687533540, |
| "tableName": "lineage_test_db.lineage_test_tbl4", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypes.string_col", |
| "id": 19, |
| "metadata": { |
| "tableCreateTime": 1687531612, |
| "tableName": "functional_hbase.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "timestamp_col", |
| "id": 20, |
| "metadata": { |
| "tableCreateTime": 1687533540, |
| "tableName": "lineage_test_db.lineage_test_tbl4", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypes.timestamp_col", |
| "id": 21, |
| "metadata": { |
| "tableCreateTime": 1687531612, |
| "tableName": "functional_hbase.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "tinyint_col", |
| "id": 22, |
| "metadata": { |
| "tableCreateTime": 1687533540, |
| "tableName": "lineage_test_db.lineage_test_tbl4", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypes.tinyint_col", |
| "id": 23, |
| "metadata": { |
| "tableCreateTime": 1687531612, |
| "tableName": "functional_hbase.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "year", |
| "id": 24, |
| "metadata": { |
| "tableCreateTime": 1687533540, |
| "tableName": "lineage_test_db.lineage_test_tbl4", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypes.year", |
| "id": 25, |
| "metadata": { |
| "tableCreateTime": 1687531612, |
| "tableName": "functional_hbase.alltypes", |
| "tableType": "hbase" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 7 |
| ], |
| "targets": [ |
| 6 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 9 |
| ], |
| "targets": [ |
| 8 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 11 |
| ], |
| "targets": [ |
| 10 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 13 |
| ], |
| "targets": [ |
| 12 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 15 |
| ], |
| "targets": [ |
| 14 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 17 |
| ], |
| "targets": [ |
| 16 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 19 |
| ], |
| "targets": [ |
| 18 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 21 |
| ], |
| "targets": [ |
| 20 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 23 |
| ], |
| "targets": [ |
| 22 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 25 |
| ], |
| "targets": [ |
| 24 |
| ], |
| "edgeType": "PROJECTION" |
| } |
| ], |
| "queryId": "a940f90c620a7259:2137273d00000000", |
| "user": "anurag", |
| "queryText": "create table lineage_test_db.lineage_test_tbl4 as select * from functional_hbase.alltypes limit 5", |
| "endTime": 1687533540 |
| } |
| ---- QUERY |
| # CTAS from HBase table |
| create table lineage_test_db.lineage_test_tbl4 as select * from functional_hbase.alltypes limit 5 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "861e5f5d91ce0e9e23db6e7f7f451fac", |
| "timestamp": 1687533545, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "bigint_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "bool_col", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "date_string_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "double_col", |
| "id": 4, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "float_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "int_col", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "month", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "smallint_col", |
| "id": 8, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "string_col", |
| "id": 9, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "timestamp_col", |
| "id": 10, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "tinyint_col", |
| "id": 11, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "year", |
| "id": 12, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypes", |
| "tableType": "hbase" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 1 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 3 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 5 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 6 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 7 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 8 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 9 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 10 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 11 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 12 |
| ], |
| "edgeType": "PROJECTION" |
| } |
| ], |
| "queryId": "664e64cfb1d3fd09:ef8dd84600000000", |
| "user": "anurag", |
| "queryText": "insert into lineage_test_db.alltypes values (1, 1, true, \"1999-12-01\", 2.0, 1.0, 1, 12, 2, \"abs\", cast(now() as timestamp), 1, 1999)", |
| "endTime": 1687533546 |
| } |
| ---- QUERY |
| # Insert into an HBase table |
| insert into |
| lineage_test_db.alltypes |
| values (1, 1, true, "1999-12-01", 2.0, 1.0, 1, 12, 2, "abs", |
| cast(now() as timestamp), 1, 1999) |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "8b1ddbc13665ae532868598c025d1668", |
| "timestamp": 1687533551, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypesnopart", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.id", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "bool_col", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypesnopart", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.bool_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "tinyint_col", |
| "id": 4, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypesnopart", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "smallint_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypesnopart", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "int_col", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypesnopart", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "bigint_col", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypesnopart", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "float_col", |
| "id": 8, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypesnopart", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "double_col", |
| "id": 9, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypesnopart", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "date_string_col", |
| "id": 10, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypesnopart", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "string_col", |
| "id": 11, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypesnopart", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "timestamp_col", |
| "id": 12, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypesnopart", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.timestamp_col", |
| "id": 13, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 5 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 6 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 7 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 8 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 9 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 10 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 11 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 13 |
| ], |
| "targets": [ |
| 12 |
| ], |
| "edgeType": "PROJECTION" |
| } |
| ], |
| "queryId": "784e59087868d497:418a666800000000", |
| "user": "anurag", |
| "queryText": "insert into table lineage_test_db.alltypesnopart (id, bool_col, timestamp_col) select id, bool_col, timestamp_col from functional.alltypes", |
| "endTime": 1687533551 |
| } |
| ---- QUERY |
| insert into table lineage_test_db.alltypesnopart (id, bool_col, timestamp_col) |
| select id, bool_col, timestamp_col |
| from functional.alltypes |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "e7c1630d4593c64d0372e32c28484667", |
| "timestamp": 1687533556, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "year", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "month", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "bool_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "tinyint_col", |
| "id": 4, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "smallint_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.smallint_col", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "int_col", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.int_col", |
| "id": 8, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "bigint_col", |
| "id": 9, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "float_col", |
| "id": 10, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "double_col", |
| "id": 11, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "date_string_col", |
| "id": 12, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "string_col", |
| "id": 13, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "timestamp_col", |
| "id": 14, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.month", |
| "id": 15, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.year", |
| "id": 16, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 1 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 3 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 6 |
| ], |
| "targets": [ |
| 5 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 8 |
| ], |
| "targets": [ |
| 7 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 9 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 10 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 11 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 12 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 13 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 14 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 15, |
| 16 |
| ], |
| "targets": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 7, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "50459cec26bebcb8:6d71e38a00000000", |
| "user": "anurag", |
| "queryText": "insert into table lineage_test_db.alltypessmall (smallint_col, int_col) partition (year=2009, month=04) select smallint_col, int_col from functional.alltypes where year=2009 and month=05", |
| "endTime": 1687533556 |
| } |
| ---- QUERY |
| insert into table lineage_test_db.alltypessmall (smallint_col, int_col) |
| partition (year=2009, month=04) |
| select smallint_col, int_col |
| from functional.alltypes |
| where year=2009 and month=05 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "727d89dd58676c0d36995a90e625607e", |
| "timestamp": 1687533561, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "year", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_seq_snap.alltypes.year", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531272, |
| "tableName": "functional_seq_snap.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "month", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_seq_snap.alltypes.month", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531272, |
| "tableName": "functional_seq_snap.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 4, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_seq_snap.alltypes.id", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531272, |
| "tableName": "functional_seq_snap.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "bool_col", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "tinyint_col", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "smallint_col", |
| "id": 8, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "int_col", |
| "id": 9, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_seq_snap.alltypes.int_col", |
| "id": 10, |
| "metadata": { |
| "tableCreateTime": 1687531272, |
| "tableName": "functional_seq_snap.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "bigint_col", |
| "id": 11, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "float_col", |
| "id": 12, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "double_col", |
| "id": 13, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "date_string_col", |
| "id": 14, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "string_col", |
| "id": 15, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_seq_snap.alltypes.string_col", |
| "id": 16, |
| "metadata": { |
| "tableCreateTime": 1687531272, |
| "tableName": "functional_seq_snap.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "timestamp_col", |
| "id": 17, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 6 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 7 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 8 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 10 |
| ], |
| "targets": [ |
| 9 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 11 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 12 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 13 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 14 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 16 |
| ], |
| "targets": [ |
| 15 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 17 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1, |
| 3 |
| ], |
| "targets": [ |
| 0, |
| 2, |
| 4, |
| 6, |
| 7, |
| 8, |
| 9, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 17 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "ca4e2ac78776624d:6e44f11c00000000", |
| "user": "anurag", |
| "queryText": "insert into table lineage_test_db.alltypessmall (id, string_col, int_col) partition (year, month) select id, string_col, int_col, year, month from functional_seq_snap.alltypes where year=2009 and month>10", |
| "endTime": 1687533562 |
| } |
| ---- QUERY |
| insert into table lineage_test_db.alltypessmall (id, string_col, int_col) |
| partition (year, month) |
| select id, string_col, int_col, year, month |
| from functional_seq_snap.alltypes |
| where year=2009 and month>10 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "516fa0d094aca2d64042484a84e19208", |
| "timestamp": 1687533567, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "year", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "month", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.month", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.id", |
| "id": 4, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "bool_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.bool_col", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "tinyint_col", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.tinyint_col", |
| "id": 8, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "smallint_col", |
| "id": 9, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.smallint_col", |
| "id": 10, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "int_col", |
| "id": 11, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.int_col", |
| "id": 12, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "bigint_col", |
| "id": 13, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.bigint_col", |
| "id": 14, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "float_col", |
| "id": 15, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.float_col", |
| "id": 16, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "double_col", |
| "id": 17, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.double_col", |
| "id": 18, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "date_string_col", |
| "id": 19, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.date_string_col", |
| "id": 20, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "string_col", |
| "id": 21, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.string_col", |
| "id": 22, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "timestamp_col", |
| "id": 23, |
| "metadata": { |
| "tableCreateTime": 1687533490, |
| "tableName": "lineage_test_db.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.timestamp_col", |
| "id": 24, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.year", |
| "id": 25, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 2 |
| ], |
| "targets": [ |
| 1 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 4 |
| ], |
| "targets": [ |
| 3 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 6 |
| ], |
| "targets": [ |
| 5 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 8 |
| ], |
| "targets": [ |
| 7 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 10 |
| ], |
| "targets": [ |
| 9 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 12 |
| ], |
| "targets": [ |
| 11 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 14 |
| ], |
| "targets": [ |
| 13 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 16 |
| ], |
| "targets": [ |
| 15 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 18 |
| ], |
| "targets": [ |
| 17 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 20 |
| ], |
| "targets": [ |
| 19 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 22 |
| ], |
| "targets": [ |
| 21 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 24 |
| ], |
| "targets": [ |
| 23 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 2, |
| 4, |
| 25 |
| ], |
| "targets": [ |
| 0, |
| 1, |
| 3, |
| 5, |
| 7, |
| 9, |
| 11, |
| 13, |
| 15, |
| 17, |
| 19, |
| 21, |
| 23 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "e64b728b24bbd5e9:d50607e600000000", |
| "user": "anurag", |
| "queryText": "insert into table lineage_test_db.alltypessmall partition (year=2009, month) select min(id), min(bool_col), min(tinyint_col), min(smallint_col), min(int_col), min(bigint_col), min(float_col), min(double_col), min(date_string_col), min(string_col), min(timestamp_col), month from functional.alltypes where year=2009 and month>10 group by month having min(id) > 10", |
| "endTime": 1687533567 |
| } |
| ---- QUERY |
| insert into table lineage_test_db.alltypessmall |
| partition (year=2009, month) |
| select min(id), min(bool_col), min(tinyint_col), min(smallint_col), min(int_col), |
| min(bigint_col), min(float_col), min(double_col), min(date_string_col), min(string_col), |
| min(timestamp_col), month |
| from functional.alltypes |
| where year=2009 and month>10 |
| group by month |
| having min(id) > 10 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "c82935c5165916f6c56db043120b1b45", |
| "timestamp": 1687533572, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "max(tinyint_col) OVER(...)", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.tinyint_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.int_col", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 2 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "b4403b6c2b81f788:b8af2f7700000000", |
| "user": "anurag", |
| "queryText": "select max(tinyint_col) over(partition by int_col) from functional.alltypes group by int_col, tinyint_col", |
| "endTime": 1687533573 |
| } |
| ---- QUERY |
| # Select statements with analytic functions |
| select |
| max(tinyint_col) over(partition by int_col) |
| from functional.alltypes |
| group by int_col, tinyint_col |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "6c646de10e62d1155b5216ae8589ff6d", |
| "timestamp": 1687533578, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "int_col", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypesagg.int_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531275, |
| "tableName": "functional.alltypesagg", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "rank() OVER(...)", |
| "id": 2 |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "fb4794254fb75b20:d1daf37500000000", |
| "user": "anurag", |
| "queryText": "select int_col, rank() over(order by int_col) from functional.alltypesagg", |
| "endTime": 1687533578 |
| } |
| ---- QUERY |
| select int_col, rank() over(order by int_col) from functional.alltypesagg |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "3c1677d7fcf4b2db27e84f30f5b3e96f", |
| "timestamp": 1687533583, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "tinyint_col", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.tinyint_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "int_col", |
| "id": 2 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.int_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "count(a.double_col) OVER(...)", |
| "id": 4 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.double_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.id", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.id", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1, |
| 3 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PREDICATE" |
| }, |
| { |
| "sources": [ |
| 6, |
| 7 |
| ], |
| "targets": [ |
| 0, |
| 2, |
| 4 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "8d4ddc6897bb8c9f:2f295b1700000000", |
| "user": "anurag", |
| "queryText": "select a.tinyint_col, a.int_col, count(a.double_col) over(partition by a.tinyint_col order by a.int_col desc rows between 1 preceding and 1 following) from functional.alltypes a inner join functional.alltypessmall b on a.id = b.id order by a.tinyint_col, a.int_col", |
| "endTime": 1687533583 |
| } |
| ---- QUERY |
| select a.tinyint_col, a.int_col, count(a.double_col) |
| over(partition by a.tinyint_col order by a.int_col desc rows between 1 preceding and 1 following) |
| from functional.alltypes a inner join functional.alltypessmall b on a.id = b.id |
| order by a.tinyint_col, a.int_col |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "547ead4bd734c7e6720ba5df3f7da88c", |
| "timestamp": 1687533588, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "double_col", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.double_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "a", |
| "id": 2 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.int_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "b", |
| "id": 4 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.bigint_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.bool_col", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "a + b", |
| "id": 7 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "double_col + a", |
| "id": 8 |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3, |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 6 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PREDICATE" |
| }, |
| { |
| "sources": [ |
| 3, |
| 5 |
| ], |
| "targets": [ |
| 7 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 6 |
| ], |
| "targets": [ |
| 7 |
| ], |
| "edgeType": "PREDICATE" |
| }, |
| { |
| "sources": [ |
| 1, |
| 3 |
| ], |
| "targets": [ |
| 8 |
| ], |
| "edgeType": "PROJECTION" |
| } |
| ], |
| "queryId": "c14107654ce903b7:9bd3363200000000", |
| "user": "anurag", |
| "queryText": "with v2 as (select double_col, count(int_col) over() a, sum(int_col + bigint_col) over(partition by bool_col) b from (select * from functional.alltypes) v1) select double_col, a, b, a + b, double_col + a from v2 order by 2, 3, 4", |
| "endTime": 1687533589 |
| } |
| ---- QUERY |
| with v2 as |
| (select |
| double_col, |
| count(int_col) over() a, |
| sum(int_col + bigint_col) over(partition by bool_col) b |
| from |
| (select * from functional.alltypes) v1) |
| select double_col, a, b, a + b, double_col + a from v2 |
| order by 2, 3, 4 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "952f8a7042ff661ed34fbc8167349899", |
| "timestamp": 1687533594, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "double_col", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.double_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "a", |
| "id": 2 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.int_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "b", |
| "id": 4 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.bigint_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.bool_col", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "a + b", |
| "id": 7 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "double_col + a", |
| "id": 8 |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3, |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 6 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PREDICATE" |
| }, |
| { |
| "sources": [ |
| 3, |
| 5 |
| ], |
| "targets": [ |
| 7 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 6 |
| ], |
| "targets": [ |
| 7 |
| ], |
| "edgeType": "PREDICATE" |
| }, |
| { |
| "sources": [ |
| 1, |
| 3 |
| ], |
| "targets": [ |
| 8 |
| ], |
| "edgeType": "PROJECTION" |
| } |
| ], |
| "queryId": "1744e089e7cc73a9:ed50bff000000000", |
| "user": "anurag", |
| "queryText": "select double_col, a, b, a + b, double_col + a from (select double_col, count(int_col) over() a, sum(int_col + bigint_col) over(partition by bool_col) b from (select * from functional.alltypes) v1) v2 order by 2, 3, 4", |
| "endTime": 1687533594 |
| } |
| ---- QUERY |
| select double_col, a, b, a + b, double_col + a from |
| (select |
| double_col, |
| count(int_col) over() a, |
| sum(int_col + bigint_col) over(partition by bool_col) b |
| from |
| (select * from functional.alltypes) v1) v2 |
| order by 2, 3, 4 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "7b677a184cce15efd15f3033b9b2feeb", |
| "timestamp": 1687533599, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "month", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.month", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "year", |
| "id": 2 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.year", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "int_col", |
| "id": 4 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.int_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.month", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 6 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1, |
| 6 |
| ], |
| "targets": [ |
| 0, |
| 2, |
| 4 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "4740735360ac58fe:543457eb00000000", |
| "user": "anurag", |
| "queryText": "select a.month, a.year, b.int_col, b.month from (select year, month from functional.alltypes union all select year, month from functional.alltypes) a inner join functional.alltypessmall b on (a.month = b.month) where b.month = 1", |
| "endTime": 1687533599 |
| } |
| ---- QUERY |
| # Union statements |
| select a.month, a.year, b.int_col, b.month |
| from |
| (select year, month from functional.alltypes |
| union all |
| select year, month from functional.alltypes) a |
| inner join |
| functional.alltypessmall b |
| on (a.month = b.month) |
| where b.month = 1 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "2f2aab8dd7a5ca2112311dc4bc676e5d", |
| "timestamp": 1687533604, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "int_col", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.int_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "month", |
| "id": 2 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.month", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "t2.int_col + 1", |
| "id": 4 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.int_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.month", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1, |
| 3, |
| 5, |
| 6 |
| ], |
| "targets": [ |
| 0, |
| 2, |
| 4 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "704c9e102ed9744a:97d7530700000000", |
| "user": "anurag", |
| "queryText": "select t1.int_col, t2.month, t2.int_col + 1 from ( select int_col, count(*) from functional.alltypessmall where month = 1 group by int_col having count(*) > 1 order by count(*) desc limit 5 ) t1 join functional.alltypes t2 on (t1.int_col = t2.int_col) where month = 1", |
| "endTime": 1687533604 |
| } |
| ---- QUERY |
| # Select statements with inline views |
| select t1.int_col, t2.month, t2.int_col + 1 |
| from ( |
| select int_col, count(*) |
| from functional.alltypessmall |
| where month = 1 |
| group by int_col |
| having count(*) > 1 |
| order by count(*) desc limit 5 |
| ) t1 |
| join functional.alltypes t2 on (t1.int_col = t2.int_col) |
| where month = 1 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "d86f371388b5a5024bca2b7fc487d8f7", |
| "timestamp": 1687533609, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "smallint_col", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypesagg.smallint_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531275, |
| "tableName": "functional.alltypesagg", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 2 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.id", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "tinyint_col", |
| "id": 4 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypesagg.tinyint_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531275, |
| "tableName": "functional.alltypesagg", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "int_col", |
| "id": 6 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypesagg.int_col", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687531275, |
| "tableName": "functional.alltypesagg", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "float_col", |
| "id": 8 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.float_col", |
| "id": 9, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "string_col", |
| "id": 10 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.string_col", |
| "id": 11, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypesagg.day", |
| "id": 12, |
| "metadata": { |
| "tableCreateTime": 1687531275, |
| "tableName": "functional.alltypesagg", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypesagg.month", |
| "id": 13, |
| "metadata": { |
| "tableCreateTime": 1687531275, |
| "tableName": "functional.alltypesagg", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 7 |
| ], |
| "targets": [ |
| 6 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 9 |
| ], |
| "targets": [ |
| 8 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 11 |
| ], |
| "targets": [ |
| 10 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1, |
| 3, |
| 5, |
| 7, |
| 9, |
| 11, |
| 12, |
| 13 |
| ], |
| "targets": [ |
| 0, |
| 2, |
| 4, |
| 6, |
| 8, |
| 10 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "d5413148c4203ab5:24a6dc2f00000000", |
| "user": "anurag", |
| "queryText": "select x.smallint_col, x.id, x.tinyint_col, c.id, x.int_col, x.float_col, c.string_col from functional.alltypessmall c join ( select a.smallint_col smallint_col, a.tinyint_col tinyint_col, a.day day, a.int_col int_col, a.month month, b.float_col float_col, b.id id from ( select * from functional.alltypesagg a where month=1 ) a join functional.alltypessmall b on (a.smallint_col = b.id) ) x on (x.tinyint_col = c.id) where x.day=1 and x.int_col > 899 and x.float_col > 4.5 and c.string_col < '7' and x.int_col + x.float_col + cast(c.string_col as float) < 1000", |
| "endTime": 1687533610 |
| } |
| ---- QUERY |
| select x.smallint_col, x.id, x.tinyint_col, c.id, x.int_col, x.float_col, c.string_col |
| from functional.alltypessmall c |
| join ( |
| select a.smallint_col smallint_col, a.tinyint_col tinyint_col, a.day day, |
| a.int_col int_col, a.month month, b.float_col float_col, b.id id |
| from ( select * from functional.alltypesagg a where month=1 ) a |
| join functional.alltypessmall b on (a.smallint_col = b.id) |
| ) x on (x.tinyint_col = c.id) |
| where x.day=1 |
| and x.int_col > 899 |
| and x.float_col > 4.5 |
| and c.string_col < '7' |
| and x.int_col + x.float_col + cast(c.string_col as float) < 1000 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "38cda699ddffba2fa5bab14c8a9c4fb4", |
| "timestamp": 1687533615, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "c1", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypessmall.int_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531613, |
| "tableName": "functional_hbase.alltypessmall", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "c2", |
| "id": 2 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypessmall.float_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531613, |
| "tableName": "functional_hbase.alltypessmall", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "c3", |
| "id": 4 |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1, |
| 3 |
| ], |
| "targets": [ |
| 0, |
| 2, |
| 4 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "d94c43f20a44f454:87b6d0d300000000", |
| "user": "anurag", |
| "queryText": "select c1, c2, c3 from (select c1, c2, c3 from (select int_col c1, sum(float_col) c2, min(float_col) c3 from functional_hbase.alltypessmall group by 1) x order by 2,3 desc limit 5 ) y", |
| "endTime": 1687533616 |
| } |
| ---- QUERY |
| select c1, c2, c3 |
| from |
| (select c1, c2, c3 |
| from |
| (select int_col c1, sum(float_col) c2, min(float_col) c3 |
| from functional_hbase.alltypessmall |
| group by 1) x |
| order by 2,3 desc |
| limit 5 |
| ) y |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "4e0d48d11ef88837d0267a212fb67716", |
| "timestamp": 1687533621, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "c1", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypessmall.int_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531613, |
| "tableName": "functional_hbase.alltypessmall", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "x2", |
| "id": 2 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypessmall.tinyint_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531613, |
| "tableName": "functional_hbase.alltypessmall", |
| "tableType": "hbase" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1, |
| 3 |
| ], |
| "targets": [ |
| 0, |
| 2 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "674320549904c2a7:e85c13d400000000", |
| "user": "anurag", |
| "queryText": "select c1, x2 from ( select c1, min(c2) x2 from ( select c1, c2, c3 from ( select int_col c1, tinyint_col c2, min(float_col) c3 from functional_hbase.alltypessmall group by 1, 2 order by 1,2 limit 1 ) x ) x2 group by c1 ) y order by 2,1 desc limit 0", |
| "endTime": 1687533621 |
| } |
| ---- QUERY |
| select c1, x2 |
| from ( |
| select c1, min(c2) x2 |
| from ( |
| select c1, c2, c3 |
| from ( |
| select int_col c1, tinyint_col c2, min(float_col) c3 |
| from functional_hbase.alltypessmall |
| group by 1, 2 |
| order by 1,2 |
| limit 1 |
| ) x |
| ) x2 |
| group by c1 |
| ) y |
| order by 2,1 desc |
| limit 0 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "073a1917cdbbb2b53ac89c62d0852270", |
| "timestamp": 1687533626, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.id", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| } |
| ], |
| "queryId": "1f45c1b8f43f6b96:a19ff31600000000", |
| "user": "anurag", |
| "queryText": "select id from functional.view_view", |
| "endTime": 1687533626 |
| } |
| ---- QUERY |
| # Views |
| select id from functional.view_view |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "b0ebbaee6fd5fcc500e03aa8dbb08707", |
| "timestamp": 1687533631, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.id", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "b7476b57060dceb7:b59a484600000000", |
| "user": "anurag", |
| "queryText": "select t.id from (select id from functional.alltypes_view) t where t.id < 10", |
| "endTime": 1687533631 |
| } |
| ---- QUERY |
| select t.id from (select id from functional.alltypes_view) t |
| where t.id < 10 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "70664c8c26f6a2da7c7f53c4b97b8892", |
| "timestamp": 1687533636, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "string_col", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.string_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "float_col", |
| "id": 2 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.float_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "bool_col", |
| "id": 4 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.bool_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.id", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypesagg.id", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687531275, |
| "tableName": "functional.alltypesagg", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 6, |
| 7 |
| ], |
| "targets": [ |
| 0, |
| 2, |
| 4 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "09489cc02e60feec:c8ddfb0b00000000", |
| "user": "anurag", |
| "queryText": "select string_col, float_col, bool_col from functional.alltypes where id in (select id from functional.alltypesagg)", |
| "endTime": 1687533637 |
| } |
| ---- QUERY |
| # Subqueries |
| select string_col, float_col, bool_col |
| from functional.alltypes |
| where id in |
| (select id from functional.alltypesagg) |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "97267ebbbeae23fa3bb8ddeb8375e7b9", |
| "timestamp": 1687533642, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "1", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypesagg.tinyint_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531275, |
| "tableName": "functional.alltypesagg", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypestiny.tinyint_col", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687531270, |
| "tableName": "functional.alltypestiny", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1, |
| 2 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "684366081fd04903:ce9b652700000000", |
| "user": "anurag", |
| "queryText": "select 1 from functional.alltypesagg a where exists (select id, count(int_col) over (partition by bool_col) from functional.alltypestiny b where a.tinyint_col = b.tinyint_col group by id, int_col, bool_col) and tinyint_col < 10", |
| "endTime": 1687533642 |
| } |
| ---- QUERY |
| select 1 |
| from functional.alltypesagg a |
| where exists |
| (select id, count(int_col) over (partition by bool_col) |
| from functional.alltypestiny b |
| where a.tinyint_col = b.tinyint_col |
| group by id, int_col, bool_col) |
| and tinyint_col < 10 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "b59cc1541d8583e3f1359e9f48c0963d", |
| "timestamp": 1687533647, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "int_col + 1", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.int_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "tinyint_col - 1", |
| "id": 2 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.tinyint_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.bigint_col", |
| "id": 4, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypesagg.bool_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531275, |
| "tableName": "functional.alltypesagg", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypesagg.int_col", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687531275, |
| "tableName": "functional.alltypesagg", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1, |
| 4, |
| 5, |
| 6 |
| ], |
| "targets": [ |
| 0, |
| 2 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "bc4aa55c595aa11f:b69f692600000000", |
| "user": "anurag", |
| "queryText": "select int_col + 1, tinyint_col - 1 from functional.alltypes a where a.int_col < (select max(int_col) from functional.alltypesagg g where g.bool_col = true) and a.bigint_col > 10", |
| "endTime": 1687533647 |
| } |
| ---- QUERY |
| select int_col + 1, tinyint_col - 1 |
| from functional.alltypes a |
| where a.int_col < |
| (select max(int_col) from functional.alltypesagg g where g.bool_col = true) |
| and a.bigint_col > 10 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "ff421b2c2b249764028350613be74fb8", |
| "timestamp": 1687533652, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "x", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.int_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "y", |
| "id": 2 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.bigint_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| } |
| ], |
| "queryId": "a54406d43e4870fb:cf7baf2100000000", |
| "user": "anurag", |
| "queryText": "with t as (select int_col x, bigint_col y from functional.alltypes) select x, y from t", |
| "endTime": 1687533652 |
| } |
| ---- QUERY |
| # With clause |
| with t as (select int_col x, bigint_col y from functional.alltypes) select x, y from t |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "b2519388c17a48c91e5bb32948bce1c0", |
| "timestamp": 1687533657, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "year", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687533491, |
| "tableName": "lineage_test_db.alltypesinsert", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypestiny.year", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531270, |
| "tableName": "functional.alltypestiny", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "month", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687533491, |
| "tableName": "lineage_test_db.alltypesinsert", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypestiny.month", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531270, |
| "tableName": "functional.alltypestiny", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 4, |
| "metadata": { |
| "tableCreateTime": 1687533491, |
| "tableName": "lineage_test_db.alltypesinsert", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypestiny.id", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531270, |
| "tableName": "functional.alltypestiny", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "bool_col", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687533491, |
| "tableName": "lineage_test_db.alltypesinsert", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "tinyint_col", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687533491, |
| "tableName": "lineage_test_db.alltypesinsert", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "smallint_col", |
| "id": 8, |
| "metadata": { |
| "tableCreateTime": 1687533491, |
| "tableName": "lineage_test_db.alltypesinsert", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "int_col", |
| "id": 9, |
| "metadata": { |
| "tableCreateTime": 1687533491, |
| "tableName": "lineage_test_db.alltypesinsert", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypestiny.int_col", |
| "id": 10, |
| "metadata": { |
| "tableCreateTime": 1687531270, |
| "tableName": "functional.alltypestiny", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "bigint_col", |
| "id": 11, |
| "metadata": { |
| "tableCreateTime": 1687533491, |
| "tableName": "lineage_test_db.alltypesinsert", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "float_col", |
| "id": 12, |
| "metadata": { |
| "tableCreateTime": 1687533491, |
| "tableName": "lineage_test_db.alltypesinsert", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "double_col", |
| "id": 13, |
| "metadata": { |
| "tableCreateTime": 1687533491, |
| "tableName": "lineage_test_db.alltypesinsert", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "date_string_col", |
| "id": 14, |
| "metadata": { |
| "tableCreateTime": 1687533491, |
| "tableName": "lineage_test_db.alltypesinsert", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "string_col", |
| "id": 15, |
| "metadata": { |
| "tableCreateTime": 1687533491, |
| "tableName": "lineage_test_db.alltypesinsert", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypestiny.string_col", |
| "id": 16, |
| "metadata": { |
| "tableCreateTime": 1687531270, |
| "tableName": "functional.alltypestiny", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "timestamp_col", |
| "id": 17, |
| "metadata": { |
| "tableCreateTime": 1687533491, |
| "tableName": "lineage_test_db.alltypesinsert", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 6 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 7 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 8 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 10 |
| ], |
| "targets": [ |
| 9 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 11 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 12 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 13 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 14 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 16 |
| ], |
| "targets": [ |
| 15 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [], |
| "targets": [ |
| 17 |
| ], |
| "edgeType": "PROJECTION" |
| } |
| ], |
| "queryId": "7d4e988e98626451:c0be63a200000000", |
| "user": "anurag", |
| "queryText": "with t1 as (select * from functional.alltypestiny) insert into lineage_test_db.alltypesinsert (id, int_col, string_col) partition(year, month) select id, int_col, string_col, year, month from t1", |
| "endTime": 1687533658 |
| } |
| ---- QUERY |
| with t1 as (select * from functional.alltypestiny) |
| insert into lineage_test_db.alltypesinsert (id, int_col, string_col) partition(year, month) |
| select id, int_col, string_col, year, month from t1 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "b4d1a3120fe9c4e63e9a50e2bcf0ff22", |
| "timestamp": 1687533663, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "lead(a, 1, NULL) OVER(...)", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.id", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.bigint_col", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.bool_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.int_col", |
| "id": 4, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.string_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.tinyint_col", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "cf494d06aa9e201a:7047aec600000000", |
| "user": "anurag", |
| "queryText": "select lead(a) over (partition by b order by c) from (select lead(id) over (partition by int_col order by bigint_col) as a, max(id) over (partition by tinyint_col order by int_col) as b, min(int_col) over (partition by string_col order by bool_col) as c from functional.alltypes) v", |
| "endTime": 1687533663 |
| } |
| ---- QUERY |
| # Nested analytic functions |
| select lead(a) over (partition by b order by c) |
| from |
| (select lead(id) over (partition by int_col order by bigint_col) as a, |
| max(id) over (partition by tinyint_col order by int_col) as b, |
| min(int_col) over (partition by string_col order by bool_col) as c |
| from functional.alltypes) v |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "bc19b44ea5cab4531beb8a1dc0779df6", |
| "timestamp": 1687533668, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687533668, |
| "tableName": "lineage_test_db.lineage_test_view1", |
| "tableType": "view" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypestiny.id", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531270, |
| "tableName": "functional.alltypestiny", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| } |
| ], |
| "queryId": "574dce023537d84c:5de5f32b00000000", |
| "user": "anurag", |
| "queryText": "create view lineage_test_db.lineage_test_view1 as select id from functional.alltypestiny", |
| "endTime": 1687533668 |
| } |
| ---- QUERY |
| # Create view statement (IMPALA-4219) |
| create view lineage_test_db.lineage_test_view1 as select id from functional.alltypestiny |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "4e9912a672b341c75a234945ac3a63d8", |
| "timestamp": 1687533673, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "a", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687533673, |
| "tableName": "lineage_test_db.lineage_test_view2", |
| "tableType": "view" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypessmall.int_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531613, |
| "tableName": "functional_hbase.alltypessmall", |
| "tableType": "hbase" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "b", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687533673, |
| "tableName": "lineage_test_db.lineage_test_view2", |
| "tableType": "view" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_hbase.alltypessmall.tinyint_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531613, |
| "tableName": "functional_hbase.alltypessmall", |
| "tableType": "hbase" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1, |
| 3 |
| ], |
| "targets": [ |
| 0, |
| 2 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "b54cc29caca2cf8a:99ff97ae00000000", |
| "user": "anurag", |
| "queryText": "create view lineage_test_db.lineage_test_view2 (a, b) as select c1, x2 from ( select c1, min(c2) x2 from ( select c1, c2, c3 from ( select int_col c1, tinyint_col c2, min(float_col) c3 from functional_hbase.alltypessmall group by 1, 2 order by 1,2 limit 1 ) x ) x2 group by c1 ) y order by 2,1 desc limit 0", |
| "endTime": 1687533673 |
| } |
| ---- QUERY |
| # Create view with nested inline views (IMPALA-4219) |
| create view lineage_test_db.lineage_test_view2 (a, b) as select c1, x2 |
| from ( |
| select c1, min(c2) x2 |
| from ( |
| select c1, c2, c3 |
| from ( |
| select int_col c1, tinyint_col c2, min(float_col) c3 |
| from functional_hbase.alltypessmall |
| group by 1, 2 |
| order by 1,2 |
| limit 1 |
| ) x |
| ) x2 |
| group by c1 |
| ) y |
| order by 2,1 desc |
| limit 0 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "fb45884e3b8668e1239a677983ffc706", |
| "timestamp": 1687533678, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "a1", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687533679, |
| "tableName": "lineage_test_db.lineage_test_view3", |
| "tableType": "view" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypesagg.smallint_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531275, |
| "tableName": "functional.alltypesagg", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "a2", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687533679, |
| "tableName": "lineage_test_db.lineage_test_view3", |
| "tableType": "view" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.id", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "a3", |
| "id": 4, |
| "metadata": { |
| "tableCreateTime": 1687533679, |
| "tableName": "lineage_test_db.lineage_test_view3", |
| "tableType": "view" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypesagg.tinyint_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531275, |
| "tableName": "functional.alltypesagg", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "a4", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687533679, |
| "tableName": "lineage_test_db.lineage_test_view3", |
| "tableType": "view" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "a5", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687533679, |
| "tableName": "lineage_test_db.lineage_test_view3", |
| "tableType": "view" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypesagg.int_col", |
| "id": 8, |
| "metadata": { |
| "tableCreateTime": 1687531275, |
| "tableName": "functional.alltypesagg", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "a6", |
| "id": 9, |
| "metadata": { |
| "tableCreateTime": 1687533679, |
| "tableName": "lineage_test_db.lineage_test_view3", |
| "tableType": "view" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.float_col", |
| "id": 10, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "a7", |
| "id": 11, |
| "metadata": { |
| "tableCreateTime": 1687533679, |
| "tableName": "lineage_test_db.lineage_test_view3", |
| "tableType": "view" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.string_col", |
| "id": 12, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypesagg.day", |
| "id": 13, |
| "metadata": { |
| "tableCreateTime": 1687531275, |
| "tableName": "functional.alltypesagg", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypesagg.month", |
| "id": 14, |
| "metadata": { |
| "tableCreateTime": 1687531275, |
| "tableName": "functional.alltypesagg", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 6 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 8 |
| ], |
| "targets": [ |
| 7 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 10 |
| ], |
| "targets": [ |
| 9 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 12 |
| ], |
| "targets": [ |
| 11 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1, |
| 3, |
| 5, |
| 8, |
| 10, |
| 12, |
| 13, |
| 14 |
| ], |
| "targets": [ |
| 0, |
| 2, |
| 4, |
| 6, |
| 7, |
| 9, |
| 11 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "2a43cfbaea01dda0:c08ec40600000000", |
| "user": "anurag", |
| "queryText": "create view lineage_test_db.lineage_test_view3 (a1, a2, a3, a4, a5, a6, a7) as select x.smallint_col, x.id, x.tinyint_col, c.id, x.int_col, x.float_col, c.string_col from functional.alltypessmall c join ( select a.smallint_col smallint_col, a.tinyint_col tinyint_col, a.day day, a.int_col int_col, a.month month, b.float_col float_col, b.id id from ( select * from functional.alltypesagg a where month=1 ) a join functional.alltypessmall b on (a.smallint_col = b.id) ) x on (x.tinyint_col = c.id) where x.day=1 and x.int_col > 899 and x.float_col > 4.5 and c.string_col < '7' and x.int_col + x.float_col + cast(c.string_col as float) < 1000", |
| "endTime": 1687533679 |
| } |
| ---- QUERY |
| # Create view with complex select statement (IMPALA-4219) |
| create view lineage_test_db.lineage_test_view3 (a1, a2, a3, a4, a5, a6, a7) as |
| select x.smallint_col, x.id, x.tinyint_col, c.id, x.int_col, x.float_col, c.string_col |
| from functional.alltypessmall c |
| join ( |
| select a.smallint_col smallint_col, a.tinyint_col tinyint_col, a.day day, |
| a.int_col int_col, a.month month, b.float_col float_col, b.id id |
| from ( select * from functional.alltypesagg a where month=1 ) a |
| join functional.alltypessmall b on (a.smallint_col = b.id) |
| ) x on (x.tinyint_col = c.id) |
| where x.day=1 |
| and x.int_col > 899 |
| and x.float_col > 4.5 |
| and c.string_col < '7' |
| and x.int_col + x.float_col + cast(c.string_col as float) < 1000 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "51e5187935df610408fd670895640190", |
| "timestamp": 1687533684, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "_c0", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687533684, |
| "tableName": "lineage_test_db.lineage_test_view4", |
| "tableType": "view" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.tinyint_col", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.id", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.smallint_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "_c1", |
| "id": 4, |
| "metadata": { |
| "tableCreateTime": 1687533684, |
| "tableName": "lineage_test_db.lineage_test_view4", |
| "tableType": "view" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.string_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "timestamp_col", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687533684, |
| "tableName": "lineage_test_db.lineage_test_view4", |
| "tableType": "view" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.timestamp_col", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.int_col", |
| "id": 8, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.year", |
| "id": 9, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.bigint_col", |
| "id": 10, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.float_col", |
| "id": 11, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypessmall.id", |
| "id": 12, |
| "metadata": { |
| "tableCreateTime": 1687531269, |
| "tableName": "functional.alltypessmall", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 2, |
| 3 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PREDICATE" |
| }, |
| { |
| "sources": [ |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 7 |
| ], |
| "targets": [ |
| 6 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1, |
| 2, |
| 3, |
| 5, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12 |
| ], |
| "targets": [ |
| 0, |
| 4, |
| 6 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "d448e01e4832881e:59ee2b4a00000000", |
| "user": "anurag", |
| "queryText": "create view lineage_test_db.lineage_test_view4 as select * from ( select sum(a.tinyint_col) over (partition by a.smallint_col order by a.id), count(b.string_col), b.timestamp_col from functional.alltypes a join functional.alltypessmall b on (a.id = b.id) where a.year = 2010 and b.float_col > 0 group by a.tinyint_col, a.smallint_col, a.id, b.string_col, b.timestamp_col, b.bigint_col having count(a.int_col) > 10 order by b.bigint_col limit 10) t", |
| "endTime": 1687533684 |
| } |
| ---- QUERY |
| # Create view with select statement containing an analytic function (IMPALA-4219) |
| create view lineage_test_db.lineage_test_view4 as |
| select * from ( |
| select sum(a.tinyint_col) over (partition by a.smallint_col order by a.id), |
| count(b.string_col), b.timestamp_col |
| from functional.alltypes a join functional.alltypessmall b on (a.id = b.id) |
| where a.year = 2010 and b.float_col > 0 |
| group by a.tinyint_col, a.smallint_col, a.id, b.string_col, b.timestamp_col, b.bigint_col |
| having count(a.int_col) > 10 |
| order by b.bigint_col limit 10) t |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "47f1c5e1dc11a2764f9ed27f9f2ef1ce", |
| "timestamp": 1687533689, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687533513, |
| "tableName": "lineage_test_db.alltypes_view", |
| "tableType": "view" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypestiny.id", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531270, |
| "tableName": "functional.alltypestiny", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| } |
| ], |
| "queryId": "b54e1dc4a678c946:eb23e54700000000", |
| "user": "anurag", |
| "queryText": "alter view lineage_test_db.alltypes_view as select id from functional.alltypestiny", |
| "endTime": 1687533689 |
| } |
| ---- QUERY |
| # Alter view statement (IMPALA-4219) |
| alter view lineage_test_db.alltypes_view as select id from functional.alltypestiny |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "b187fe7a4a6b47249f3084d3890b2c74", |
| "timestamp": 1687533694, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "x", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.complex_struct_col.f1", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.int_struct_col.f1", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.int_struct_col.f2", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.nested_struct_col.f2.f12.f21", |
| "id": 4, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.year", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1, |
| 2, |
| 3 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 4, |
| 5 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "ee4f63ccf2181e85:2b38e8dd00000000", |
| "user": "anurag", |
| "queryText": "select * from ( select int_struct_col.f1 + int_struct_col.f2 x from functional.allcomplextypes where year = 2000 order by nested_struct_col.f2.f12.f21 limit 10 union all select sum(f1) y from (select complex_struct_col.f1 f1 from functional.allcomplextypes group by 1) v1) v2", |
| "endTime": 1687533694 |
| } |
| ---- QUERY |
| # Test references to struct fields. |
| select * from ( |
| select int_struct_col.f1 + int_struct_col.f2 x from functional.allcomplextypes |
| where year = 2000 |
| order by nested_struct_col.f2.f12.f21 limit 10 |
| union all |
| select sum(f1) y from |
| (select complex_struct_col.f1 f1 from functional.allcomplextypes |
| group by 1) v1) v2 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "43303560d034d3c1b8fa72962da76f62", |
| "timestamp": 1687533699, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "item", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.int_array_col.item", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "key", |
| "id": 2 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.struct_map_col.key", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "f1", |
| "id": 4 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.struct_map_col.value.f1", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "f2", |
| "id": 6 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.struct_map_col.value.f2", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 7 |
| ], |
| "targets": [ |
| 6 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1, |
| 5 |
| ], |
| "targets": [ |
| 0, |
| 2, |
| 4, |
| 6 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "394a9bd88d04af98:9452981700000000", |
| "user": "anurag", |
| "queryText": "select * from functional.allcomplextypes.int_array_col a inner join functional.allcomplextypes.struct_map_col m on (a.item = m.f1)", |
| "endTime": 1687533700 |
| } |
| ---- QUERY |
| # Test absolute collection table refs. Ensure that the vertex ids of sources are |
| # always an absolute explicit path. |
| select * from functional.allcomplextypes.int_array_col a inner join |
| functional.allcomplextypes.struct_map_col m on (a.item = m.f1) |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "d34d7d50b599df3912e4265f15706810", |
| "timestamp": 1687533705, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.id", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "year", |
| "id": 2 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.year", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "month", |
| "id": 4 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.month", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "item", |
| "id": 6 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.int_array_col.item", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "key", |
| "id": 8 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.struct_map_col.key", |
| "id": 9, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "f1", |
| "id": 10 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.struct_map_col.value.f1", |
| "id": 11, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "f2", |
| "id": 12 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.struct_map_col.value.f2", |
| "id": 13, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.int_array_col", |
| "id": 14, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.struct_map_col", |
| "id": 15, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 7 |
| ], |
| "targets": [ |
| 6 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 9 |
| ], |
| "targets": [ |
| 8 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 11 |
| ], |
| "targets": [ |
| 10 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 13 |
| ], |
| "targets": [ |
| 12 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 7, |
| 11, |
| 14, |
| 15 |
| ], |
| "targets": [ |
| 0, |
| 2, |
| 4, |
| 6, |
| 8, |
| 10, |
| 12 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "b043e925908e15a2:1a5b1afe00000000", |
| "user": "anurag", |
| "queryText": "select * from functional.allcomplextypes t, t.int_array_col a, t.struct_map_col m where a.item = m.f1", |
| "endTime": 1687533705 |
| } |
| ---- QUERY |
| # Test relative collection table refs. Ensure that the vertex ids of sources are |
| # always an absolute explicit path. |
| select * from functional.allcomplextypes t, t.int_array_col a, t.struct_map_col m |
| where a.item = m.f1 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "e1684998c49aeacac122c78e7b47867f", |
| "timestamp": 1687533710, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "ab", |
| "id": 0 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.int_array_col.item", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.struct_map_col.value.f1", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "c", |
| "id": 3 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.map_map_col.value.value", |
| "id": 4, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.map_map_col.value.key", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "d", |
| "id": 6 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.int_map_col.value", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "e", |
| "id": 8 |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.complex_nested_struct_col.f2.item.f12.value.f21", |
| "id": 9, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.complex_nested_struct_col.f2.item.f12.key", |
| "id": 10, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.allcomplextypes.struct_map_col.key", |
| "id": 11, |
| "metadata": { |
| "tableCreateTime": 1687531278, |
| "tableName": "functional.allcomplextypes", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1, |
| 2 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 4 |
| ], |
| "targets": [ |
| 3 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 5 |
| ], |
| "targets": [ |
| 3 |
| ], |
| "edgeType": "PREDICATE" |
| }, |
| { |
| "sources": [ |
| 1, |
| 7 |
| ], |
| "targets": [ |
| 6 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 9 |
| ], |
| "targets": [ |
| 8 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1, |
| 10, |
| 11 |
| ], |
| "targets": [ |
| 0, |
| 3, |
| 6, |
| 8 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "04414cb7de8797d4:4caad29500000000", |
| "user": "anurag", |
| "queryText": "select a + b as ab, c, d, e from functional.allcomplextypes t, (select sum(item) a from t.int_array_col where item < 10) v1, (select count(f1) b from t.struct_map_col group by key) v2, (select avg(value) over(partition by key) c from t.map_map_col.value) v3, (select item d from t.int_array_col union all select value from t.int_map_col) v4, (select f21 e from t.complex_nested_struct_col.f2.f12 order by key limit 10) v5", |
| "endTime": 1687533710 |
| } |
| ---- QUERY |
| # Test relative collection table refs in inline views. |
| select a + b as ab, c, d, e from functional.allcomplextypes t, |
| (select sum(item) a from t.int_array_col |
| where item < 10) v1, |
| (select count(f1) b from t.struct_map_col |
| group by key) v2, |
| (select avg(value) over(partition by key) c from t.map_map_col.value) v3, |
| (select item d from t.int_array_col |
| union all |
| select value from t.int_map_col) v4, |
| (select f21 e from t.complex_nested_struct_col.f2.f12 order by key limit 10) v5 |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "5b43652748f22f00515ddf6ed699aec0", |
| "timestamp": 1687533715, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687533715, |
| "tableName": "lineage_test_db.lineage_test_view5", |
| "tableType": "view" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional.alltypes.id", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531267, |
| "tableName": "functional.alltypes", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PREDICATE" |
| } |
| ], |
| "queryId": "764a4986b0d631c2:a52f087b00000000", |
| "user": "anurag", |
| "queryText": "create view lineage_test_db.lineage_test_view5 as select id from functional.alltypes_view v where not exists (select 1 from functional.alltypes a where v.id = a.id)", |
| "endTime": 1687533715 |
| } |
| ---- QUERY |
| # IMPALA-4206: Test creating a view whose definition has a subquery and a view reference. |
| create view lineage_test_db.lineage_test_view5 as |
| select id from functional.alltypes_view v |
| where not exists (select 1 from functional.alltypes a where v.id = a.id) |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "f69515f6a2cfcd95c1914fc7da09834a", |
| "timestamp": 1687615426, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687615426, |
| "tableName": "lineage_test_db.ice_ctas", |
| "tableType": "iceberg" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_parquet.alltypes.id", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531213, |
| "tableName": "functional_parquet.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "string_col", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687615426, |
| "tableName": "lineage_test_db.ice_ctas", |
| "tableType": "iceberg" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_parquet.alltypes.string_col", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531213, |
| "tableName": "functional_parquet.alltypes", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "timestamp_col", |
| "id": 4, |
| "metadata": { |
| "tableCreateTime": 1687615426, |
| "tableName": "lineage_test_db.ice_ctas", |
| "tableType": "iceberg" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_parquet.alltypes.timestamp_col", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531213, |
| "tableName": "functional_parquet.alltypes", |
| "tableType": "hive" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| } |
| ], |
| "queryId": "db47d0dcf9aa4267:6cfbbef700000000", |
| "user": "boroknagyz", |
| "queryText": "create table lineage_test_db.ice_ctas partitioned by spec (truncate(1, string_col)) stored by iceberg as select id, string_col, timestamp_col from functional_parquet.alltypes", |
| "endTime": 1687615427 |
| } |
| ---- QUERY |
| # Execute lineage tests for Iceberg tables. |
| create table lineage_test_db.ice_ctas partitioned by spec (truncate(1, string_col)) stored by iceberg as |
| select id, string_col, timestamp_col from functional_parquet.alltypes |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "c37e1110bfc63dfe2e3e4c132a9def2f", |
| "timestamp": 1687615432, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687615420, |
| "tableName": "lineage_test_db.iceberg_partitioned", |
| "tableType": "iceberg" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_parquet.iceberg_partitioned.id", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531274, |
| "tableName": "functional_parquet.iceberg_partitioned", |
| "tableType": "iceberg" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "user", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687615420, |
| "tableName": "lineage_test_db.iceberg_partitioned", |
| "tableType": "iceberg" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_parquet.iceberg_partitioned.user", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531274, |
| "tableName": "functional_parquet.iceberg_partitioned", |
| "tableType": "iceberg" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "action", |
| "id": 4, |
| "metadata": { |
| "tableCreateTime": 1687615420, |
| "tableName": "lineage_test_db.iceberg_partitioned", |
| "tableType": "iceberg" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_parquet.iceberg_partitioned.action", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531274, |
| "tableName": "functional_parquet.iceberg_partitioned", |
| "tableType": "iceberg" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "event_time", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687615420, |
| "tableName": "lineage_test_db.iceberg_partitioned", |
| "tableType": "iceberg" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_parquet.iceberg_partitioned.event_time", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687531274, |
| "tableName": "functional_parquet.iceberg_partitioned", |
| "tableType": "iceberg" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 7 |
| ], |
| "targets": [ |
| 6 |
| ], |
| "edgeType": "PROJECTION" |
| } |
| ], |
| "queryId": "1942b41abdcfba11:1d8e573d00000000", |
| "user": "boroknagyz", |
| "queryText": "insert into lineage_test_db.iceberg_partitioned select * from functional_parquet.iceberg_partitioned", |
| "endTime": 1687615432 |
| } |
| ---- QUERY |
| insert into lineage_test_db.iceberg_partitioned |
| select * from functional_parquet.iceberg_partitioned |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "c37e1110bfc63dfe2e3e4c132a9def2f", |
| "timestamp": 1687615437, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687615420, |
| "tableName": "lineage_test_db.iceberg_partitioned", |
| "tableType": "iceberg" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_parquet.iceberg_partitioned.id", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687531274, |
| "tableName": "functional_parquet.iceberg_partitioned", |
| "tableType": "iceberg" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "user", |
| "id": 2, |
| "metadata": { |
| "tableCreateTime": 1687615420, |
| "tableName": "lineage_test_db.iceberg_partitioned", |
| "tableType": "iceberg" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_parquet.iceberg_partitioned.user", |
| "id": 3, |
| "metadata": { |
| "tableCreateTime": 1687531274, |
| "tableName": "functional_parquet.iceberg_partitioned", |
| "tableType": "iceberg" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "action", |
| "id": 4, |
| "metadata": { |
| "tableCreateTime": 1687615420, |
| "tableName": "lineage_test_db.iceberg_partitioned", |
| "tableType": "iceberg" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_parquet.iceberg_partitioned.action", |
| "id": 5, |
| "metadata": { |
| "tableCreateTime": 1687531274, |
| "tableName": "functional_parquet.iceberg_partitioned", |
| "tableType": "iceberg" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "event_time", |
| "id": 6, |
| "metadata": { |
| "tableCreateTime": 1687615420, |
| "tableName": "lineage_test_db.iceberg_partitioned", |
| "tableType": "iceberg" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "functional_parquet.iceberg_partitioned.event_time", |
| "id": 7, |
| "metadata": { |
| "tableCreateTime": 1687531274, |
| "tableName": "functional_parquet.iceberg_partitioned", |
| "tableType": "iceberg" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 3 |
| ], |
| "targets": [ |
| 2 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 5 |
| ], |
| "targets": [ |
| 4 |
| ], |
| "edgeType": "PROJECTION" |
| }, |
| { |
| "sources": [ |
| 7 |
| ], |
| "targets": [ |
| 6 |
| ], |
| "edgeType": "PROJECTION" |
| } |
| ], |
| "queryId": "5e4febca0b4e3922:96171f9700000000", |
| "user": "boroknagyz", |
| "queryText": "insert into lineage_test_db.iceberg_partitioned select * from functional_parquet.iceberg_partitioned", |
| "endTime": 1687615438 |
| } |
| ---- QUERY |
| insert into lineage_test_db.iceberg_partitioned |
| select * from functional_parquet.iceberg_partitioned |
| ==== |
| ---- LINEAGE |
| { |
| "hash": "2d727b3181ee6d49a388eec56f315806", |
| "timestamp": 1687615443, |
| "vertices": [ |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "id", |
| "id": 0, |
| "metadata": { |
| "tableCreateTime": 1687615420, |
| "tableName": "lineage_test_db.foo", |
| "tableType": "hive" |
| } |
| }, |
| { |
| "vertexType": "COLUMN", |
| "vertexId": "lineage_test_db.iceberg_partitioned.id", |
| "id": 1, |
| "metadata": { |
| "tableCreateTime": 1687615420, |
| "tableName": "lineage_test_db.iceberg_partitioned", |
| "tableType": "iceberg" |
| } |
| } |
| ], |
| "edges": [ |
| { |
| "sources": [ |
| 1 |
| ], |
| "targets": [ |
| 0 |
| ], |
| "edgeType": "PROJECTION" |
| } |
| ], |
| "queryId": "46450d3ee417fa15:9c00c0ce00000000", |
| "user": "boroknagyz", |
| "queryText": "insert into lineage_test_db.foo select id from lineage_test_db.iceberg_partitioned", |
| "endTime": 1687615443 |
| } |
| ---- QUERY |
| insert into lineage_test_db.foo |
| select id from lineage_test_db.iceberg_partitioned |
| ==== |