| { |
| "paragraphs": [ |
| { |
| "title": "Overview", |
| "text": "%md\n\nThis tutorial demonstrates how to use Flink do batch ETL via its batch sql + udf (scala, python \u0026 hive). Here\u0027s what we do in this tutorial\n\n* Download [bank](https://archive.ics.uci.edu/ml/datasets/bank+marketing) data via shell interpreter to local\n* Process the raw data via flink batch sql \u0026 scala udf which parse and clean the raw data\n* Write the structured and cleaned data to another flink table via sql\n", |
| "user": "anonymous", |
| "dateUpdated": "2021-07-26 05:45:49.528", |
| "progress": 0, |
| "config": { |
| "runOnSelectionChange": true, |
| "title": true, |
| "checkEmpty": true, |
| "colWidth": 12.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "markdown", |
| "editOnDblClick": true, |
| "completionKey": "TAB", |
| "completionSupport": false |
| }, |
| "editorMode": "ace/mode/markdown", |
| "editorHide": true, |
| "tableHide": false |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [ |
| { |
| "type": "HTML", |
| "data": "\u003cdiv class\u003d\"markdown-body\"\u003e\n\u003cp\u003eThis tutorial demonstrates how to use Flink do batch ETL via its batch sql + udf (scala, python \u0026amp; hive). Here\u0026rsquo;s what we do in this tutorial\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003eDownload \u003ca href\u003d\"https://archive.ics.uci.edu/ml/datasets/bank+marketing\"\u003ebank\u003c/a\u003e data via shell interpreter to local\u003c/li\u003e\n\u003cli\u003eProcess the raw data via flink batch sql \u0026amp; scala udf which parse and clean the raw data\u003c/li\u003e\n\u003cli\u003eWrite the structured and cleaned data to another flink table via sql\u003c/li\u003e\n\u003c/ul\u003e\n\n\u003c/div\u003e" |
| } |
| ] |
| }, |
| "apps": [], |
| "runtimeInfos": {}, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1579052523153_721650872", |
| "id": "paragraph_1579052523153_721650872", |
| "dateCreated": "2020-01-15 09:42:03.156", |
| "dateStarted": "2021-07-26 05:45:49.552", |
| "dateFinished": "2021-07-26 05:45:51.463", |
| "status": "FINISHED" |
| }, |
| { |
| "title": "Download bank data", |
| "text": "%sh\n\ncd /tmp\nrm -rf bank*\nwget https://archive.ics.uci.edu/ml/machine-learning-databases/00222/bank.zip\nunzip bank.zip\n# upload data to hdfs if you want to run it in yarn mode\n# hadoop fs -put /tmp/bank.csv /tmp/bank.csv\n", |
| "user": "anonymous", |
| "dateUpdated": "2021-07-26 05:45:51.555", |
| "progress": 0, |
| "config": { |
| "runOnSelectionChange": true, |
| "title": true, |
| "checkEmpty": true, |
| "colWidth": 12.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "sh", |
| "editOnDblClick": false, |
| "completionKey": "TAB", |
| "completionSupport": false |
| }, |
| "editorMode": "ace/mode/sh" |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [ |
| { |
| "type": "TEXT", |
| "data": "--2021-07-26 05:45:53-- https://archive.ics.uci.edu/ml/machine-learning-databases/00222/bank.zip\nResolving archive.ics.uci.edu (archive.ics.uci.edu)... 128.195.10.252\nConnecting to archive.ics.uci.edu (archive.ics.uci.edu)|128.195.10.252|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 579043 (565K) [application/x-httpd-php]\nSaving to: ‘bank.zip’\n\n 0K .......... .......... .......... .......... .......... 8% 136K 4s\n 50K .......... .......... .......... .......... .......... 17% 275K 3s\n 100K .......... .......... .......... .......... .......... 26% 40.4M 2s\n 150K .......... .......... .......... .......... .......... 35% 277K 1s\n 200K .......... .......... .......... .......... .......... 44% 35.2M 1s\n 250K .......... .......... .......... .......... .......... 53% 44.4M 1s\n 300K .......... .......... .......... .......... .......... 61% 280K 1s\n 350K .......... .......... .......... .......... .......... 70% 46.1M 0s\n 400K .......... .......... .......... .......... .......... 79% 33.1M 0s\n 450K .......... .......... .......... .......... .......... 88% 45.9M 0s\n 500K .......... .......... .......... .......... .......... 97% 43.9M 0s\n 550K .......... ..... 100% 68.9M\u003d0.9s\n\n2021-07-26 05:45:55 (617 KB/s) - ‘bank.zip’ saved [579043/579043]\n\nArchive: bank.zip\n inflating: bank-full.csv \n inflating: bank-names.txt \n inflating: bank.csv \n" |
| } |
| ] |
| }, |
| "apps": [], |
| "runtimeInfos": {}, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1578045094400_1030344935", |
| "id": "paragraph_1578045094400_1030344935", |
| "dateCreated": "2020-01-03 17:51:34.400", |
| "dateStarted": "2021-07-26 05:45:51.561", |
| "dateFinished": "2021-07-26 05:45:55.681", |
| "status": "FINISHED" |
| }, |
| { |
| "title": "Raw Data Preview", |
| "text": "%sh\n\nhead -n 10 /tmp/bank.csv", |
| "user": "anonymous", |
| "dateUpdated": "2021-07-26 05:45:55.766", |
| "progress": 0, |
| "config": { |
| "runOnSelectionChange": true, |
| "title": true, |
| "checkEmpty": true, |
| "colWidth": 12.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "sh", |
| "editOnDblClick": false, |
| "completionKey": "TAB", |
| "completionSupport": false |
| }, |
| "editorMode": "ace/mode/sh" |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [ |
| { |
| "type": "TEXT", |
| "data": "\"age\";\"job\";\"marital\";\"education\";\"default\";\"balance\";\"housing\";\"loan\";\"contact\";\"day\";\"month\";\"duration\";\"campaign\";\"pdays\";\"previous\";\"poutcome\";\"y\"\n30;\"unemployed\";\"married\";\"primary\";\"no\";1787;\"no\";\"no\";\"cellular\";19;\"oct\";79;1;-1;0;\"unknown\";\"no\"\n33;\"services\";\"married\";\"secondary\";\"no\";4789;\"yes\";\"yes\";\"cellular\";11;\"may\";220;1;339;4;\"failure\";\"no\"\n35;\"management\";\"single\";\"tertiary\";\"no\";1350;\"yes\";\"no\";\"cellular\";16;\"apr\";185;1;330;1;\"failure\";\"no\"\n30;\"management\";\"married\";\"tertiary\";\"no\";1476;\"yes\";\"yes\";\"unknown\";3;\"jun\";199;4;-1;0;\"unknown\";\"no\"\n59;\"blue-collar\";\"married\";\"secondary\";\"no\";0;\"yes\";\"no\";\"unknown\";5;\"may\";226;1;-1;0;\"unknown\";\"no\"\n35;\"management\";\"single\";\"tertiary\";\"no\";747;\"no\";\"no\";\"cellular\";23;\"feb\";141;2;176;3;\"failure\";\"no\"\n36;\"self-employed\";\"married\";\"tertiary\";\"no\";307;\"yes\";\"no\";\"cellular\";14;\"may\";341;1;330;2;\"other\";\"no\"\n39;\"technician\";\"married\";\"secondary\";\"no\";147;\"yes\";\"no\";\"cellular\";6;\"may\";151;2;-1;0;\"unknown\";\"no\"\n41;\"entrepreneur\";\"married\";\"tertiary\";\"no\";221;\"yes\";\"no\";\"unknown\";14;\"may\";57;2;-1;0;\"unknown\";\"no\"\n" |
| } |
| ] |
| }, |
| "apps": [], |
| "runtimeInfos": {}, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1579053112778_2010129053", |
| "id": "paragraph_1579053112778_2010129053", |
| "dateCreated": "2020-01-15 09:51:52.779", |
| "dateStarted": "2021-07-26 05:45:55.770", |
| "dateFinished": "2021-07-26 05:45:55.789", |
| "status": "FINISHED" |
| }, |
| { |
| "title": "Define source table which represents the raw data", |
| "text": "%flink.bsql\n\nDROP TABLE IF EXISTS bank_raw;\nCREATE TABLE bank_raw (\n content STRING\n) WITH (\n\u0027format.field-delimiter\u0027\u003d\u0027\\n\u0027,\n\u0027connector.type\u0027\u003d\u0027filesystem\u0027,\n\u0027format.derive-schema\u0027\u003d\u0027true\u0027,\n\u0027connector.path\u0027\u003d\u0027/tmp/bank.csv\u0027,\n\u0027format.type\u0027\u003d\u0027csv\u0027\n);", |
| "user": "anonymous", |
| "dateUpdated": "2021-07-26 05:45:55.870", |
| "progress": 0, |
| "config": { |
| "colWidth": 6.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "sql", |
| "editOnDblClick": false, |
| "completionKey": "TAB", |
| "completionSupport": true |
| }, |
| "editorMode": "ace/mode/sql", |
| "runOnSelectionChange": true, |
| "title": true, |
| "checkEmpty": true |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [ |
| { |
| "type": "TEXT", |
| "data": "Table has been dropped.\nTable has been created.\n" |
| } |
| ] |
| }, |
| "apps": [], |
| "runtimeInfos": {}, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1578044954921_-1188487356", |
| "id": "paragraph_1578044954921_-1188487356", |
| "dateCreated": "2020-01-03 17:49:14.921", |
| "dateStarted": "2021-07-26 05:45:55.874", |
| "dateFinished": "2021-07-26 05:46:09.686", |
| "status": "FINISHED" |
| }, |
| { |
| "title": "Define sink table which represents the cleaned data", |
| "text": "%flink.bsql\n\nDROP TABLE IF EXISTS bank;\nCREATE TABLE bank (\n age int, \n job string,\n marital string,\n education string,\n `default` string,\n balance string,\n housing string,\n loan string,\n contact string, \n `day` string,\n `month` string,\n duration int,\n campaign int,\n pdays int,\n previous int,\n poutcome string,\n y string\n) WITH (\n\u0027format.field-delimiter\u0027\u003d\u0027,\u0027,\n\u0027connector.type\u0027\u003d\u0027filesystem\u0027,\n\u0027format.derive-schema\u0027\u003d\u0027true\u0027,\n\u0027connector.path\u0027\u003d\u0027/tmp/bank_cleaned\u0027,\n\u0027format.type\u0027\u003d\u0027csv\u0027\n);", |
| "user": "anonymous", |
| "dateUpdated": "2021-07-26 05:46:09.691", |
| "progress": 0, |
| "config": { |
| "runOnSelectionChange": true, |
| "title": true, |
| "checkEmpty": true, |
| "colWidth": 6.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "sql", |
| "editOnDblClick": false, |
| "completionKey": "TAB", |
| "completionSupport": true |
| }, |
| "editorMode": "ace/mode/sql" |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [ |
| { |
| "type": "TEXT", |
| "data": "Table has been dropped.\nTable has been created.\n" |
| } |
| ] |
| }, |
| "apps": [], |
| "runtimeInfos": {}, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1578045204379_-1427374232", |
| "id": "paragraph_1578045204379_-1427374232", |
| "dateCreated": "2020-01-03 17:53:24.379", |
| "dateStarted": "2021-07-26 05:46:09.694", |
| "dateFinished": "2021-07-26 05:46:09.894", |
| "status": "FINISHED" |
| }, |
| { |
| "title": "Show tables", |
| "text": "%flink.bsql\n\nshow tables", |
| "user": "anonymous", |
| "dateUpdated": "2021-07-26 05:46:09.994", |
| "progress": 0, |
| "config": { |
| "colWidth": 12.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": { |
| "0": { |
| "graph": { |
| "mode": "table", |
| "height": 300.0, |
| "optionOpen": false, |
| "setting": { |
| "table": { |
| "tableGridState": {}, |
| "tableColumnTypeState": { |
| "names": { |
| "table": "string" |
| }, |
| "updated": false |
| }, |
| "tableOptionSpecHash": "[{\"name\":\"useFilter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable filter for columns\"},{\"name\":\"showPagination\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable pagination for better navigation\"},{\"name\":\"showAggregationFooter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable a footer for displaying aggregated values\"}]", |
| "tableOptionValue": { |
| "useFilter": false, |
| "showPagination": false, |
| "showAggregationFooter": false |
| }, |
| "updated": false, |
| "initialized": false |
| } |
| }, |
| "commonSetting": {} |
| } |
| } |
| }, |
| "editorSetting": { |
| "language": "sql", |
| "editOnDblClick": false, |
| "completionKey": "TAB", |
| "completionSupport": true |
| }, |
| "editorMode": "ace/mode/sql", |
| "title": true |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [ |
| { |
| "type": "TABLE", |
| "data": "table\nbank\nbank_raw\n" |
| } |
| ] |
| }, |
| "apps": [], |
| "runtimeInfos": {}, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1587958743728_1444404682", |
| "id": "paragraph_1587958743728_1444404682", |
| "dateCreated": "2020-04-27 11:39:03.728", |
| "dateStarted": "2021-07-26 05:46:09.998", |
| "dateFinished": "2021-07-26 05:46:10.214", |
| "status": "FINISHED" |
| }, |
| { |
| "title": "Define UDTF ParseFunction to parse the raw data", |
| "text": "%flink\n\nimport org.apache.flink.api.java.typeutils.RowTypeInfo\nimport org.apache.flink.api.common.typeinfo.Types\nimport org.apache.flink.api.java.typeutils._\nimport org.apache.flink.api.scala.typeutils._\nimport org.apache.flink.api.scala._\n\nclass Person(val age:Int, val job: String, val marital: String, val education: String, val default: String, val balance: String, val housing: String, val loan: String, val contact: String, val day: String, val month: String, val duration: Int, val campaign: Int, val pdays: Int, val previous: Int, val poutcome: String, val y: String)\n\nclass ParseFunction extends TableFunction[Row] {\n def eval(line: String) {\n val tokens \u003d line.split(\";\")\n // parse the line\n if (!line.startsWith(\"\\\"age\\\"\")) {\n collect(Row.of(new Integer(tokens(0).toInt), normalize(tokens(1)), normalize(tokens(2)), normalize(tokens(3)), normalize(tokens(4)), normalize(tokens(5)), normalize(tokens(6)), normalize(tokens(7)), normalize(tokens(8)), normalize(tokens(9)), normalize(tokens(10)), new Integer(tokens(11).toInt), new Integer(tokens(12).toInt), \n new Integer(tokens(13).toInt), new Integer(tokens(14).toInt), normalize(tokens(15)), normalize(tokens(16))))\n }\n }\n \n override def getResultType() \u003d {\n val cls \u003d classOf[Person]\n Types.ROW(Types.INT, Types.STRING, Types.STRING, Types.STRING,Types.STRING,Types.STRING,Types.STRING,Types.STRING,Types.STRING,Types.STRING,Types.STRING,\n Types.INT, Types.INT, Types.INT, Types.INT, Types.STRING, Types.STRING)\n }\n\n // remove the quote\n private def normalize(token: String) \u003d {\n if (token.startsWith(\"\\\"\")) {\n token.substring(1, token.length - 1)\n } else {\n token\n }\n }\n}\n\nbtenv.registerFunction(\"parse\", new ParseFunction())", |
| "user": "anonymous", |
| "dateUpdated": "2021-07-26 05:46:10.297", |
| "progress": 0, |
| "config": { |
| "runOnSelectionChange": true, |
| "title": true, |
| "checkEmpty": true, |
| "colWidth": 12.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "scala", |
| "editOnDblClick": false, |
| "completionKey": "TAB", |
| "completionSupport": true |
| }, |
| "editorMode": "ace/mode/scala" |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [ |
| { |
| "type": "TEXT", |
| "data": "import org.apache.flink.api.java.typeutils.RowTypeInfo\nimport org.apache.flink.api.common.typeinfo.Types\nimport org.apache.flink.api.java.typeutils._\nimport org.apache.flink.api.scala.typeutils._\nimport org.apache.flink.api.scala._\ndefined class Person\ndefined class ParseFunction\n" |
| } |
| ] |
| }, |
| "apps": [], |
| "runtimeInfos": {}, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1578888628353_1621411444", |
| "id": "paragraph_1578888628353_1621411444", |
| "dateCreated": "2020-01-13 12:10:28.359", |
| "dateStarted": "2021-07-26 05:46:10.300", |
| "dateFinished": "2021-07-26 05:46:13.351", |
| "status": "FINISHED" |
| }, |
| { |
| "title": "Clean data", |
| "text": "%sh\n\nrm -rf /tmp/bank_cleaned\n#hadoop fs -rmr /tmp/bank_cleaned", |
| "user": "anonymous", |
| "dateUpdated": "2021-07-26 05:46:13.404", |
| "progress": 0, |
| "config": { |
| "runOnSelectionChange": true, |
| "title": true, |
| "checkEmpty": true, |
| "colWidth": 12.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "sh", |
| "editOnDblClick": false, |
| "completionKey": "TAB", |
| "completionSupport": false |
| }, |
| "editorMode": "ace/mode/sh" |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [] |
| }, |
| "apps": [], |
| "runtimeInfos": {}, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1579061020460_-113987164", |
| "id": "paragraph_1579061020460_-113987164", |
| "dateCreated": "2020-01-15 12:03:40.468", |
| "dateStarted": "2021-07-26 05:46:13.414", |
| "dateFinished": "2021-07-26 05:46:13.450", |
| "status": "FINISHED" |
| }, |
| { |
| "title": "Parse the data and write it into sink table", |
| "text": "%flink.bsql\n\ninsert into bank select T.* from bank_raw, LATERAL TABLE(parse(content)) as T(age, job, marital, education, `default`, balance, housing, loan, contact, `day`, `month`, duration, campaign, pdays, previous, poutcome, y) ", |
| "user": "anonymous", |
| "dateUpdated": "2021-07-26 05:46:13.507", |
| "progress": 0, |
| "config": { |
| "runOnSelectionChange": true, |
| "title": true, |
| "checkEmpty": true, |
| "colWidth": 12.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "sql", |
| "editOnDblClick": false, |
| "completionKey": "TAB", |
| "completionSupport": true |
| }, |
| "editorMode": "ace/mode/sql" |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [ |
| { |
| "type": "TEXT", |
| "data": "Insertion successfully.\n" |
| } |
| ] |
| }, |
| "apps": [], |
| "runtimeInfos": { |
| "jobUrl": { |
| "propertyName": "jobUrl", |
| "label": "FLINK JOB", |
| "tooltip": "View in Flink web UI", |
| "group": "flink", |
| "values": [ |
| { |
| "jobUrl": "http://localhost:8083#/job/3cf7019160025b634b6bcb5a127d08af" |
| } |
| ], |
| "interpreterSettingId": "flink" |
| } |
| }, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1578669828368_-1923137601", |
| "id": "paragraph_1578669828368_-1923137601", |
| "dateCreated": "2020-01-10 23:23:48.368", |
| "dateStarted": "2021-07-26 05:46:13.510", |
| "dateFinished": "2021-07-26 05:46:19.576", |
| "status": "FINISHED" |
| }, |
| { |
| "title": "Preview output data", |
| "text": "%flink.bsql\n\nselect * from bank limit 10\n", |
| "user": "anonymous", |
| "dateUpdated": "2021-07-26 05:46:19.614", |
| "progress": 0, |
| "config": { |
| "colWidth": 12.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": { |
| "0": { |
| "graph": { |
| "mode": "table", |
| "height": 300.0, |
| "optionOpen": false, |
| "setting": { |
| "table": { |
| "tableGridState": {}, |
| "tableColumnTypeState": { |
| "names": { |
| "age": "string", |
| "job": "string", |
| "marital": "string", |
| "education": "string", |
| "default": "string", |
| "balance": "string", |
| "housing": "string", |
| "loan": "string", |
| "contact": "string", |
| "day": "string", |
| "month": "string", |
| "duration": "string", |
| "campaign": "string", |
| "pdays": "string", |
| "previous": "string", |
| "poutcome": "string", |
| "y": "string" |
| }, |
| "updated": false |
| }, |
| "tableOptionSpecHash": "[{\"name\":\"useFilter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable filter for columns\"},{\"name\":\"showPagination\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable pagination for better navigation\"},{\"name\":\"showAggregationFooter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable a footer for displaying aggregated values\"}]", |
| "tableOptionValue": { |
| "useFilter": false, |
| "showPagination": false, |
| "showAggregationFooter": false |
| }, |
| "updated": false, |
| "initialized": false |
| } |
| }, |
| "commonSetting": {} |
| } |
| } |
| }, |
| "editorSetting": { |
| "language": "sql", |
| "editOnDblClick": false, |
| "completionKey": "TAB", |
| "completionSupport": true |
| }, |
| "editorMode": "ace/mode/sql", |
| "title": true, |
| "runOnSelectionChange": true, |
| "checkEmpty": true |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [ |
| { |
| "type": "TABLE", |
| "data": "age\tjob\tmarital\teducation\tdefault\tbalance\thousing\tloan\tcontact\tday\tmonth\tduration\tcampaign\tpdays\tprevious\tpoutcome\ty\n30\tunemployed\tmarried\tprimary\tno\t1787\tno\tno\tcellular\t19\toct\t79\t1\t-1\t0\tunknown\tno\n33\tservices\tmarried\tsecondary\tno\t4789\tyes\tyes\tcellular\t11\tmay\t220\t1\t339\t4\tfailure\tno\n35\tmanagement\tsingle\ttertiary\tno\t1350\tyes\tno\tcellular\t16\tapr\t185\t1\t330\t1\tfailure\tno\n30\tmanagement\tmarried\ttertiary\tno\t1476\tyes\tyes\tunknown\t3\tjun\t199\t4\t-1\t0\tunknown\tno\n59\tblue-collar\tmarried\tsecondary\tno\t0\tyes\tno\tunknown\t5\tmay\t226\t1\t-1\t0\tunknown\tno\n35\tmanagement\tsingle\ttertiary\tno\t747\tno\tno\tcellular\t23\tfeb\t141\t2\t176\t3\tfailure\tno\n36\tself-employed\tmarried\ttertiary\tno\t307\tyes\tno\tcellular\t14\tmay\t341\t1\t330\t2\tother\tno\n39\ttechnician\tmarried\tsecondary\tno\t147\tyes\tno\tcellular\t6\tmay\t151\t2\t-1\t0\tunknown\tno\n41\tentrepreneur\tmarried\ttertiary\tno\t221\tyes\tno\tunknown\t14\tmay\t57\t2\t-1\t0\tunknown\tno\n43\tservices\tmarried\tprimary\tno\t-88\tyes\tyes\tcellular\t17\tapr\t313\t1\t147\t2\tfailure\tno\n" |
| } |
| ] |
| }, |
| "apps": [], |
| "runtimeInfos": { |
| "jobUrl": { |
| "propertyName": "jobUrl", |
| "label": "FLINK JOB", |
| "tooltip": "View in Flink web UI", |
| "group": "flink", |
| "values": [ |
| { |
| "jobUrl": "http://localhost:8083#/job/74ef1d3bc865544ce75b0a598ab416c3" |
| } |
| ], |
| "interpreterSettingId": "flink" |
| } |
| }, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1578068480238_-1678045273", |
| "id": "paragraph_1578068480238_-1678045273", |
| "dateCreated": "2020-01-04 00:21:20.267", |
| "dateStarted": "2021-07-26 05:46:19.631", |
| "dateFinished": "2021-07-26 05:46:23.857", |
| "status": "FINISHED" |
| }, |
| { |
| "title": "Display table via z.show", |
| "text": "%flink\n\nval table \u003d btenv.sqlQuery(\"select * from bank limit 10\")\nz.show(table)", |
| "user": "anonymous", |
| "dateUpdated": "2021-07-26 05:46:23.921", |
| "progress": 100, |
| "config": { |
| "colWidth": 12.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": { |
| "1": { |
| "graph": { |
| "mode": "table", |
| "height": 300.0, |
| "optionOpen": false, |
| "setting": { |
| "table": { |
| "tableGridState": {}, |
| "tableColumnTypeState": { |
| "names": { |
| "age": "string", |
| "job": "string", |
| "marital": "string", |
| "education": "string", |
| "default": "string", |
| "balance": "string", |
| "housing": "string", |
| "loan": "string", |
| "contact": "string", |
| "day": "string", |
| "month": "string", |
| "duration": "string", |
| "campaign": "string", |
| "pdays": "string", |
| "previous": "string", |
| "poutcome": "string", |
| "y": "string" |
| }, |
| "updated": false |
| }, |
| "tableOptionSpecHash": "[{\"name\":\"useFilter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable filter for columns\"},{\"name\":\"showPagination\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable pagination for better navigation\"},{\"name\":\"showAggregationFooter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable a footer for displaying aggregated values\"}]", |
| "tableOptionValue": { |
| "useFilter": false, |
| "showPagination": false, |
| "showAggregationFooter": false |
| }, |
| "updated": false, |
| "initialized": false |
| } |
| }, |
| "commonSetting": {} |
| } |
| } |
| }, |
| "editorSetting": { |
| "language": "scala", |
| "editOnDblClick": false, |
| "completionKey": "TAB", |
| "completionSupport": true |
| }, |
| "editorMode": "ace/mode/scala", |
| "title": true |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [ |
| { |
| "type": "TEXT", |
| "data": "\u001b[1m\u001b[34mtable\u001b[0m: \u001b[1m\u001b[32morg.apache.flink.table.api.Table\u001b[0m \u003d UnnamedTable$0\n" |
| }, |
| { |
| "type": "TABLE", |
| "data": "age\tjob\tmarital\teducation\tdefault\tbalance\thousing\tloan\tcontact\tday\tmonth\tduration\tcampaign\tpdays\tprevious\tpoutcome\ty\n30\tunemployed\tmarried\tprimary\tno\t1787\tno\tno\tcellular\t19\toct\t79\t1\t-1\t0\tunknown\tno\n33\tservices\tmarried\tsecondary\tno\t4789\tyes\tyes\tcellular\t11\tmay\t220\t1\t339\t4\tfailure\tno\n35\tmanagement\tsingle\ttertiary\tno\t1350\tyes\tno\tcellular\t16\tapr\t185\t1\t330\t1\tfailure\tno\n30\tmanagement\tmarried\ttertiary\tno\t1476\tyes\tyes\tunknown\t3\tjun\t199\t4\t-1\t0\tunknown\tno\n59\tblue-collar\tmarried\tsecondary\tno\t0\tyes\tno\tunknown\t5\tmay\t226\t1\t-1\t0\tunknown\tno\n35\tmanagement\tsingle\ttertiary\tno\t747\tno\tno\tcellular\t23\tfeb\t141\t2\t176\t3\tfailure\tno\n36\tself-employed\tmarried\ttertiary\tno\t307\tyes\tno\tcellular\t14\tmay\t341\t1\t330\t2\tother\tno\n39\ttechnician\tmarried\tsecondary\tno\t147\tyes\tno\tcellular\t6\tmay\t151\t2\t-1\t0\tunknown\tno\n41\tentrepreneur\tmarried\ttertiary\tno\t221\tyes\tno\tunknown\t14\tmay\t57\t2\t-1\t0\tunknown\tno\n43\tservices\tmarried\tprimary\tno\t-88\tyes\tyes\tcellular\t17\tapr\t313\t1\t147\t2\tfailure\tno\n" |
| } |
| ] |
| }, |
| "apps": [], |
| "runtimeInfos": { |
| "jobUrl": { |
| "propertyName": "jobUrl", |
| "label": "FLINK JOB", |
| "tooltip": "View in Flink web UI", |
| "group": "flink", |
| "values": [ |
| { |
| "jobUrl": "http://localhost:8083#/job/4edf4d7107556b387d17d1f3d090ae20" |
| } |
| ], |
| "interpreterSettingId": "flink" |
| } |
| }, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1579061037737_-1577558456", |
| "id": "paragraph_1579061037737_-1577558456", |
| "dateCreated": "2020-01-15 12:03:57.737", |
| "dateStarted": "2021-07-26 05:46:23.925", |
| "dateFinished": "2021-07-26 05:46:28.599", |
| "status": "FINISHED" |
| }, |
| { |
| "title": "Display table via z.show in PyFlink", |
| "text": "%flink.pyflink\n\ntable \u003d bt_env.sql_query(\"select * from bank limit 10\")\nz.show(table)", |
| "user": "anonymous", |
| "dateUpdated": "2021-07-26 05:46:28.627", |
| "progress": 100, |
| "config": { |
| "colWidth": 12.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": { |
| "0": { |
| "graph": { |
| "mode": "table", |
| "height": 300.0, |
| "optionOpen": false, |
| "setting": { |
| "table": { |
| "tableGridState": {}, |
| "tableColumnTypeState": { |
| "names": { |
| "age": "string", |
| "job": "string", |
| "marital": "string", |
| "education": "string", |
| "default": "string", |
| "balance": "string", |
| "housing": "string", |
| "loan": "string", |
| "contact": "string", |
| "day": "string", |
| "month": "string", |
| "duration": "string", |
| "campaign": "string", |
| "pdays": "string", |
| "previous": "string", |
| "poutcome": "string", |
| "y": "string" |
| }, |
| "updated": false |
| }, |
| "tableOptionSpecHash": "[{\"name\":\"useFilter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable filter for columns\"},{\"name\":\"showPagination\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable pagination for better navigation\"},{\"name\":\"showAggregationFooter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable a footer for displaying aggregated values\"}]", |
| "tableOptionValue": { |
| "useFilter": false, |
| "showPagination": false, |
| "showAggregationFooter": false |
| }, |
| "updated": false, |
| "initialized": false |
| } |
| }, |
| "commonSetting": {} |
| } |
| } |
| }, |
| "editorSetting": { |
| "language": "python", |
| "editOnDblClick": false, |
| "completionKey": "TAB", |
| "completionSupport": true |
| }, |
| "editorMode": "ace/mode/python", |
| "title": true |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [ |
| { |
| "type": "TABLE", |
| "data": "age\tjob\tmarital\teducation\tdefault\tbalance\thousing\tloan\tcontact\tday\tmonth\tduration\tcampaign\tpdays\tprevious\tpoutcome\ty\n30\tunemployed\tmarried\tprimary\tno\t1787\tno\tno\tcellular\t19\toct\t79\t1\t-1\t0\tunknown\tno\n33\tservices\tmarried\tsecondary\tno\t4789\tyes\tyes\tcellular\t11\tmay\t220\t1\t339\t4\tfailure\tno\n35\tmanagement\tsingle\ttertiary\tno\t1350\tyes\tno\tcellular\t16\tapr\t185\t1\t330\t1\tfailure\tno\n30\tmanagement\tmarried\ttertiary\tno\t1476\tyes\tyes\tunknown\t3\tjun\t199\t4\t-1\t0\tunknown\tno\n59\tblue-collar\tmarried\tsecondary\tno\t0\tyes\tno\tunknown\t5\tmay\t226\t1\t-1\t0\tunknown\tno\n35\tmanagement\tsingle\ttertiary\tno\t747\tno\tno\tcellular\t23\tfeb\t141\t2\t176\t3\tfailure\tno\n36\tself-employed\tmarried\ttertiary\tno\t307\tyes\tno\tcellular\t14\tmay\t341\t1\t330\t2\tother\tno\n39\ttechnician\tmarried\tsecondary\tno\t147\tyes\tno\tcellular\t6\tmay\t151\t2\t-1\t0\tunknown\tno\n41\tentrepreneur\tmarried\ttertiary\tno\t221\tyes\tno\tunknown\t14\tmay\t57\t2\t-1\t0\tunknown\tno\n43\tservices\tmarried\tprimary\tno\t-88\tyes\tyes\tcellular\t17\tapr\t313\t1\t147\t2\tfailure\tno\n" |
| } |
| ] |
| }, |
| "apps": [], |
| "runtimeInfos": { |
| "jobUrl": { |
| "propertyName": "jobUrl", |
| "label": "FLINK JOB", |
| "tooltip": "View in Flink web UI", |
| "group": "flink", |
| "values": [ |
| { |
| "jobUrl": "http://localhost:8083#/job/d9cb2c76334d123e4e8c2a0a3a4eb8d5" |
| } |
| ], |
| "interpreterSettingId": "flink" |
| } |
| }, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1588690392097_1159956807", |
| "id": "paragraph_1588690392097_1159956807", |
| "dateCreated": "2020-05-05 22:53:12.097", |
| "dateStarted": "2021-07-26 05:46:28.633", |
| "dateFinished": "2021-07-26 05:46:36.757", |
| "status": "FINISHED" |
| }, |
| { |
| "text": "%flink.pyflink\n", |
| "user": "anonymous", |
| "dateUpdated": "2021-07-26 05:46:36.837", |
| "progress": 0, |
| "config": { |
| "colWidth": 12.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "python", |
| "editOnDblClick": false, |
| "completionKey": "TAB", |
| "completionSupport": true |
| }, |
| "editorMode": "ace/mode/python" |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [] |
| }, |
| "apps": [], |
| "runtimeInfos": {}, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1588693027989_1331448600", |
| "id": "paragraph_1588693027989_1331448600", |
| "dateCreated": "2020-05-05 23:37:07.989", |
| "dateStarted": "2021-07-26 05:46:36.841", |
| "dateFinished": "2021-07-26 05:46:37.763", |
| "status": "FINISHED" |
| } |
| ], |
| "name": "6. Batch ETL", |
| "id": "2EW19CSPA", |
| "defaultInterpreterGroup": "flink", |
| "version": "0.9.0-SNAPSHOT", |
| "noteParams": {}, |
| "noteForms": {}, |
| "angularObjects": {}, |
| "config": { |
| "isZeppelinNotebookCronEnable": false |
| }, |
| "info": { |
| "isRunning": true |
| } |
| } |
