| { |
| "paragraphs": [ |
| { |
| "text": "%md\n\n# Introduction\n\nThis note is to demonstrate how to do machine learning in flink. Here we use [Alink](https://github.com/alibaba/Alink/). \nWe use logics regression to do classification task. We use the same data as other tutorials [bank](https://archive.ics.uci.edu/ml/datasets/bank+marketing).\n", |
| "user": "anonymous", |
| "dateUpdated": "2021-01-25 14:30:35.800", |
| "progress": 0, |
| "config": { |
| "colWidth": 12.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "markdown", |
| "editOnDblClick": true, |
| "completionKey": "TAB", |
| "completionSupport": false |
| }, |
| "editorMode": "ace/mode/markdown", |
| "editorHide": true, |
| "tableHide": false |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "apps": [], |
| "runtimeInfos": {}, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1588147625869_1181490991", |
| "id": "paragraph_1588147625869_1181490991", |
| "dateCreated": "2020-04-29 16:07:05.869", |
| "dateStarted": "2021-01-25 14:30:35.800", |
| "dateFinished": "2021-01-25 14:30:35.807", |
| "status": "FINISHED" |
| }, |
| { |
| "text": "%flink.pyflink\n\nimport pyflink\nfrom pyflink.dataset import ExecutionEnvironment\nfrom pyflink.datastream import StreamExecutionEnvironment\nfrom pyalink.alink.env import useCustomEnv\nmlenv \u003d useCustomEnv(gateway,\n b_env,bt_env_2, s_env, st_env_2)\nfrom pyalink.alink import *\n\nt \u003d bt_env_2.from_elements([(1, 2), (2, 5), (3, 1)], [\u0027a\u0027, \u0027b\u0027])\nsource \u003d TableSourceBatchOp(t)\nsource.print()", |
| "user": "anonymous", |
| "dateUpdated": "2020-04-27 13:48:06.523", |
| "progress": 0, |
| "config": { |
| "editorMode": "ace/mode/python", |
| "editorHide": false, |
| "colWidth": 6.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "python", |
| "editOnDblClick": false, |
| "completionSupport": true, |
| "completionKey": "TAB" |
| } |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "apps": [], |
| "runtimeInfos": {}, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1583768872979_-705704388", |
| "id": "20200309-234752_541772059", |
| "dateCreated": "2020-03-09 23:47:52.979", |
| "dateStarted": "2020-04-27 13:48:06.528", |
| "dateFinished": "2020-04-27 13:48:29.623", |
| "status": "FINISHED" |
| }, |
| { |
| "text": "%flink.pyflink\n\n\ntest_data_path \u003d \"/tmp/bank.csv\"\nfull_data_path \u003d \"/tmp/bank-full.csv\"\nschema_str \u003d \"age int, job string, marital string, education string, default string, balance string, housing string, loan string, contact string, day string, month string, duration int, campaign int, pdays int, previous int, poutcome string, y string\"\n\ntest_data \u003d CsvSourceBatchOp() \\\n .setFilePath(test_data_path) \\\n .setSchemaStr(schema_str) \\\n .setIgnoreFirstLine(True) \\\n .setFieldDelimiter(\";\")\n \nfull_data \u003d CsvSourceBatchOp() \\\n .setFilePath(full_data_path) \\\n .setSchemaStr(schema_str) \\\n .setIgnoreFirstLine(True) \\\n .setFieldDelimiter(\";\")\n \ntrain_set \u003d UnionAllBatchOp().linkFrom(\n full_data.filter(\"y\u003d\u0027yes\u0027\"),\n full_data.filter(\"y\u003d\u0027no\u0027\").sample(0.25)\n \n)", |
| "user": "anonymous", |
| "dateUpdated": "2020-04-29 16:10:03.433", |
| "progress": 0, |
| "config": { |
| "editorMode": "ace/mode/python", |
| "editorHide": false, |
| "colWidth": 6.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "python", |
| "editOnDblClick": false, |
| "completionSupport": true, |
| "completionKey": "TAB" |
| } |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "apps": [], |
| "runtimeInfos": {}, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1583768872982_-1596320538", |
| "id": "20200309-234752_30368548", |
| "dateCreated": "2020-03-09 23:47:52.982", |
| "dateStarted": "2020-04-27 13:48:33.820", |
| "dateFinished": "2020-04-27 13:48:34.969", |
| "status": "FINISHED" |
| }, |
| { |
| "text": "%flink.pyflink\n\n\ncategoricalColNames \u003d [\"job\", \"marital\", \"education\", \"default\",\n \"balance\", \"housing\", \"loan\", \"contact\", \"poutcome\" ]\nnumerialColNames \u003d [\"age\", \"duration\", \"campaign\", \"pdays\",\n \"previous\"]\nlabelColName \u003d \"y\"\n\nonehot \u003d OneHotEncoder().setSelectedCols(categoricalColNames) \\\n .setOutputCols([\"output\"])\nassembler \u003d VectorAssembler().setSelectedCols([\"output\"] + numerialColNames) \\\n .setOutputCol(\"vec\")\npipeline \u003d Pipeline().add(onehot).add(assembler)", |
| "user": "anonymous", |
| "dateUpdated": "2020-04-27 13:48:37.221", |
| "progress": 0, |
| "config": { |
| "editorMode": "ace/mode/python", |
| "editorHide": false, |
| "colWidth": 6.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "python", |
| "editOnDblClick": false, |
| "completionSupport": true, |
| "completionKey": "TAB" |
| } |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "apps": [], |
| "runtimeInfos": {}, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1583768872983_-260771927", |
| "id": "20200309-234752_1624274051", |
| "dateCreated": "2020-03-09 23:47:52.983", |
| "dateStarted": "2020-04-27 13:48:37.242", |
| "dateFinished": "2020-04-27 13:48:37.911", |
| "status": "FINISHED" |
| }, |
| { |
| "text": "%flink.pyflink\n\n\nlogistic \u003d LogisticRegression().setVectorCol(\"vec\").setLabelCol(labelColName) \\\n .setPredictionCol(\"pred\").setPredictionDetailCol(\"detail\")\nmodel \u003d pipeline.add(logistic).fit(train_set)\n\npredict \u003d model.transform(test_data)\n\nmetrics \u003d EvalBinaryClassBatchOp().setLabelCol(labelColName) \\\n .setPredictionDetailCol(\"detail\").linkFrom(predict).collectMetrics()\n \n \n ", |
| "user": "anonymous", |
| "dateUpdated": "2020-04-27 13:48:39.232", |
| "progress": 0, |
| "config": { |
| "editorMode": "ace/mode/python", |
| "editorHide": false, |
| "colWidth": 6.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "python", |
| "editOnDblClick": false, |
| "completionSupport": true, |
| "completionKey": "TAB" |
| } |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "apps": [], |
| "runtimeInfos": {}, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1583768872983_-2044786839", |
| "id": "20200309-234752_452787710", |
| "dateCreated": "2020-03-09 23:47:52.983", |
| "dateStarted": "2020-04-27 13:48:39.249", |
| "dateFinished": "2020-04-27 13:48:50.221", |
| "status": "FINISHED" |
| }, |
| { |
| "text": "%flink.pyflink\n\nprint(\"AUC:\", metrics.getAuc())\nprint(\"KS:\", metrics.getKs())\nprint(\"PRC:\", metrics.getPrc())\nprint(\"Precision:\", metrics.getPrecision())\nprint(\"Recall:\", metrics.getRecall())\nprint(\"F1:\", metrics.getF1())\nprint(\"ConfusionMatrix:\", metrics.getConfusionMatrix())\nprint(\"LabelArray:\", metrics.getLabelArray())\nprint(\"LogLoss:\", metrics.getLogLoss())\nprint(\"TotalSamples:\", metrics.getTotalSamples())\nprint(\"ActualLabelProportion:\", metrics.getActualLabelProportion())\nprint(\"ActualLabelFrequency:\", metrics.getActualLabelFrequency())\nprint(\"Accuracy:\", metrics.getAccuracy())\nprint(\"Kappa:\", metrics.getKappa())", |
| "user": "anonymous", |
| "dateUpdated": "2020-04-27 13:48:51.441", |
| "progress": 0, |
| "config": { |
| "editorMode": "ace/mode/python", |
| "editorHide": false, |
| "colWidth": 6.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "python", |
| "editOnDblClick": false, |
| "completionSupport": true, |
| "completionKey": "TAB" |
| } |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "apps": [], |
| "runtimeInfos": {}, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1583768872984_389357281", |
| "id": "20200309-234752_912989550", |
| "dateCreated": "2020-03-09 23:47:52.984", |
| "dateStarted": "2020-04-27 13:48:51.451", |
| "dateFinished": "2020-04-27 13:48:52.073", |
| "status": "FINISHED" |
| }, |
| { |
| "text": "%flink.pyflink\n\ndf \u003d predict.filter(\"y\u003c\u003epred\").firstN(300).collectToDataframe()\n\nz.show(df)", |
| "user": "anonymous", |
| "dateUpdated": "2020-04-27 13:48:54.873", |
| "progress": 0, |
| "config": { |
| "editorMode": "ace/mode/python", |
| "editorHide": false, |
| "colWidth": 12.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": { |
| "0": { |
| "graph": { |
| "mode": "table", |
| "height": 300.0, |
| "optionOpen": false, |
| "setting": { |
| "table": { |
| "tableGridState": {}, |
| "tableColumnTypeState": { |
| "names": { |
| "age": "string", |
| "job": "string", |
| "marital": "string", |
| "education": "string", |
| "default": "string", |
| "balance": "string", |
| "housing": "string", |
| "loan": "string", |
| "contact": "string", |
| "day": "string", |
| "month": "string", |
| "duration": "string", |
| "campaign": "string", |
| "pdays": "string", |
| "previous": "string", |
| "poutcome": "string", |
| "y": "string", |
| "output": "string", |
| "vec": "string", |
| "pred": "string", |
| "detail": "string" |
| }, |
| "updated": false |
| }, |
| "tableOptionSpecHash": "[{\"name\":\"useFilter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable filter for columns\"},{\"name\":\"showPagination\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable pagination for better navigation\"},{\"name\":\"showAggregationFooter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable a footer for displaying aggregated values\"}]", |
| "tableOptionValue": { |
| "useFilter": false, |
| "showPagination": false, |
| "showAggregationFooter": false |
| }, |
| "updated": false, |
| "initialized": false |
| } |
| }, |
| "commonSetting": {} |
| } |
| } |
| }, |
| "editorSetting": { |
| "language": "python", |
| "editOnDblClick": false, |
| "completionSupport": true, |
| "completionKey": "TAB" |
| } |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "apps": [], |
| "runtimeInfos": {}, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1583768872984_-1459808170", |
| "id": "20200309-234752_1664292334", |
| "dateCreated": "2020-03-09 23:47:52.984", |
| "dateStarted": "2020-04-27 13:48:54.934", |
| "dateFinished": "2020-04-27 13:49:01.921", |
| "status": "FINISHED" |
| }, |
| { |
| "text": "%flink.pyflink\n", |
| "user": "anonymous", |
| "dateUpdated": "2020-03-10 11:04:48.771", |
| "progress": 0, |
| "config": { |
| "editorMode": "ace/mode/python", |
| "editorHide": false, |
| "colWidth": 12.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "python", |
| "editOnDblClick": false, |
| "completionSupport": true, |
| "completionKey": "TAB" |
| } |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "apps": [], |
| "runtimeInfos": {}, |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1583768872984_-575920971", |
| "id": "20200309-234752_517801851", |
| "dateCreated": "2020-03-09 23:47:52.984", |
| "dateStarted": "2020-03-10 00:25:39.912", |
| "dateFinished": "2020-03-10 00:25:39.981", |
| "status": "FINISHED" |
| } |
| ], |
| "name": "8. Logistic Regression (Alink)", |
| "id": "2F4HJNWVN", |
| "defaultInterpreterGroup": "flink", |
| "version": "0.9.0-SNAPSHOT", |
| "noteParams": {}, |
| "noteForms": {}, |
| "angularObjects": {}, |
| "config": { |
| "isZeppelinNotebookCronEnable": false |
| }, |
| "info": {} |
| } |
