| { |
| "paragraphs": [ |
| { |
| "text": "%md\n\n# Introduction\n\nThis is a tutorial for using spark [delta lake](https://delta.io/) in Zeppelin. You need to run the following paragraph first to load delta package.\n\n", |
| "user": "anonymous", |
| "dateUpdated": "2020-05-04 14:11:57.999", |
| "config": { |
| "colWidth": 12.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "markdown", |
| "editOnDblClick": true, |
| "completionKey": "TAB", |
| "completionSupport": false |
| }, |
| "editorMode": "ace/mode/markdown", |
| "editorHide": true, |
| "tableHide": false |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [ |
| { |
| "type": "HTML", |
| "data": "\u003cdiv class\u003d\"markdown-body\"\u003e\n\u003ch1\u003eIntroduction\u003c/h1\u003e\n\u003cp\u003eThis is a tutorial for using spark \u003ca href\u003d\"https://delta.io/\"\u003edelta lake\u003c/a\u003e in Zeppelin. You need to run the following paragraph first to load delta package.\u003c/p\u003e\n\n\u003c/div\u003e" |
| } |
| ] |
| }, |
| "apps": [], |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1588572279774_1507831415", |
| "id": "paragraph_1588572279774_1507831415", |
| "dateCreated": "2020-05-04 14:04:39.775", |
| "dateStarted": "2020-05-04 14:11:57.999", |
| "dateFinished": "2020-05-04 14:11:58.021", |
| "status": "FINISHED" |
| }, |
| { |
| "text": "%spark.conf\n\nspark.jars.packages io.delta:delta-core_2.11:0.6.0", |
| "user": "anonymous", |
| "dateUpdated": "2020-05-04 14:12:12.254", |
| "config": { |
| "colWidth": 12.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "text", |
| "editOnDblClick": false, |
| "completionKey": "TAB", |
| "completionSupport": true |
| }, |
| "editorMode": "ace/mode/text" |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [] |
| }, |
| "apps": [], |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1588147206215_1200788867", |
| "id": "paragraph_1588147206215_1200788867", |
| "dateCreated": "2020-04-29 16:00:06.215", |
| "dateStarted": "2020-04-29 16:10:33.429", |
| "dateFinished": "2020-04-29 16:10:33.434", |
| "status": "FINISHED" |
| }, |
| { |
| "title": "Create a table", |
| "text": "%spark\n\nval data \u003d spark.range(0, 5)\ndata.write.format(\"delta\").save(\"/tmp/delta-table\")\n", |
| "user": "anonymous", |
| "dateUpdated": "2020-04-29 16:13:31.957", |
| "config": { |
| "colWidth": 6.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "scala", |
| "editOnDblClick": false, |
| "completionKey": "TAB", |
| "completionSupport": true |
| }, |
| "editorMode": "ace/mode/scala", |
| "title": true |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [ |
| { |
| "type": "TEXT", |
| "data": "\u001b[1m\u001b[34mdata\u001b[0m: \u001b[1m\u001b[32morg.apache.spark.sql.Dataset[Long]\u001b[0m \u003d [id: bigint]\n" |
| } |
| ] |
| }, |
| "apps": [], |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1588147833426_1914590471", |
| "id": "paragraph_1588147833426_1914590471", |
| "dateCreated": "2020-04-29 16:10:33.426", |
| "dateStarted": "2020-04-29 16:11:45.197", |
| "dateFinished": "2020-04-29 16:11:49.694", |
| "status": "FINISHED" |
| }, |
| { |
| "title": "Read a table", |
| "text": "%spark\n\nval df \u003d spark.read.format(\"delta\").load(\"/tmp/delta-table\")\ndf.show()", |
| "user": "anonymous", |
| "dateUpdated": "2020-04-29 16:13:35.297", |
| "config": { |
| "colWidth": 6.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "scala", |
| "editOnDblClick": false, |
| "completionKey": "TAB", |
| "completionSupport": true |
| }, |
| "editorMode": "ace/mode/scala", |
| "title": true |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [ |
| { |
| "type": "TEXT", |
| "data": "+---+\n| id|\n+---+\n| 0|\n| 3|\n| 1|\n| 2|\n| 4|\n+---+\n\n\u001b[1m\u001b[34mdf\u001b[0m: \u001b[1m\u001b[32morg.apache.spark.sql.DataFrame\u001b[0m \u003d [id: bigint]\n" |
| } |
| ] |
| }, |
| "apps": [], |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1588147853461_1624743216", |
| "id": "paragraph_1588147853461_1624743216", |
| "dateCreated": "2020-04-29 16:10:53.462", |
| "dateStarted": "2020-04-29 16:11:55.302", |
| "dateFinished": "2020-04-29 16:11:56.658", |
| "status": "FINISHED" |
| }, |
| { |
| "title": "Overwrite", |
| "text": "%spark\n\nval data \u003d spark.range(5, 10)\ndata.write.format(\"delta\").mode(\"overwrite\").save(\"/tmp/delta-table\")\ndf.show()", |
| "user": "anonymous", |
| "dateUpdated": "2020-04-29 16:14:41.855", |
| "config": { |
| "colWidth": 6.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "scala", |
| "editOnDblClick": false, |
| "completionKey": "TAB", |
| "completionSupport": true |
| }, |
| "editorMode": "ace/mode/scala", |
| "title": true |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [ |
| { |
| "type": "TEXT", |
| "data": "+---+\n| id|\n+---+\n| 5|\n| 6|\n| 7|\n| 9|\n| 8|\n+---+\n\n\u001b[1m\u001b[34mdata\u001b[0m: \u001b[1m\u001b[32morg.apache.spark.sql.Dataset[Long]\u001b[0m \u003d [id: bigint]\n" |
| } |
| ] |
| }, |
| "apps": [], |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1588148062120_1790808564", |
| "id": "paragraph_1588148062120_1790808564", |
| "dateCreated": "2020-04-29 16:14:22.120", |
| "dateStarted": "2020-04-29 16:14:41.863", |
| "dateFinished": "2020-04-29 16:14:45.093", |
| "status": "FINISHED" |
| }, |
| { |
| "title": "Conditional update without overwrite", |
| "text": "%spark\n\nimport io.delta.tables._\nimport org.apache.spark.sql.functions._\n\nval deltaTable \u003d DeltaTable.forPath(\"/tmp/delta-table\")\n\n// Update every even value by adding 100 to it\ndeltaTable.update(\n condition \u003d expr(\"id % 2 \u003d\u003d 0\"),\n set \u003d Map(\"id\" -\u003e expr(\"id + 100\")))\n\n// Delete every even value\ndeltaTable.delete(condition \u003d expr(\"id % 2 \u003d\u003d 0\"))\n\n// Upsert (merge) new data\nval newData \u003d spark.range(0, 20).toDF\n\ndeltaTable.as(\"oldData\")\n .merge(\n newData.as(\"newData\"),\n \"oldData.id \u003d newData.id\")\n .whenMatched\n .update(Map(\"id\" -\u003e col(\"newData.id\")))\n .whenNotMatched\n .insert(Map(\"id\" -\u003e col(\"newData.id\")))\n .execute()\n\ndeltaTable.toDF.show()", |
| "user": "anonymous", |
| "dateUpdated": "2020-04-29 16:15:33.129", |
| "config": { |
| "colWidth": 6.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "scala", |
| "editOnDblClick": false, |
| "completionKey": "TAB", |
| "completionSupport": true |
| }, |
| "editorMode": "ace/mode/scala", |
| "title": true |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [ |
| { |
| "type": "TEXT", |
| "data": "+---+\n| id|\n+---+\n| 15|\n| 16|\n| 1|\n| 18|\n| 14|\n| 4|\n| 8|\n| 17|\n| 0|\n| 10|\n| 6|\n| 2|\n| 3|\n| 13|\n| 5|\n| 12|\n| 19|\n| 7|\n| 9|\n| 11|\n+---+\n\nimport io.delta.tables._\nimport org.apache.spark.sql.functions._\n\u001b[1m\u001b[34mdeltaTable\u001b[0m: \u001b[1m\u001b[32mio.delta.tables.DeltaTable\u001b[0m \u003d io.delta.tables.DeltaTable@355329ee\n\u001b[1m\u001b[34mnewData\u001b[0m: \u001b[1m\u001b[32morg.apache.spark.sql.DataFrame\u001b[0m \u003d [id: bigint]\n" |
| } |
| ] |
| }, |
| "apps": [], |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1588147954117_626957150", |
| "id": "paragraph_1588147954117_626957150", |
| "dateCreated": "2020-04-29 16:12:34.117", |
| "dateStarted": "2020-04-29 16:15:33.132", |
| "dateFinished": "2020-04-29 16:15:48.086", |
| "status": "FINISHED" |
| }, |
| { |
| "title": "Read older versions of data using time travel", |
| "text": "%spark\n\nval df \u003d spark.read.format(\"delta\").option(\"versionAsOf\", 0).load(\"/tmp/delta-table\")\ndf.show()", |
| "user": "anonymous", |
| "dateUpdated": "2020-04-29 16:16:04.935", |
| "config": { |
| "colWidth": 6.0, |
| "fontSize": 9.0, |
| "enabled": true, |
| "results": {}, |
| "editorSetting": { |
| "language": "scala", |
| "editOnDblClick": false, |
| "completionKey": "TAB", |
| "completionSupport": true |
| }, |
| "editorMode": "ace/mode/scala", |
| "title": true |
| }, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "results": { |
| "code": "SUCCESS", |
| "msg": [ |
| { |
| "type": "TEXT", |
| "data": "+---+\n| id|\n+---+\n| 0|\n| 3|\n| 1|\n| 2|\n| 4|\n+---+\n\n\u001b[1m\u001b[34mdf\u001b[0m: \u001b[1m\u001b[32morg.apache.spark.sql.DataFrame\u001b[0m \u003d [id: bigint]\n" |
| } |
| ] |
| }, |
| "apps": [], |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1588148133131_1770029903", |
| "id": "paragraph_1588148133131_1770029903", |
| "dateCreated": "2020-04-29 16:15:33.131", |
| "dateStarted": "2020-04-29 16:16:04.937", |
| "dateFinished": "2020-04-29 16:16:08.415", |
| "status": "FINISHED" |
| }, |
| { |
| "text": "%spark\n", |
| "user": "anonymous", |
| "dateUpdated": "2020-04-29 16:18:21.603", |
| "config": {}, |
| "settings": { |
| "params": {}, |
| "forms": {} |
| }, |
| "apps": [], |
| "progressUpdateIntervalMs": 500, |
| "jobName": "paragraph_1588148301603_1997345504", |
| "id": "paragraph_1588148301603_1997345504", |
| "dateCreated": "2020-04-29 16:18:21.603", |
| "status": "READY" |
| } |
| ], |
| "name": "6. Spark Delta Lake Tutorial", |
| "id": "2F8VDBMMT", |
| "defaultInterpreterGroup": "spark", |
| "version": "0.9.0-SNAPSHOT", |
| "noteParams": {}, |
| "noteForms": {}, |
| "angularObjects": {}, |
| "config": { |
| "isZeppelinNotebookCronEnable": false |
| }, |
| "info": {} |
| } |
