﻿{"paragraphs":[{"title":"Init Spark","text":"%INTERPRETER_NAME\nfrom pyspark.sql import SQLContext\nfrom pyspark.sql import DataFrame\nfrom pyspark.sql import Row\nfrom pyspark.sql.types import *\nimport pandas as pd\nimport StringIO\nimport matplotlib.pyplot as plt\nhc = sc._jsc.hadoopConfiguration()\nhc.set(\"hive.execution.engine\", \"mr\")","dateUpdated":"2018-01-03T14:13:24+0000","config":{"editorSetting":{"language":"text","editOnDblClick":false},"colWidth":6,"editorMode":"ace/mode/text","title":true,"results":{},"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1514975882432_-581784801","id":"20170116-185159_818793480","dateCreated":"2018-01-03T10:38:02+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:2275"},{"title":"Define functions","text":"%INTERPRETER_NAME\r\nimport csv\r\n\r\nworking_storage = \"WORKING_STORAGE\"\r\noutput_directory = 'zeppelin/py2'\r\nprotocol_name = 'PROTOCOL_NAME'\r\n\r\ndef parseCsv(csvStr):\r\n    f = StringIO.StringIO(csvStr)\r\n    reader = csv.reader(f, delimiter=',')\r\n    row = reader.next()\r\n    return row\r\n\r\ndef bucket_path(part_path):\r\n    return '{}://{}/zeppelin_dataset/{}'.format(protocol_name, working_storage, part_path)\r\n\r\ndef full_path(part_path):\r\n    return '{}://{}/{}/{}'.format(protocol_name, working_storage, output_directory, part_path)\r\n\r\nscsv = '\"02Q\",\"Titan Airways\"'\r\nrow = parseCsv(scsv)\r\nprint row[0]\r\nprint row[1]","dateUpdated":"2018-01-03T14:14:20+0000","config":{"editorSetting":{"language":"text","editOnDblClick":false},"colWidth":6,"editorMode":"ace/mode/text","title":true,"results":{},"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1514975882433_-582169550","id":"20170116-193003_477574066","dateCreated":"2018-01-03T10:38:02+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2276"},{"title":"Parse and convert Carrier data to parquet","text":"%INTERPRETER_NAME\r\n\r\ncarriersHeader = 'Code,Description'\r\ncarriersText = sc.textFile(bucket_path(\"carriers.csv\")).filter(lambda x: x != carriersHeader)\r\ncarriers = carriersText.map(lambda s: parseCsv(s)) \\\r\n    .map(lambda s: Row(code=s[0], description=s[1])).cache().toDF()\r\ncarriers.write.mode(\"overwrite\").parquet(full_path(\"carriers\"))    \r\nsqlContext.registerDataFrameAsTable(carriers, \"carriers\")\r\ncarriers.limit(20).toPandas()","dateUpdated":"2018-01-03T14:13:25+0000","config":{"editorSetting":{"language":"text","editOnDblClick":false},"colWidth":12,"editorMode":"ace/mode/text","title":true,"results":{},"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1514975882433_-582169550","id":"20170116-193845_1563104751","dateCreated":"2018-01-03T10:38:02+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2277"},{"title":"Parse and convert to parquet Airport data","text":"%INTERPRETER_NAME\n\nairportsHeader= '\"iata\",\"airport\",\"city\",\"state\",\"country\",\"lat\",\"long\"'\nairports = sc.textFile(bucket_path(\"airports.csv\")) \\\n    .filter(lambda x: x != airportsHeader) \\\n    .map(lambda s: parseCsv(s)) \\\n    .map(lambda p: Row(iata=p[0], \\\n                       airport=p[1], \\\n                       city=p[2], \\\n                       state=p[3], \\\n                       country=p[4], \\\n                       lat=float(p[5]), \\\n                       longt=float(p[6])) \\\n        ).cache().toDF()\nairports.write.mode(\"overwrite\").parquet(full_path(\"airports\"))    \nsqlContext.registerDataFrameAsTable(airports, \"airports\")\nairports.limit(20).toPandas()","dateUpdated":"2018-01-03T14:13:26+0000","config":{"editorSetting":{"language":"text","editOnDblClick":false},"colWidth":12,"editorMode":"ace/mode/text","title":true,"results":{},"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1514975882434_-581015303","id":"20170116-194608_52076348","dateCreated":"2018-01-03T10:38:02+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2278"},{"title":"Parse and convert Flights data to parquet","text":"%INTERPRETER_NAME\n\nflightsHeader = 'Year,Month,DayofMonth,DayOfWeek,DepTime,CRSDepTime,ArrTime,CRSArrTime,UniqueCarrier,FlightNum,TailNum,ActualElapsedTime,CRSElapsedTime,AirTime,ArrDelay,DepDelay,Origin,Dest,Distance,TaxiIn,TaxiOut,Cancelled,CancellationCode,Diverted,CarrierDelay,WeatherDelay,NASDelay,SecurityDelay,LateAircraftDelay'\nflights = sc.textFile(bucket_path(\"2008.csv.bz2\")) \\\n    .filter(lambda x: x!= flightsHeader) \\\n    .map(lambda s: parseCsv(s)) \\\n    .map(lambda p: Row(Year=int(p[0]), \\\n                       Month=int(p[1]), \\\n                       DayofMonth=int(p[2]), \\\n                       DayOfWeek=int(p[3]), \\\n                       DepTime=p[4], \\\n                       CRSDepTime=p[5], \\\n                       ArrTime=p[6], \\\n                       CRSArrTime=p[7], \\\n                       UniqueCarrier=p[8], \\\n                       FlightNum=p[9], \\\n                       TailNum=p[10], \\\n                       ActualElapsedTime=p[11], \\\n                       CRSElapsedTime=p[12], \\\n                       AirTime=p[13], \\\n                       ArrDelay=int(p[14].replace(\"NA\", \"0\")), \\\n                       DepDelay=int(p[15].replace(\"NA\", \"0\")), \\\n                       Origin=p[16], \\\n                       Dest=p[17], \\\n                       Distance=long(p[18]), \\\n                       TaxiIn=p[19], \\\n                       TaxiOut=p[20], \\\n                       Cancelled=p[21], \\\n                       CancellationCode=p[22], \\\n                       Diverted=p[23], \\\n                       CarrierDelay=int(p[24].replace(\"NA\", \"0\")), \\\n                                              CarrierDelayStr=p[24], \\\n                       WeatherDelay=int(p[25].replace(\"NA\", \"0\")), \\\n                                              WeatherDelayStr=p[25], \\\n                       NASDelay=int(p[26].replace(\"NA\", \"0\")), \\\n                       SecurityDelay=int(p[27].replace(\"NA\", \"0\")), \\\n                       LateAircraftDelay=int(p[28].replace(\"NA\", \"0\")))) \\\n         .toDF()\n\nflights.write.mode(\"ignore\").parquet(full_path(\"flights\"))\nsqlContext.registerDataFrameAsTable(flights, \"flights\")\nflights.limit(10).toPandas()[[\"ArrDelay\",\"CarrierDelay\",\"CarrierDelayStr\",\"WeatherDelay\",\"WeatherDelayStr\",\"Distance\"]]","dateUpdated":"2018-01-03T14:13:31+0000","config":{"editorSetting":{"language":"text","editOnDblClick":false},"colWidth":12,"editorMode":"ace/mode/text","title":true,"results":{},"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1514975882434_-581015303","id":"20170116-194514_1558643741","dateCreated":"2018-01-03T10:38:02+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2279"},{"text":"","dateUpdated":"2018-01-03T10:38:02+0000","config":{"editorSetting":{"language":"scala"},"colWidth":12,"editorMode":"ace/mode/scala","results":{},"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1514975882435_-581400052","id":"20170116-200314_1592643376","dateCreated":"2018-01-03T10:38:02+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2280"}],"name":"Python 2 data preparation","id":"2D3UHPA39","angularObjects":{"2C6RJRBD2:shared_process":[],"2C6RJRBD1:shared_process":[]},"config":{"looknfeel":"default","personalizedMode":"false"},"info":{}}