| # |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| """ |
| Linear Regression With SGD Example. |
| """ |
| from pyspark import SparkContext |
| # $example on$ |
| from pyspark.mllib.regression import LabeledPoint, LinearRegressionWithSGD, LinearRegressionModel |
| # $example off$ |
| |
| if __name__ == "__main__": |
| |
| sc = SparkContext(appName="PythonLinearRegressionWithSGDExample") |
| |
| # $example on$ |
| # Load and parse the data |
| def parsePoint(line): |
| values = [float(x) for x in line.replace(',', ' ').split(' ')] |
| return LabeledPoint(values[0], values[1:]) |
| |
| data = sc.textFile("data/mllib/ridge-data/lpsa.data") |
| parsedData = data.map(parsePoint) |
| |
| # Build the model |
| model = LinearRegressionWithSGD.train(parsedData, iterations=100, step=0.00000001) |
| |
| # Evaluate the model on training data |
| valuesAndPreds = parsedData.map(lambda p: (p.label, model.predict(p.features))) |
| MSE = valuesAndPreds \ |
| .map(lambda vp: (vp[0] - vp[1])**2) \ |
| .reduce(lambda x, y: x + y) / valuesAndPreds.count() |
| print("Mean Squared Error = " + str(MSE)) |
| |
| # Save and load model |
| model.save(sc, "target/tmp/pythonLinearRegressionWithSGDModel") |
| sameModel = LinearRegressionModel.load(sc, "target/tmp/pythonLinearRegressionWithSGDModel") |
| # $example off$ |