commit 453be73abb19eaf2448d1a63347afb8ca5911e1a
author Jongyoul Lee <jongyoul@gmail.com> Tue Jul 17 10:31:00 2018 +0900
committer Jongyoul Lee <jongyoul@apache.org> Tue Jul 17 11:23:37 2018 +0900
tree caaeada77f9a5d1a748ea261c52bdffc5bccaefc
parent c195b5609bb4dbb9fd037381ea58d1650cee92bf

[HOTFIX] Change Spark download url for spark 1.6.x

### What is this PR for?
Spark 1.6 was removed from CDN. We need to download it from apache archive.

### What type of PR is it?
[Hot Fix]

### Todos
* [x] - Change the location for downloading Spark 1.6.x

### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No

Author: Jongyoul Lee <jongyoul@gmail.com>

Closes #3075 from jongyoul/hotfix/change-spark-download-location and squashes the following commits:

87bfd2abf [Jongyoul Lee] Removed comments Reduce changes
a51d5b93b [Jongyoul Lee] Change download url for spark 1.6.x
9f63f8358 [Jongyoul Lee] Change download url for spark 1.6.x

zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkDownloadUtils.java

diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkDownloadUtils.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkDownloadUtils.java
index e027bb0..0455dd8 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkDownloadUtils.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkDownloadUtils.java
@@ -1,17 +1,16 @@
 package org.apache.zeppelin.interpreter;
 
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.URL;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class SparkDownloadUtils {
   private static Logger LOGGER = LoggerFactory.getLogger(SparkDownloadUtils.class);
@@ -34,18 +33,39 @@
       return targetSparkHomeFolder.getAbsolutePath();
     }
     // Try mirrors a few times until one succeeds
+    boolean downloaded = false;
     for (int i = 0; i < 3; i++) {
       try {
         String preferredMirror = IOUtils.toString(new URL("https://www.apache.org/dyn/closer.lua?preferred=true"));
         File downloadFile = new File(downloadFolder + "/spark-" + version + "-bin-hadoop2.6.tgz");
-        String downloadURL = preferredMirror + "/spark/spark-" + version + "/spark-" + version + "-bin-hadoop2.6.tgz";;
+        String downloadURL = preferredMirror + "/spark/spark-" + version + "/spark-" + version + "-bin-hadoop2.6.tgz";
         runShellCommand(new String[] {"wget", downloadURL, "-P", downloadFolder});
         runShellCommand(new String[]{"tar", "-xvf", downloadFile.getAbsolutePath(), "-C", downloadFolder});
+        downloaded = true;
         break;
       } catch (Exception e) {
         LOGGER.warn("Failed to download Spark", e);
       }
     }
+
+    // fallback to use apache archive
+    // https://archive.apache.org/dist/spark/spark-1.6.3/spark-1.6.3-bin-hadoop2.6.tgz
+    if (!downloaded) {
+      File downloadFile = new File(downloadFolder + "/spark-" + version + "-bin-hadoop2.6.tgz");
+      String downloadURL =
+          "https://archive.apache.org/dist/spark/spark-"
+              + version
+              + "/spark-"
+              + version
+              + "-bin-hadoop2.6.tgz";
+      try {
+        runShellCommand(new String[] {"wget", downloadURL, "-P", downloadFolder});
+        runShellCommand(
+            new String[] {"tar", "-xvf", downloadFile.getAbsolutePath(), "-C", downloadFolder});
+      } catch (Exception e) {
+        throw new RuntimeException("Fail to download spark " + version, e);
+      }
+    }
     return targetSparkHomeFolder.getAbsolutePath();
   }