[PIO-161] Spark 2.3 support
Closes #461
diff --git a/.travis.yml b/.travis.yml
index d2a8d2f..49cbb65 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -132,6 +132,35 @@
PIO_SPARK_VERSION=2.2.0
PIO_ELASTICSEARCH_VERSION=5.5.2
+ - BUILD_TYPE=Unit
+ METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
+ PIO_SCALA_VERSION=2.11.12
+ PIO_SPARK_VERSION=2.3.1
+ PIO_HADOOP_VERSION=2.7.7
+ - BUILD_TYPE=Integration
+ METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
+ PIO_SCALA_VERSION=2.11.12
+ PIO_SPARK_VERSION=2.3.1
+ PIO_HADOOP_VERSION=2.7.7
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
+ PIO_SCALA_VERSION=2.11.12
+ PIO_SPARK_VERSION=2.3.1
+ PIO_ELASTICSEARCH_VERSION=1.7.3
+ PIO_HADOOP_VERSION=2.7.7
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
+ PIO_SCALA_VERSION=2.11.12
+ PIO_SPARK_VERSION=2.3.1
+ PIO_ELASTICSEARCH_VERSION=5.5.2
+ PIO_HADOOP_VERSION=2.7.7
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3
+ PIO_SCALA_VERSION=2.11.12
+ PIO_SPARK_VERSION=2.3.1
+ PIO_ELASTICSEARCH_VERSION=5.5.2
+ PIO_HADOOP_VERSION=2.7.7
+
- BUILD_TYPE=LicenseCheck
before_install:
diff --git a/build.sbt b/build.sbt
index 0243610..c753f76 100644
--- a/build.sbt
+++ b/build.sbt
@@ -46,6 +46,10 @@
"2.2" -> Map(
"akka" -> "2.4.17",
"hadoop" -> "2.7.3",
+ "json4s" -> "3.2.11"),
+ "2.3" -> Map(
+ "akka" -> "2.4.17",
+ "hadoop" -> "2.7.3",
"json4s" -> "3.2.11")))
name := "apache-predictionio-parent"
diff --git a/conf/pio-vendors.sh b/conf/pio-vendors.sh
index 6420207..162372f 100644
--- a/conf/pio-vendors.sh
+++ b/conf/pio-vendors.sh
@@ -51,7 +51,8 @@
HADOOP_MAJOR=`echo $PIO_HADOOP_VERSION | awk -F. '{print $1 "." $2}'`
SPARK_DIR=spark-${PIO_SPARK_VERSION}-bin-hadoop${HADOOP_MAJOR}
SPARK_ARCHIVE=${SPARK_DIR}.tgz
-SPARK_DOWNLOAD=http://d3kbcqa49mib13.cloudfront.net/${SPARK_ARCHIVE}
+SPARK_DOWNLOAD_MIRROR=https://www.apache.org/dyn/closer.lua\?action=download\&filename=spark/spark-${PIO_SPARK_VERSION}/${SPARK_ARCHIVE}
+SPARK_DOWNLOAD_ARCHIVE=https://archive.apache.org/dist/spark/spark-${PIO_SPARK_VERSION}/${SPARK_ARCHIVE}
# ELASTICSEARCH_DOWNLOAD
# 5.x https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-${PIO_ELASTICSEARCH_VERSION}.tar.gz
# 1.x https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${PIO_ELASTICSEARCH_VERSION}.tar.gz
diff --git a/tests/build_docker.sh b/tests/build_docker.sh
index b17a304..2c6f8f8 100755
--- a/tests/build_docker.sh
+++ b/tests/build_docker.sh
@@ -1,4 +1,4 @@
-#!/bin/bash -ex
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -28,7 +28,10 @@
mv ${PGSQL_JAR} $DIR/docker-files/
fi
if [ ! -f $DIR/docker-files/${SPARK_ARCHIVE} ]; then
- wget $SPARK_DOWNLOAD
+ curl -fLo $SPARK_ARCHIVE $SPARK_DOWNLOAD_MIRROR
+ if [[ $? -ne 0 ]]; then
+ curl -fLo $SPARK_ARCHIVE $SPARK_DOWNLOAD_ARCHIVE
+ fi
mv $SPARK_ARCHIVE $DIR/docker-files/
fi
