[PIO-188] Update the build matrix on travis (#489)
diff --git a/.travis.yml b/.travis.yml
index abd4ab0..2a756e9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -47,131 +47,60 @@
matrix:
- BUILD_TYPE=Unit
METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
- PIO_SCALA_VERSION=2.11.8
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=S3
+ PIO_ELASTICSEARCH_VERSION=1.7.3
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3
+ PIO_ELASTICSEARCH_VERSION=5.6.9
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3
+ PIO_ELASTICSEARCH_VERSION=6.4.2
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
+ PIO_HBASE_VERSION=1.2.6
+
+ - BUILD_TYPE=Integration
+ METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
+ PIO_SCALA_VERSION=2.11.12
PIO_SPARK_VERSION=2.0.2
PIO_HADOOP_VERSION=2.6.5
- BUILD_TYPE=Integration
METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
- PIO_SCALA_VERSION=2.11.8
- PIO_SPARK_VERSION=2.0.2
+ PIO_SCALA_VERSION=2.11.12
+ PIO_SPARK_VERSION=2.1.3
PIO_HADOOP_VERSION=2.6.5
- BUILD_TYPE=Integration
- METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
- PIO_SCALA_VERSION=2.11.8
- PIO_SPARK_VERSION=2.0.2
- PIO_ELASTICSEARCH_VERSION=5.6.9
+ METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
+ PIO_SCALA_VERSION=2.11.12
+ PIO_SPARK_VERSION=2.2.2
PIO_HADOOP_VERSION=2.6.5
- BUILD_TYPE=Integration
- METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
- PIO_SCALA_VERSION=2.11.8
- PIO_SPARK_VERSION=2.0.2
- PIO_ELASTICSEARCH_VERSION=5.6.9
- PIO_HADOOP_VERSION=2.6.5
- - BUILD_TYPE=Integration
- METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
- PIO_SCALA_VERSION=2.11.8
- PIO_SPARK_VERSION=2.0.2
- PIO_ELASTICSEARCH_VERSION=1.7.3
- PIO_HADOOP_VERSION=2.6.5
- - BUILD_TYPE=Integration
- METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
- PIO_SCALA_VERSION=2.11.8
- PIO_SPARK_VERSION=2.0.2
- PIO_ELASTICSEARCH_VERSION=1.7.3
- PIO_HADOOP_VERSION=2.6.5
- - BUILD_TYPE=Integration
- METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3
- PIO_SCALA_VERSION=2.11.8
- PIO_SPARK_VERSION=2.0.2
- PIO_ELASTICSEARCH_VERSION=5.6.9
+ METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
+ PIO_SCALA_VERSION=2.11.12
+ PIO_SPARK_VERSION=2.3.2
PIO_HADOOP_VERSION=2.6.5
- - BUILD_TYPE=Unit
- METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
- PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.1.1
- BUILD_TYPE=Integration
METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.1.1
- - BUILD_TYPE=Integration
- METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
- PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.1.1
- PIO_ELASTICSEARCH_VERSION=5.6.9
- - BUILD_TYPE=Integration
- METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
- PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.1.1
- PIO_ELASTICSEARCH_VERSION=5.6.9
- - BUILD_TYPE=Integration
- METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
- PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.1.1
- PIO_ELASTICSEARCH_VERSION=1.7.3
- - BUILD_TYPE=Integration
- METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
- PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.1.1
- PIO_ELASTICSEARCH_VERSION=1.7.3
- - BUILD_TYPE=Integration
- METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3
- PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.1.1
- PIO_ELASTICSEARCH_VERSION=5.6.9
-
- - BUILD_TYPE=Unit
- METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
- PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.2.0
+ PIO_SPARK_VERSION=2.0.2
+ PIO_HADOOP_VERSION=2.7.3
- BUILD_TYPE=Integration
METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.2.0
- - BUILD_TYPE=Integration
- METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
- PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.2.0
- PIO_ELASTICSEARCH_VERSION=1.7.3
- - BUILD_TYPE=Integration
- METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
- PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.2.0
- PIO_ELASTICSEARCH_VERSION=5.6.9
- - BUILD_TYPE=Integration
- METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3
- PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.2.0
- PIO_ELASTICSEARCH_VERSION=5.6.9
-
- - BUILD_TYPE=Unit
- METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
- PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.3.1
- PIO_HADOOP_VERSION=2.7.7
+ PIO_SPARK_VERSION=2.1.3
+ PIO_HADOOP_VERSION=2.7.3
- BUILD_TYPE=Integration
METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.3.1
- PIO_HADOOP_VERSION=2.7.7
+ PIO_SPARK_VERSION=2.2.2
+ PIO_HADOOP_VERSION=2.7.3
- BUILD_TYPE=Integration
- METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
+ METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.3.1
- PIO_ELASTICSEARCH_VERSION=1.7.3
- PIO_HADOOP_VERSION=2.7.7
- - BUILD_TYPE=Integration
- METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
- PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.3.1
- PIO_ELASTICSEARCH_VERSION=5.6.9
- PIO_HADOOP_VERSION=2.7.7
- - BUILD_TYPE=Integration
- METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3
- PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.3.1
- PIO_ELASTICSEARCH_VERSION=5.6.9
- PIO_HADOOP_VERSION=2.7.7
+ PIO_SPARK_VERSION=2.3.2
+ PIO_HADOOP_VERSION=2.7.3
- BUILD_TYPE=LicenseCheck
diff --git a/build.sbt b/build.sbt
index 94739ff..5027fd9 100644
--- a/build.sbt
+++ b/build.sbt
@@ -56,7 +56,7 @@
"-Xlint:deprecation", "-Xlint:unchecked")
// Ignore differentiation of Spark patch levels
-sparkVersion in ThisBuild := sys.props.getOrElse("spark.version", "2.1.2")
+sparkVersion in ThisBuild := sys.props.getOrElse("spark.version", "2.1.3")
sparkBinaryVersion in ThisBuild := binaryVersion(sparkVersion.value)
@@ -314,6 +314,7 @@
printBuildInfo := {
println(s"PIO_SCALA_VERSION=${scalaVersion.value}")
println(s"PIO_SPARK_VERSION=${sparkVersion.value}")
- println(s"PIO_ELASTICSEARCH_VERSION=${elasticsearchVersion.value}")
println(s"PIO_HADOOP_VERSION=${hadoopVersion.value}")
+ println(s"PIO_ELASTICSEARCH_VERSION=${elasticsearchVersion.value}")
+ println(s"PIO_HBASE_VERSION=${hbaseVersion.value}")
}
diff --git a/conf/pio-vendors.sh b/conf/pio-vendors.sh
index 0489e85..f657a7c 100644
--- a/conf/pio-vendors.sh
+++ b/conf/pio-vendors.sh
@@ -20,21 +20,25 @@
# `source conf/set_build_profile.sh $BUILD_PROFILE` to get the proper versions
if [ -z "$PIO_SCALA_VERSION" ]; then
- PIO_SCALA_VERSION="2.11.8"
+ PIO_SCALA_VERSION="2.11.12"
fi
if [ -z "$PIO_SPARK_VERSION" ]; then
- PIO_SPARK_VERSION="2.1.1"
+ PIO_SPARK_VERSION="2.1.3"
fi
if [ -z "$PIO_HADOOP_VERSION" ]; then
- PIO_HADOOP_VERSION="2.6.5"
+ PIO_HADOOP_VERSION="2.7.3"
fi
if [ -z "$PIO_ELASTICSEARCH_VERSION" ]; then
PIO_ELASTICSEARCH_VERSION="5.6.9"
fi
+if [ -z "$PIO_HBASE_VERSION" ]; then
+ PIO_HBASE_VERSION="1.2.6"
+fi
+
ES_MAJOR=`echo $PIO_ELASTICSEARCH_VERSION | awk -F. '{print $1}'`
if [ "$ES_MAJOR" = "1" ]; then
@@ -42,9 +46,12 @@
export ES_TAG="1"
else
export ES_IMAGE="docker.elastic.co/elasticsearch/elasticsearch"
- export ES_TAG="5.6.9"
+ export ES_TAG="$PIO_ELASTICSEARCH_VERSION"
fi
+HBASE_MAJOR=`echo $PIO_HBASE_VERSION | awk -F. '{print $1 "." $2}'`
+export HBASE_TAG="$HBASE_MAJOR"
+
PGSQL_JAR=postgresql-9.4-1204.jdbc41.jar
PGSQL_DOWNLOAD=https://jdbc.postgresql.org/download/${PGSQL_JAR}
diff --git a/tests/build_docker.sh b/tests/build_docker.sh
index 5879a58..0adf3b0 100755
--- a/tests/build_docker.sh
+++ b/tests/build_docker.sh
@@ -41,7 +41,8 @@
-Dscala.version=$PIO_SCALA_VERSION \
-Dspark.version=$PIO_SPARK_VERSION \
-Dhadoop.version=$PIO_HADOOP_VERSION \
- -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION
+ -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION \
+ -Dhbase.version=$PIO_HBASE_VERSION
sbt/sbt clean storage/clean
assembly_folder=assembly/src/universal/lib
diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
index e0eda34..0c196f1 100644
--- a/tests/docker-compose.yml
+++ b/tests/docker-compose.yml
@@ -21,7 +21,7 @@
- xpack.security.enabled=false
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
hbase:
- image: harisekhon/hbase:1.0
+ image: harisekhon/hbase:${HBASE_TAG}
postgres:
image: postgres:9
environment:
diff --git a/tests/unit.sh b/tests/unit.sh
index 1421dce..a9c84fc 100755
--- a/tests/unit.sh
+++ b/tests/unit.sh
@@ -32,13 +32,15 @@
-Dscala.version=$PIO_SCALA_VERSION \
-Dspark.version=$PIO_SPARK_VERSION \
-Dhadoop.version=$PIO_HADOOP_VERSION \
- -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION
+ -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION \
+ -Dhbase.version=$PIO_HBASE_VERSION
# Run all unit tests
sbt/sbt dataJdbc/compile test storage/test \
-Dscala.version=$PIO_SCALA_VERSION \
-Dspark.version=$PIO_SPARK_VERSION \
-Dhadoop.version=$PIO_HADOOP_VERSION \
- -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION
+ -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION \
+ -Dhbase.version=$PIO_HBASE_VERSION
popd
