Merge branch 'release/0.13.0' into livedoc
diff --git a/.travis.yml b/.travis.yml
index 65f4166..49cbb65 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -132,22 +132,51 @@
PIO_SPARK_VERSION=2.2.0
PIO_ELASTICSEARCH_VERSION=5.5.2
+ - BUILD_TYPE=Unit
+ METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
+ PIO_SCALA_VERSION=2.11.12
+ PIO_SPARK_VERSION=2.3.1
+ PIO_HADOOP_VERSION=2.7.7
+ - BUILD_TYPE=Integration
+ METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
+ PIO_SCALA_VERSION=2.11.12
+ PIO_SPARK_VERSION=2.3.1
+ PIO_HADOOP_VERSION=2.7.7
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
+ PIO_SCALA_VERSION=2.11.12
+ PIO_SPARK_VERSION=2.3.1
+ PIO_ELASTICSEARCH_VERSION=1.7.3
+ PIO_HADOOP_VERSION=2.7.7
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
+ PIO_SCALA_VERSION=2.11.12
+ PIO_SPARK_VERSION=2.3.1
+ PIO_ELASTICSEARCH_VERSION=5.5.2
+ PIO_HADOOP_VERSION=2.7.7
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3
+ PIO_SCALA_VERSION=2.11.12
+ PIO_SPARK_VERSION=2.3.1
+ PIO_ELASTICSEARCH_VERSION=5.5.2
+ PIO_HADOOP_VERSION=2.7.7
+
- BUILD_TYPE=LicenseCheck
before_install:
- unset SBT_OPTS JVM_OPTS
- sudo rm /usr/local/bin/docker-compose
- - curl -L https://github.com/docker/compose/releases/download/1.11.1/docker-compose-`uname -s`-`uname -m` > docker-compose
+ - travis_retry curl -L https://github.com/docker/compose/releases/download/1.11.1/docker-compose-`uname -s`-`uname -m` > docker-compose
- chmod +x docker-compose
- sudo mv docker-compose /usr/local/bin
before_script:
- sudo sysctl -w vm.max_map_count=262144
- docker-compose -v
- - ./tests/before_script.travis.sh
+ - travis_retry ./tests/before_script.travis.sh
script:
- - ./tests/script.travis.sh
+ - travis_retry ./tests/script.travis.sh
after_script:
- ./tests/after_script.travis.sh
diff --git a/KEYS b/KEYS
index 0dcd781..029eb68 100644
--- a/KEYS
+++ b/KEYS
@@ -186,3 +186,61 @@
gslTn9eM5A==
=Jacn
-----END PGP PUBLIC KEY BLOCK-----
+pub rsa4096/9F4FBC3D 2018-09-11 [SC]
+uid [ultimate] Takako Shimamoto (CODE SIGNING KEY) <shimamoto@apache.org>
+sig 3 9F4FBC3D 2018-09-11 Takako Shimamoto (CODE SIGNING KEY) <shimamoto@apache.org>
+sub rsa4096/EC151981 2018-09-11 [E]
+sig 9F4FBC3D 2018-09-11 Takako Shimamoto (CODE SIGNING KEY) <shimamoto@apache.org>
+
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBFuXgEQBEADA2p47lLog6fWkm3yXB7+jcvzzhZVkLweePBkEi5I0QBOX7PpL
+CFNGGDdJW0L6p+8PhHWkzEeCdzYEJj74TGuMT9pZ+ibbjw3BLw3CvFaJa24/g83j
+1jfoKOBLL7xdsvYyrMr/U3ZZYOpD6UkW4LjMWooGYcthlQgpuTXhmLswrym+b1YA
+9xJbjFFL7gktB9O/XPf80nKDv+/duCtCcLKsWRuVsfFmnabJGJsok17wT9j5gjjc
+GfADZvsQdXJDYFS8Z7Os5fczPzx+xpIKioqLUN1bmXDuwF1+e+hgQuK4WS3RfOu6
+N9bp2R/cnYOcPWIGi955wKkjbUo6ujFFg7ICxhWTEqALZuXXScDoA0SkjtD/E8u6
+d8L43Hb64v4TA6qc7sTzyUGsKjzs/j8iTCFu7H9rl+MvpTZj6BnovHKcqufWO9Aj
+ndPPdVsnNse8MoBL7yxZ/eohVILA0LdHu/AnolfQTHtKkKFiCke18OhnS7x8Vg8j
+Q1rUDllsG77LhyA3EwecR//E518nOrxreD4PVXzQvkQz3HK9V2LBAeKrAzCtn0h8
+fpBfCgcN3r5+f8eA34Y3f4b6SGgyRRYYQtIwJ1w0CLGCWm06CKK8rKBK1wa988QA
+3W4r+vCNVKEKUjPBltOvSpebk3DS4Ymk6plRXxLWedS8c89UJE7jb9wF3wARAQAB
+tDpUYWtha28gU2hpbWFtb3RvIChDT0RFIFNJR05JTkcgS0VZKSA8c2hpbWFtb3Rv
+QGFwYWNoZS5vcmc+iQJOBBMBCgA4FiEEGRmUSApCN8H1BfYxFyTwJ59PvD0FAluX
+gEQCGwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQFyTwJ59PvD3quw/+KJ7k
+lNPkF0ogvBW48bn9HZgm3M6I/fxmHoqqEF1q8uCjnSXuHboEb1LhQO+BKLA4WH+F
+fXUAlAIdzbGrUVIvuExr5QMhVY5oDofUMbUOouJSPG/1JXjikjnS3UP2eZYKyWst
+bNZH7OrMiXqKtGaF8HT0BgVsNYxIEeKAKo4N0QWaUS0n1ep/GqdCBKuFGfXnxH7e
+qMzJCEyuRhXu+S7t7EGdBUGz/2kYAHcfsuAj4y24xzyQOUYYox1wBGoMwg4MGzIy
+WmVflIO7Unqz33dquEfNrOopK6kAbHmI1MBloHcNOVclF8sDTBH2kkkBfnFPy+H7
+munXFMg4Dtk+4fMsdPp8+wkvQd6J67ao67D9KBN/jnUSohw4bEuzOl6cy+dVlTf9
+XI+t/vKNJq+N8gRRTHvbaDfT9j5JgH51abFnh7Y9UDILU9JmvIFbjkBIQPpUg/ZV
+CPaNzafFPvTn3G/KTVkpc75IDMjgEX4i9scPosvJL8rpGJnfMIEjt7tSwlLJ8lWf
+DsW7XkWo6KpKdlve0e36KV23EdyUAAZ3+Oy05nAdZo/DhaC9LmmSlJwg9l/dwoUG
+HEdPFdEWquoqkAQCbw3JXNuISugrpq7l6gD2cckAEOMg4ub5nGUVCLgojrWa3vGN
+eb0YLBR5HlcRHhCnAQj69l7jgID4/VLNCUMjKfi5Ag0EW5eARAEQANf2H038aioy
+0wFO27pERyVbPQrDr9kmN3AX4QoJBQ8U67jEkO+/vjz8S33H/Y/x8crMm8QCly6l
+ECukdPCFmne+gloHlJm2pL2Qre/6YKuEDHMq/wut1/HDfKrkA86zfKkX5aut+Vtx
+jdh7awf8ub6XcSTmmACBk+g/bvVoKCBD21bdw6h1I1/rYR+X+cCSCTSzvDLlv8ye
+JeMEeLdld0/lTDUHXUyYf/z8AVr2IrtcFJFlA27ixtCqU3getyhBT1zGhlSFtQar
+4tBzQ7UMOKQcxczHOCHM7lyB58zRsDv2PSr5LCV9tysR/CzNHdxIxvsK8qCf4wNp
+YQLqwZa0bMdk/1vfiHf2f9L5PIWvfXUcfLFEUqG0GvARia1JC/YCA2vFEluVRPPg
+EZgMsrUWoBOj+4qPaABAYFe8Tv7WW+vHKB9/+sETzUmo3TV0wcZz9qKiQwYoxWZQ
+/hME5q7SLWHd7kLfyTbHBZrYtIvYLRFBljH9RZQR7i+VMThG9yWPrmpiY9Pcxxfv
+sefGISNwlNWmXbK55+LuFHIcjPI+FTUQBF0SqvxJuCXjMmTsxjP3w+BiCrKgJEt4
+ZVWN7VI8LYHP6IIctg65TPNti/rkd1AU4MDpSESN0o8b5R+04dIb05s2oEIGPY2R
+tr2WUZY6YwuEmcKQ/igsYq0TwOtIqEtdABEBAAGJAjYEGAEKACAWIQQZGZRICkI3
+wfUF9jEXJPAnn0+8PQUCW5eARAIbDAAKCRAXJPAnn0+8PenjEADAIkw4B/AC7cP+
+xJzjYsE0kVOKLRYo+2+5jRvyoLWffcU6WMs44sF9XI4BRDHAGgDC0xvK03LPeh0d
+mfhIMVEiqG8Zz+6Qkt4upCkXDuJ5QnjKZh4SWXNpW/avzOwCOX2f8JWz11Qoq+J4
+Vnd0BbIjjI+rDiBbccr5Kc6tN43QhihLclJ5hO2QpdWIHGFjCaekIX6zWVYAkkFk
+UB4vHG/eghXJA44lP2kUtVb0Ay5Fl23G8bNqC30/DoswE5bJZjgEbKWUGHBCAA+q
+GtYDG6ttj1AzQuwhaW3mjCaspRHYbPp76Jqh/dw1mRuwWsgw/rYhw/Ptydpzeyad
+W3NlHFj0NweMmstfmvwMd6VOP9cXni87Ynra8pWUvzCO2kzCL+IpUylPFAOvsSHe
+M0exU5/K+ClKlsL5J9oL+6Nc02LDgk56H3aQimKc4sO7/TWqIhHtvYMHJj2PK+Da
+P0qc0vJo6f3wYNU3VhzX/IZg/94luTeMiQgCBie5jXrv3EtBqCg/B/+TzOIrl6pO
+TPFJ9Q5iQiLgoJNrBc2AjV34YaCa8esLRLTZQYoVX/9pN4ECcskNX3TxF45zExUi
+8LMAftwE+fVt7zsJe3oZhPwtLN1RZoLJI/zvXJhXKAP/1LK57Ezrws0AwFSazc0I
+wEvmHAxp9J8DJY97zR+oCp/Or62cNw==
+=8icF
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/RELEASE.md b/RELEASE.md
index 1257915..1b40ce1 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -21,6 +21,32 @@
## Version History
+### 0.13.0
+
+Sep 20, 2018
+
+#### New Features
+
+- [PIO-161](https://issues.apache.org/jira/browse/PIO-161): Spark 2.3 support.
+
+#### Behavior Changes
+
+- [PIO-158](https://issues.apache.org/jira/browse/PIO-158): More officially deprecate support for Scala 2.10 and Spark 1.x.
+
+#### Other Changes
+
+- [PIO-152](https://issues.apache.org/jira/browse/PIO-152): DOAP syntax error.
+- [PIO-155](https://issues.apache.org/jira/browse/PIO-155): Fix 'Topic Labelling with Wikipedia' Template Link.
+- [PIO-156](https://issues.apache.org/jira/browse/PIO-156): Stale release on download page.
+- [PIO-160](https://issues.apache.org/jira/browse/PIO-160): Array out of bound exception in JDBCUtils when --env is not supplied to CreateWorkflow.
+
+#### Credits
+
+The following contributors have spent a great deal of effort to bring to you
+this release:
+
+Donald Szeto, Takako Shimamoto
+
### 0.12.1
Mar 11, 2018
diff --git a/bin/pio-class b/bin/pio-class
index 94cc520..9e01a40 100755
--- a/bin/pio-class
+++ b/bin/pio-class
@@ -44,7 +44,7 @@
# Make sure the Apache Spark version meets the prerequisite if it is a binary
# distribution
-MIN_SPARK_VERSION="1.3.0"
+MIN_SPARK_VERSION="1.6.3"
if [ -z "$SPARK_HOME" ]; then
echo -e "\033[0;31mSPARK_HOME must be set in conf/pio-env.sh, or in the environment!\033[0m"
exit 1
diff --git a/build.sbt b/build.sbt
index ec54104..308b409 100644
--- a/build.sbt
+++ b/build.sbt
@@ -46,11 +46,15 @@
"2.2" -> Map(
"akka" -> "2.4.17",
"hadoop" -> "2.7.3",
+ "json4s" -> "3.2.11"),
+ "2.3" -> Map(
+ "akka" -> "2.4.17",
+ "hadoop" -> "2.7.3",
"json4s" -> "3.2.11")))
name := "apache-predictionio-parent"
-version in ThisBuild := "0.12.1"
+version in ThisBuild := "0.13.0"
organization in ThisBuild := "org.apache.predictionio"
@@ -65,7 +69,7 @@
scalacOptions in (ThisBuild, Test) ++= Seq("-Yrangepos")
fork in (ThisBuild, run) := true
-javacOptions in (ThisBuild, compile) ++= Seq("-source", "1.7", "-target", "1.7",
+javacOptions in (ThisBuild, compile) ++= Seq("-source", "1.8", "-target", "1.8",
"-Xlint:deprecation", "-Xlint:unchecked")
// Ignore differentiation of Spark patch levels
@@ -323,6 +327,10 @@
testOptions in Test += Tests.Argument("-oDF")
printBuildInfo := {
+ if (scalaBinaryVersion.value == "2.10")
+ streams.value.log.warn("Support for Scala 2.10 is deprecated. Please upgrade to a newer version of Scala.")
+ if (sparkBinaryVersion.value == "1.6")
+ streams.value.log.warn("Support for Spark 1.6 is deprecated. Please upgrade to a newer version of Spark.")
println(s"PIO_SCALA_VERSION=${scalaVersion.value}")
println(s"PIO_SPARK_VERSION=${sparkVersion.value}")
println(s"PIO_ELASTICSEARCH_VERSION=${elasticsearchVersion.value}")
diff --git a/conf/pio-vendors.sh b/conf/pio-vendors.sh
index 6420207..162372f 100644
--- a/conf/pio-vendors.sh
+++ b/conf/pio-vendors.sh
@@ -51,7 +51,8 @@
HADOOP_MAJOR=`echo $PIO_HADOOP_VERSION | awk -F. '{print $1 "." $2}'`
SPARK_DIR=spark-${PIO_SPARK_VERSION}-bin-hadoop${HADOOP_MAJOR}
SPARK_ARCHIVE=${SPARK_DIR}.tgz
-SPARK_DOWNLOAD=http://d3kbcqa49mib13.cloudfront.net/${SPARK_ARCHIVE}
+SPARK_DOWNLOAD_MIRROR=https://www.apache.org/dyn/closer.lua\?action=download\&filename=spark/spark-${PIO_SPARK_VERSION}/${SPARK_ARCHIVE}
+SPARK_DOWNLOAD_ARCHIVE=https://archive.apache.org/dist/spark/spark-${PIO_SPARK_VERSION}/${SPARK_ARCHIVE}
# ELASTICSEARCH_DOWNLOAD
# 5.x https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-${PIO_ELASTICSEARCH_VERSION}.tar.gz
# 1.x https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${PIO_ELASTICSEARCH_VERSION}.tar.gz
diff --git a/core/src/main/scala/org/apache/predictionio/controller/Engine.scala b/core/src/main/scala/org/apache/predictionio/controller/Engine.scala
index 1f9d0ab..3b5f363 100644
--- a/core/src/main/scala/org/apache/predictionio/controller/Engine.scala
+++ b/core/src/main/scala/org/apache/predictionio/controller/Engine.scala
@@ -268,7 +268,7 @@
/** Extract model for persistent layer.
*
- * PredictionIO presist models for future use. It allows custom
+ * PredictionIO persists models for future use. It allows custom
* implementation for persisting models. You need to implement the
* [[org.apache.predictionio.controller.PersistentModel]] interface. This method
* traverses all models in the workflow. If the model is a
@@ -642,7 +642,7 @@
dataSource.readTrainingBase(sc)
} catch {
case e: StorageClientException =>
- logger.error(s"Error occured reading from data source. (Reason: " +
+ logger.error(s"Error occurred reading from data source. (Reason: " +
e.getMessage + ") Please see the log for debugging details.", e)
sys.exit(1)
}
diff --git a/core/src/main/scala/org/apache/predictionio/core/AbstractDoer.scala b/core/src/main/scala/org/apache/predictionio/core/AbstractDoer.scala
index 5da48ce..04d781f 100644
--- a/core/src/main/scala/org/apache/predictionio/core/AbstractDoer.scala
+++ b/core/src/main/scala/org/apache/predictionio/core/AbstractDoer.scala
@@ -48,7 +48,7 @@
// Subclasses only allows two kind of constructors.
// 1. Constructor with P <: Params.
- // 2. Emtpy constructor.
+ // 2. Empty constructor.
// First try (1), if failed, try (2).
try {
val constr = cls.getConstructor(params.getClass)
diff --git a/core/src/main/scala/org/apache/predictionio/core/SelfCleaningDataSource.scala b/core/src/main/scala/org/apache/predictionio/core/SelfCleaningDataSource.scala
index cadf6b8..be3fe05 100644
--- a/core/src/main/scala/org/apache/predictionio/core/SelfCleaningDataSource.scala
+++ b/core/src/main/scala/org/apache/predictionio/core/SelfCleaningDataSource.scala
@@ -69,7 +69,7 @@
/** :: DeveloperApi ::
*
- * Returns RDD of events happend after duration in event window params.
+ * Returns RDD of events happened after duration in event window params.
*
* @return RDD[Event] most recent PEvents.
*/
@@ -87,7 +87,7 @@
/** :: DeveloperApi ::
*
- * Returns Iterator of events happend after duration in event window params.
+ * Returns Iterator of events happened after duration in event window params.
*
* @return Iterator[Event] most recent LEvents.
*/
diff --git a/core/src/main/scala/org/apache/predictionio/workflow/FakeWorkflow.scala b/core/src/main/scala/org/apache/predictionio/workflow/FakeWorkflow.scala
index c9b1d23..8e4db51 100644
--- a/core/src/main/scala/org/apache/predictionio/workflow/FakeWorkflow.scala
+++ b/core/src/main/scala/org/apache/predictionio/workflow/FakeWorkflow.scala
@@ -72,7 +72,7 @@
override val noSave: Boolean = true
}
-/** FakeRun allows user to implement custom function under the exact enviroment
+/** FakeRun allows user to implement custom function under the exact environment
* as other PredictionIO workflow.
*
* Useful for developing new features. Only need to extend this trait and
diff --git a/data/src/test/scala/org/apache/predictionio/data/webhooks/mailchimp/MailChimpConnectorSpec.scala b/data/src/test/scala/org/apache/predictionio/data/webhooks/mailchimp/MailChimpConnectorSpec.scala
index add9c6f..9bb1847 100644
--- a/data/src/test/scala/org/apache/predictionio/data/webhooks/mailchimp/MailChimpConnectorSpec.scala
+++ b/data/src/test/scala/org/apache/predictionio/data/webhooks/mailchimp/MailChimpConnectorSpec.scala
@@ -24,7 +24,7 @@
class MailChimpConnectorSpec extends Specification with ConnectorTestUtil {
- // TOOD: test other events
+ // TODO: test other events
// TODO: test different optional fields
"MailChimpConnector" should {
@@ -87,7 +87,7 @@
"data[merges][EMAIL]" -> "api+unsub@mailchimp.com",
"data[merges][FNAME]" -> "MailChimp",
"data[merges][LNAME]" -> "API",
- "data[merges][INTERESTS]" -> "Group1,Group2", //optional
+ "data[merges][INTERESTS]" -> "Group1,Group2", //optional
"data[ip_opt]" -> "10.20.10.30",
"data[campaign_id]" -> "cb398d21d2"
)
@@ -120,7 +120,7 @@
check(MailChimpConnector, unsubscribe, expected)
}
- //check profile update to event Json
+ //check profile update to event Json
"convert profile update to event JSON" in {
val profileUpdate = Map(
@@ -162,7 +162,7 @@
check(MailChimpConnector, profileUpdate, expected)
}
- //check email update to event Json
+ //check email update to event Json
"convert email update to event JSON" in {
val emailUpdate = Map(
@@ -192,7 +192,7 @@
check(MailChimpConnector, emailUpdate, expected)
}
- //check cleaned email to event Json
+ //check cleaned email to event Json
"convert cleaned email to event JSON" in {
val cleanedEmail = Map(
@@ -221,7 +221,7 @@
check(MailChimpConnector, cleanedEmail, expected)
}
- //check campaign sending status to event Json
+ //check campaign sending status to event Json
"convert campaign sending status to event JSON" in {
val campaign = Map(
diff --git a/data/test.sh b/data/test.sh
index be0eb07..8481f47 100755
--- a/data/test.sh
+++ b/data/test.sh
@@ -481,7 +481,7 @@
checkPOST "/batch/events.json?accessKey=$accessKey" "$testdata" 200
# request with a malformed event (2nd event)
-# the response code is succesful but the error for individual event is reflected in the response's body.
+# the response code is successful but the error for individual event is reflected in the response's body.
testdata='[{
"event" : "my_event_1",
"entityType" : "user",
diff --git a/docs/manual/data/versions.yml b/docs/manual/data/versions.yml
index d21c99b..31a0659 100644
--- a/docs/manual/data/versions.yml
+++ b/docs/manual/data/versions.yml
@@ -1,4 +1,4 @@
-pio: 0.12.1
+pio: 0.13.0
spark: 2.1.1
spark_download_filename: spark-2.1.1-bin-hadoop2.6
elasticsearch_download_filename: elasticsearch-5.5.2
diff --git a/docs/manual/source/datacollection/plugin.html.md b/docs/manual/source/datacollection/plugin.html.md
index db8e671..f99cb4f 100644
--- a/docs/manual/source/datacollection/plugin.html.md
+++ b/docs/manual/source/datacollection/plugin.html.md
@@ -32,7 +32,7 @@
name := "pio-plugin-example"
version := "1.0"
scalaVersion := "2.11.11"
-libraryDependencies += "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.1"
+libraryDependencies += "org.apache.predictionio" %% "apache-predictionio-core" % "0.13.0"
```
Event server plug-ins must extend `EventServerPlugin`. Here is an example of event server plug-in:
diff --git a/docs/manual/source/deploy/plugin.html.md b/docs/manual/source/deploy/plugin.html.md
index d20f130..c9a120c 100644
--- a/docs/manual/source/deploy/plugin.html.md
+++ b/docs/manual/source/deploy/plugin.html.md
@@ -32,7 +32,7 @@
name := "pio-plugin-example"
version := "1.0"
scalaVersion := "2.11.11"
-libraryDependencies += "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.1"
+libraryDependencies += "org.apache.predictionio" %% "apache-predictionio-core" % "0.13.0"
```
Engine server plug-ins must extend `EngineServerPlugin`. Here is an example of engine server plug-in:
diff --git a/docs/manual/source/install/index.html.md.erb b/docs/manual/source/install/index.html.md.erb
index 167d3bf..3b34871 100644
--- a/docs/manual/source/install/index.html.md.erb
+++ b/docs/manual/source/install/index.html.md.erb
@@ -41,6 +41,9 @@
* Apache HBase 0.98.5
* Elasticsearch 1.7.6
+WARNING: **Note that support for Scala 2.10 and Spark 1.6.x are deprecated as of PredictionIO 0.13.0,
+and may be removed in a future release.**
+
If you are running on a single machine, we recommend a minimum of 2GB memory.
INFO: If you are using Linux, Apache Spark local mode, which is the default
diff --git a/docs/manual/source/install/install-sourcecode.html.md.erb b/docs/manual/source/install/install-sourcecode.html.md.erb
index edb6cdb..897f7e8 100644
--- a/docs/manual/source/install/install-sourcecode.html.md.erb
+++ b/docs/manual/source/install/install-sourcecode.html.md.erb
@@ -148,9 +148,9 @@
versions of dependencies. As of writing, one could build PredictionIO against
these different dependencies:
-* Scala 2.10.x, 2.11.x
-* Spark 1.6.x, 2.0.x, 2.1.x
-* Hadoop 2.4.x to 2.7.x
+* Scala 2.10.x(deprecated), 2.11.x
+* Spark 1.6.x(deprecated), 2.0.x, 2.1.x
+* Hadoop 2.6.x, 2.7.x
* Elasticsearch 1.7.x, 5.x
As an example, if you want to build PredictionIO to support Scala 2.11.8,
diff --git a/docs/manual/source/system/index.html.md b/docs/manual/source/system/index.html.md
index b4fe884..8e9667d 100644
--- a/docs/manual/source/system/index.html.md
+++ b/docs/manual/source/system/index.html.md
@@ -19,14 +19,6 @@
limitations under the License.
-->
-During the [installation](/install), you have installed the latest stable
-versions of the following software:
-
-* Apache Hadoop up to 2.7.2 (required only if YARN and HDFS are needed)
-* Apache HBase up to 1.2.4
-* Apache Spark up to 1.6.3, or up to 2.3.1
-* Elasticsearch up to 1.7.5, or up to 5.6.11
-
This section explains general rules-of-thumb for how they are used in
PredictionIO. The actual implementation of the Template will define how much of
this applies. PredictionIO is flexible about much of this configuration but its
diff --git a/docs/scaladoc/README.md b/docs/scaladoc/README.md
index 2f64966..2faa7e3 100644
--- a/docs/scaladoc/README.md
+++ b/docs/scaladoc/README.md
@@ -23,4 +23,4 @@
$ sbt/sbt pioUnidoc
```
-2. Point your web browser at `target/scala-2.10/unidoc/index.html`.
+2. Point your web browser at `target/scala-2.11/unidoc/index.html`.
diff --git a/examples/scala-parallel-classification/README.md b/examples/scala-parallel-classification/README.md
index 20f6956..82fcda0 100644
--- a/examples/scala-parallel-classification/README.md
+++ b/examples/scala-parallel-classification/README.md
@@ -15,6 +15,6 @@
limitations under the License.
-->
-This is based on Classification Engine Template v0.12.0-incubating.
+This is based on Classification Engine Template v0.13.0.
Please refer to http://predictionio.apache.org/templates/classification/how-to/
diff --git a/examples/scala-parallel-classification/add-algorithm/build.sbt b/examples/scala-parallel-classification/add-algorithm/build.sbt
index 703b5be..e929f9e 100644
--- a/examples/scala-parallel-classification/add-algorithm/build.sbt
+++ b/examples/scala-parallel-classification/add-algorithm/build.sbt
@@ -20,5 +20,5 @@
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.1" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.13.0" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
diff --git a/examples/scala-parallel-classification/reading-custom-properties/build.sbt b/examples/scala-parallel-classification/reading-custom-properties/build.sbt
index 703b5be..e929f9e 100644
--- a/examples/scala-parallel-classification/reading-custom-properties/build.sbt
+++ b/examples/scala-parallel-classification/reading-custom-properties/build.sbt
@@ -20,5 +20,5 @@
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.1" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.13.0" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
diff --git a/examples/scala-parallel-ecommercerecommendation/README.md b/examples/scala-parallel-ecommercerecommendation/README.md
index e8f5ed4..0063fde 100644
--- a/examples/scala-parallel-ecommercerecommendation/README.md
+++ b/examples/scala-parallel-ecommercerecommendation/README.md
@@ -15,6 +15,6 @@
limitations under the License.
-->
-This is based on E-Commerce Recommendation Template v0.12.0-incubating.
+This is based on E-Commerce Recommendation Template v0.13.0.
Please refer to http://predictionio.apache.org/templates/ecommercerecommendation/how-to/
diff --git a/examples/scala-parallel-ecommercerecommendation/adjust-score/build.sbt b/examples/scala-parallel-ecommercerecommendation/adjust-score/build.sbt
index e45e277..7899d60 100644
--- a/examples/scala-parallel-ecommercerecommendation/adjust-score/build.sbt
+++ b/examples/scala-parallel-ecommercerecommendation/adjust-score/build.sbt
@@ -20,5 +20,5 @@
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.1" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.13.0" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
diff --git a/examples/scala-parallel-ecommercerecommendation/train-with-rate-event/build.sbt b/examples/scala-parallel-ecommercerecommendation/train-with-rate-event/build.sbt
index e45e277..7899d60 100644
--- a/examples/scala-parallel-ecommercerecommendation/train-with-rate-event/build.sbt
+++ b/examples/scala-parallel-ecommercerecommendation/train-with-rate-event/build.sbt
@@ -20,5 +20,5 @@
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.1" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.13.0" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
diff --git a/examples/scala-parallel-recommendation/README.md b/examples/scala-parallel-recommendation/README.md
index c8ddd4a..379a750 100644
--- a/examples/scala-parallel-recommendation/README.md
+++ b/examples/scala-parallel-recommendation/README.md
@@ -15,6 +15,6 @@
limitations under the License.
-->
-This is based on Recommendation Template v0.12.0-incubating.
+This is based on Recommendation Template v0.13.0.
Please refer to http://predictionio.apache.org/templates/recommendation/how-to/
diff --git a/examples/scala-parallel-recommendation/blacklist-items/build.sbt b/examples/scala-parallel-recommendation/blacklist-items/build.sbt
index 0251958..bc124a1 100644
--- a/examples/scala-parallel-recommendation/blacklist-items/build.sbt
+++ b/examples/scala-parallel-recommendation/blacklist-items/build.sbt
@@ -20,5 +20,5 @@
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.1" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.13.0" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
\ No newline at end of file
diff --git a/examples/scala-parallel-recommendation/customize-data-prep/build.sbt b/examples/scala-parallel-recommendation/customize-data-prep/build.sbt
index 0251958..bc124a1 100644
--- a/examples/scala-parallel-recommendation/customize-data-prep/build.sbt
+++ b/examples/scala-parallel-recommendation/customize-data-prep/build.sbt
@@ -20,5 +20,5 @@
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.1" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.13.0" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
\ No newline at end of file
diff --git a/examples/scala-parallel-recommendation/customize-serving/build.sbt b/examples/scala-parallel-recommendation/customize-serving/build.sbt
index 0251958..bc124a1 100644
--- a/examples/scala-parallel-recommendation/customize-serving/build.sbt
+++ b/examples/scala-parallel-recommendation/customize-serving/build.sbt
@@ -20,5 +20,5 @@
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.1" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.13.0" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
\ No newline at end of file
diff --git a/examples/scala-parallel-recommendation/reading-custom-events/build.sbt b/examples/scala-parallel-recommendation/reading-custom-events/build.sbt
index 0251958..bc124a1 100644
--- a/examples/scala-parallel-recommendation/reading-custom-events/build.sbt
+++ b/examples/scala-parallel-recommendation/reading-custom-events/build.sbt
@@ -20,5 +20,5 @@
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.1" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.13.0" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
\ No newline at end of file
diff --git a/examples/scala-parallel-recommendation/train-with-view-event/build.sbt b/examples/scala-parallel-recommendation/train-with-view-event/build.sbt
index 0251958..bc124a1 100644
--- a/examples/scala-parallel-recommendation/train-with-view-event/build.sbt
+++ b/examples/scala-parallel-recommendation/train-with-view-event/build.sbt
@@ -20,5 +20,5 @@
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.1" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.13.0" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
\ No newline at end of file
diff --git a/examples/scala-parallel-similarproduct/README.md b/examples/scala-parallel-similarproduct/README.md
index 9f5f34a..9daae15 100644
--- a/examples/scala-parallel-similarproduct/README.md
+++ b/examples/scala-parallel-similarproduct/README.md
@@ -15,6 +15,6 @@
limitations under the License.
-->
-This is based on Similar Product Template v0.12.0-incubating.
+This is based on Similar Product Template v0.13.0.
Please refer to http://predictionio.apache.org/templates/similarproduct/how-to/
diff --git a/examples/scala-parallel-similarproduct/multi-events-multi-algos/build.sbt b/examples/scala-parallel-similarproduct/multi-events-multi-algos/build.sbt
index c03fd46..e72b6f2 100644
--- a/examples/scala-parallel-similarproduct/multi-events-multi-algos/build.sbt
+++ b/examples/scala-parallel-similarproduct/multi-events-multi-algos/build.sbt
@@ -20,5 +20,5 @@
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.1" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.13.0" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
diff --git a/examples/scala-parallel-similarproduct/recommended-user/build.sbt b/examples/scala-parallel-similarproduct/recommended-user/build.sbt
index d40f4f0..f5b86f5 100644
--- a/examples/scala-parallel-similarproduct/recommended-user/build.sbt
+++ b/examples/scala-parallel-similarproduct/recommended-user/build.sbt
@@ -20,5 +20,5 @@
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.1" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.13.0" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
diff --git a/examples/scala-parallel-similarproduct/return-item-properties/build.sbt b/examples/scala-parallel-similarproduct/return-item-properties/build.sbt
index c03fd46..e72b6f2 100644
--- a/examples/scala-parallel-similarproduct/return-item-properties/build.sbt
+++ b/examples/scala-parallel-similarproduct/return-item-properties/build.sbt
@@ -20,5 +20,5 @@
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.1" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.13.0" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
diff --git a/examples/scala-parallel-similarproduct/rid-user-set-event/build.sbt b/examples/scala-parallel-similarproduct/rid-user-set-event/build.sbt
index c03fd46..e72b6f2 100644
--- a/examples/scala-parallel-similarproduct/rid-user-set-event/build.sbt
+++ b/examples/scala-parallel-similarproduct/rid-user-set-event/build.sbt
@@ -20,5 +20,5 @@
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.1" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.13.0" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
diff --git a/examples/scala-parallel-similarproduct/train-with-rate-event/build.sbt b/examples/scala-parallel-similarproduct/train-with-rate-event/build.sbt
index c03fd46..e72b6f2 100644
--- a/examples/scala-parallel-similarproduct/train-with-rate-event/build.sbt
+++ b/examples/scala-parallel-similarproduct/train-with-rate-event/build.sbt
@@ -20,5 +20,5 @@
organization := "org.apache.predictionio"
scalaVersion := "2.11.8"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.1" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.13.0" % "provided",
"org.apache.spark" %% "spark-mllib" % "2.1.1" % "provided")
diff --git a/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBEventsUtil.scala b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBEventsUtil.scala
index 64487fb..4b0ad9a 100644
--- a/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBEventsUtil.scala
+++ b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBEventsUtil.scala
@@ -148,7 +148,7 @@
val rowKey = event.eventId.map { id =>
RowKey(id) // create rowKey from eventId
}.getOrElse {
- // TOOD: use real UUID. not pseudo random
+ // TODO: use real UUID. not pseudo random
val uuidLow: Long = UUID.randomUUID().getLeastSignificantBits
RowKey(
entityType = event.entityType,
diff --git a/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCUtils.scala b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCUtils.scala
index 3eb55ba..98fa257 100644
--- a/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCUtils.scala
+++ b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCUtils.scala
@@ -76,10 +76,14 @@
* @return Map of String to String, e.g. Map("FOO" -> "BAR", "X" -> "Y", ...)
*/
def stringToMap(str: String): Map[String, String] = {
- str.split(",").map { x =>
- val y = x.split("=")
- y(0) -> y(1)
- }.toMap[String, String]
+ if (str.isEmpty) {
+ Map.empty[String, String]
+ } else {
+ str.split(",").map { x =>
+ val y = x.split("=")
+ y(0) -> y(1)
+ }.toMap[String, String]
+ }
}
/** Generate 32-character random ID using UUID with - stripped */
diff --git a/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/JDBCUtilsSpec.scala b/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/JDBCUtilsSpec.scala
new file mode 100644
index 0000000..2282643
--- /dev/null
+++ b/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/JDBCUtilsSpec.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.data.storage.jdbc
+
+import org.specs2.Specification
+
+class JDBCUtilsSpec extends Specification {
+ def is = s2"""
+
+ PredictionIO JDBC Utilities Specification
+
+ driverType should extract the correct portion from a JDBC URL ${driverType}
+ mapToString should return an empty string with empty map input ${mapToStringEmptyInput}
+ stringToMap should correctly create mapping ${stringToMap}
+ stringToMap should return an empty map with empty string input ${stringToMapEmptyInput}
+
+ """
+
+ def driverType = {
+ JDBCUtils.driverType("jdbc:postgresql://remotehost:5432/somedbname") must beEqualTo("postgresql")
+ }
+
+ def mapToStringEmptyInput = {
+ JDBCUtils.mapToString(Map.empty[String, String]) must be empty
+ }
+
+ def stringToMap = {
+ val m = JDBCUtils.stringToMap("FOO=BAR,DEAD=BEEF")
+ m must havePairs("FOO" -> "BAR", "DEAD" -> "BEEF")
+ }
+
+ def stringToMapEmptyInput = {
+ JDBCUtils.stringToMap("") must be empty
+ }
+}
diff --git a/tests/build_docker.sh b/tests/build_docker.sh
index b17a304..2c6f8f8 100755
--- a/tests/build_docker.sh
+++ b/tests/build_docker.sh
@@ -1,4 +1,4 @@
-#!/bin/bash -ex
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -28,7 +28,10 @@
mv ${PGSQL_JAR} $DIR/docker-files/
fi
if [ ! -f $DIR/docker-files/${SPARK_ARCHIVE} ]; then
- wget $SPARK_DOWNLOAD
+ curl -fLo $SPARK_ARCHIVE $SPARK_DOWNLOAD_MIRROR
+ if [[ $? -ne 0 ]]; then
+ curl -fLo $SPARK_ARCHIVE $SPARK_DOWNLOAD_ARCHIVE
+ fi
mv $SPARK_ARCHIVE $DIR/docker-files/
fi
diff --git a/tests/pio_tests/README.md b/tests/pio_tests/README.md
index 070ac8b..cb426da 100644
--- a/tests/pio_tests/README.md
+++ b/tests/pio_tests/README.md
@@ -33,14 +33,14 @@
You can pass it arguments to:
* suppress the output of executed shell commands within the tests
* enable logging
-* specify which tests should be exectued (by names)
+* specify which tests should be executed (by names)
For more information run:
```shell
python3 tests.py -h
```
-As soon as the tests are finishied an XML file with JUnit-like test reports
+As soon as the tests are finished an XML file with JUnit-like test reports
is created in the directory of execution.
### Adding new tests
diff --git a/tests/pio_tests/engines/recommendation-engine/build.sbt b/tests/pio_tests/engines/recommendation-engine/build.sbt
index 798995c..4688165 100644
--- a/tests/pio_tests/engines/recommendation-engine/build.sbt
+++ b/tests/pio_tests/engines/recommendation-engine/build.sbt
@@ -26,6 +26,6 @@
organization := "org.apache.predictionio"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.12.1" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.13.0" % "provided",
"org.apache.spark" %% "spark-core" % sys.env.getOrElse("PIO_SPARK_VERSION", "2.1.1") % "provided",
"org.apache.spark" %% "spark-mllib" % sys.env.getOrElse("PIO_SPARK_VERSION", "2.1.1") % "provided")
diff --git a/tests/pio_tests/utils.py b/tests/pio_tests/utils.py
index 4659e54..6eecf89 100644
--- a/tests/pio_tests/utils.py
+++ b/tests/pio_tests/utils.py
@@ -55,7 +55,7 @@
def obtain_template(engine_dir, template):
"""Given a directory with engines and a template downloads an engine
- if neccessary
+ if necessary
Args:
engine_dir (str): directory where engines are stored
template (str): either the name of an engine from the engines directory
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/App.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/App.scala
index 5884ebd..06c1641 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/commands/App.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/App.scala
@@ -298,7 +298,7 @@
Right(channel.copy(id = chanId))
} else {
errStr = s"""Unable to create new channel.
- |Failed to initalize Event Store.""".stripMargin
+ |Failed to initialize Event Store.""".stripMargin
error(errStr)
// reverted back the meta data
try {
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
index ee8cd50..cd71fdd 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
@@ -109,7 +109,7 @@
val sparkHomePath = Common.getSparkHome(sparkHome)
if (new File(s"$sparkHomePath/bin/spark-submit").exists) {
info(s"Apache Spark is installed at $sparkHomePath")
- val sparkMinVersion = "1.3.0"
+ val sparkMinVersion = "1.6.3"
pioStatus = pioStatus.copy(
sparkHome = sparkHomePath,
sparkMinVersion = sparkMinVersion)
