datafu-spark/build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

// Much of this file is a variation on the Apache Samza build.gradle file

buildscript {
  repositories {
      mavenCentral()
  }
}

plugins {
  id "de.undercouch.download" version "3.4.3"
}

apply from: file("gradle/dependency-versions-scala-" + scalaVersion + ".gradle")

apply plugin: 'scala'

allprojects {
  // For all scala compilation, add extra compiler options, taken from version-specific
  // dependency-versions-scala file applied above.
  tasks.withType(ScalaCompile) {
    scalaCompileOptions.additionalParameters = [ scalaOptions ]
  }
}

archivesBaseName = 'datafu-spark_' + scalaVersion + '_' + sparkVersion

cleanEclipse {
  doLast {
    delete ".apt_generated"
    delete ".settings"
    delete ".factorypath"
    delete "bin"
  }
}

dependencies {
    compile "org.scala-lang:scala-library:$scalaLibVersion"

    testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":" + sparkVersion + "_" + sparkTestingBaseVersion
    testCompile "org.scalatest:scalatest_" + scalaVersion + ":" + sparkVersion
}

// we need to set up the build for hadoop 3
if (hadoopVersion.startsWith("2.")) {
  dependencies {
    compile "org.apache.hadoop:hadoop-common:$hadoopVersion"
    compile "org.apache.hadoop:hadoop-hdfs:$hadoopVersion"
    compile "org.apache.hadoop:hadoop-mapreduce-client-jobclient:$hadoopVersion"
    compile "org.apache.spark:spark-core_" + scalaVersion + ":" + sparkVersion
    compile "org.apache.spark:spark-hive_" + scalaVersion + ":" + sparkVersion
  }
} else {
  dependencies {
    compile "org.apache.hadoop:hadoop-core:$hadoopVersion"
  }
}

project.ext.sparkFile = file("build/spark-zips/spark-" + sparkVersion + ".zip")
project.ext.sparkUnzipped = "build/spark-unzipped/spark-" + sparkVersion

// download pyspark for testing. This is not shipped with datafu-spark.
task downloadPySpark (type: Download) {
  src 'https://github.com/apache/spark/archive/v' + sparkVersion + '.zip'
  dest project.sparkFile
  onlyIfNewer true
}

downloadPySpark.onlyIf {
  ! project.sparkFile.exists()
}

task unzipPySpark(dependsOn: downloadPySpark, type: Copy) {
  from zipTree(downloadPySpark.dest)
  into file("build/spark-unzipped/")
}

unzipPySpark.onlyIf {
  ! file(project.sparkUnzipped).exists()
}

task zipPySpark(dependsOn: unzipPySpark, type: Zip) {
  archiveName = "pyspark-" + sparkVersion + ".zip"
  include "pyspark/**/*"
  destinationDir = file("data/pysparks/")
  from file(project.sparkUnzipped + "/python/")
}

zipPySpark.onlyIf {
  ! file("data/pysparks/pyspark-" + sparkVersion + ".zip").exists()
}

// download py4j for testing. This is not shipped with datafu-spark.
project.ext.py4js = [
  "py4j-0.10.4-src.zip" : "https://files.pythonhosted.org/packages/93/a7/0e1719e8ad34d194aae72dc07a37e65fd3895db7c797a67a828333cd6067/py4j-0.10.4-py2.py3-none-any.whl",
  "py4j-0.10.6-src.zip" : "https://files.pythonhosted.org/packages/4a/08/162710786239aa72bd72bb46c64f2b02f54250412ba928cb373b30699139/py4j-0.10.6-py2.py3-none-any.whl",
  "py4j-0.10.7-src.zip" : "https://files.pythonhosted.org/packages/e3/53/c737818eb9a7dc32a7cd4f1396e787bd94200c3997c72c1dbe028587bd76/py4j-0.10.7-py2.py3-none-any.whl",
  "py4j-0.10.8.1-src.zip" : "https://files.pythonhosted.org/packages/04/de/2d314a921ef4c20b283e1de94e0780273678caac901564df06b948e4ba9b/py4j-0.10.8.1-py2.py3-none-any.whl"
]

task downloadPy4js {
  doLast {
    for (s in py4js) {
      download {
        src s.value
        dest file("data/py4js/" + s.key)
      }
    }
  }
}

downloadPy4js.onlyIf {
  ! file("data/py4js").exists()
}


// The downloads of pyspark and py4j must succeed in order to test the Scala Python bridge in Eclipse or Gradle
tasks.eclipse.dependsOn('zipPySpark')
tasks.compileTestScala.dependsOn('zipPySpark')
tasks.eclipse.dependsOn('downloadPy4js')
tasks.compileTestScala.dependsOn('downloadPy4js')


test {
  systemProperty 'datafu.jar.dir', file('build/libs')
  systemProperty 'datafu.data.dir', file('data')

  systemProperty 'datafu.spark.version', sparkVersion

  maxHeapSize = "2G"
}