| import sbt._ |
| import Keys._ |
| import sbtassembly.Plugin._ |
| import AssemblyKeys._ |
| //import com.jsuereth.pgp.sbtplugin.PgpKeys._ |
| |
| object SparkBuild extends Build { |
| // Hadoop version to build against. For example, "0.20.2", "0.20.205.0", or |
| // "1.0.1" for Apache releases, or "0.20.2-cdh3u3" for Cloudera Hadoop. |
| val HADOOP_VERSION = "0.20.205.0" |
| val HADOOP_MAJOR_VERSION = "1" |
| |
| // For Hadoop 2 versions such as "2.0.0-mr1-cdh4.1.1", set the HADOOP_MAJOR_VERSION to "2" |
| //val HADOOP_VERSION = "2.0.0-mr1-cdh4.1.1" |
| //val HADOOP_MAJOR_VERSION = "2" |
| |
| lazy val root = Project("root", file("."), settings = rootSettings) aggregate(core, repl, examples, bagel) |
| |
| lazy val core = Project("core", file("core"), settings = coreSettings) |
| |
| lazy val repl = Project("repl", file("repl"), settings = replSettings) dependsOn (core) |
| |
| lazy val examples = Project("examples", file("examples"), settings = examplesSettings) dependsOn (core) |
| |
| lazy val bagel = Project("bagel", file("bagel"), settings = bagelSettings) dependsOn (core) |
| |
| def sharedSettings = Defaults.defaultSettings ++ Seq( |
| organization := "org.spark-project", |
| version := "0.5.3-SNAPSHOT", |
| scalaVersion := "2.9.2", |
| scalacOptions := Seq(/*"-deprecation",*/ "-unchecked", "-optimize"), // -deprecation is too noisy due to usage of old Hadoop API, enable it once that's no longer an issue |
| unmanagedJars in Compile <<= baseDirectory map { base => (base / "lib" ** "*.jar").classpath }, |
| retrieveManaged := true, |
| transitiveClassifiers in Scope.GlobalScope := Seq("sources"), |
| testListeners <<= target.map(t => Seq(new eu.henkelmann.sbt.JUnitXmlTestsListener(t.getAbsolutePath))), |
| |
| libraryDependencies ++= Seq( |
| "org.eclipse.jetty" % "jetty-server" % "7.5.3.v20111011", |
| "org.scalatest" %% "scalatest" % "1.6.1" % "test", |
| "org.scalacheck" %% "scalacheck" % "1.9" % "test" |
| ), |
| |
| parallelExecution := false, |
| |
| /* Sonatype publishing settings */ |
| /* |
| resolvers ++= Seq("sonatype-snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", |
| "sonatype-staging" at "https://oss.sonatype.org/service/local/staging/deploy/maven2/"), |
| publishMavenStyle := true, |
| useGpg in Global := true, |
| pomExtra := ( |
| <url>http://spark-project.org/</url> |
| <licenses> |
| <license> |
| <name>BSD License</name> |
| <url>https://github.com/mesos/spark/blob/master/LICENSE</url> |
| <distribution>repo</distribution> |
| </license> |
| </licenses> |
| <scm> |
| <connection>scm:git:git@github.com:mesos/spark.git</connection> |
| <url>scm:git:git@github.com:mesos/spark.git</url> |
| </scm> |
| <developers> |
| <developer> |
| <id>matei</id> |
| <name>Matei Zaharia</name> |
| <email>matei@eecs.berkeley.edu</email> |
| <url>http://www.cs.berkeley.edu/~matei</url> |
| <organization>U.C. Berkeley Computer Science</organization> |
| <organizationUrl>http://www.cs.berkeley.edu/</organizationUrl> |
| </developer> |
| </developers> |
| ), |
| |
| publishTo <<= version { (v: String) => |
| val nexus = "https://oss.sonatype.org/" |
| if (v.trim.endsWith("SNAPSHOT")) |
| Some("sonatype-snapshots" at nexus + "content/repositories/snapshots") |
| else |
| Some("sonatype-staging" at nexus + "service/local/staging/deploy/maven2") |
| }, |
| |
| credentials += Credentials(Path.userHome / ".sbt" / "sonatype.credentials"), |
| */ |
| |
| /* Workaround for issue #206 (fixed after SBT 0.11.0) */ |
| watchTransitiveSources <<= Defaults.inDependencies[Task[Seq[File]]](watchSources.task, |
| const(std.TaskExtra.constant(Nil)), aggregate = true, includeRoot = true) apply { _.join.map(_.flatten) } |
| ) |
| |
| val slf4jVersion = "1.6.1" |
| |
| def coreSettings = sharedSettings ++ Seq( |
| name := "spark-core", |
| resolvers ++= Seq( |
| "Typesafe Repository" at "http://repo.typesafe.com/typesafe/releases/", |
| "JBoss Repository" at "http://repository.jboss.org/nexus/content/repositories/releases/", |
| "Cloudera Repository" at "http://repository.cloudera.com/artifactory/cloudera-repos/" |
| ), |
| libraryDependencies ++= Seq( |
| "com.google.guava" % "guava" % "11.0.1", |
| "log4j" % "log4j" % "1.2.16", |
| "org.slf4j" % "slf4j-api" % slf4jVersion, |
| "org.slf4j" % "slf4j-log4j12" % slf4jVersion, |
| "com.ning" % "compress-lzf" % "0.8.4", |
| "org.apache.hadoop" % "hadoop-core" % HADOOP_VERSION, |
| "asm" % "asm-all" % "3.3.1", |
| "com.google.protobuf" % "protobuf-java" % "2.4.1", |
| "de.javakaffee" % "kryo-serializers" % "0.9", |
| "org.jboss.netty" % "netty" % "3.2.6.Final", |
| "it.unimi.dsi" % "fastutil" % "6.4.2", |
| "colt" % "colt" % "1.2.0", |
| "org.apache.mesos" % "mesos" % "0.9.0-incubating" |
| ) ++ (if (HADOOP_MAJOR_VERSION == "2") Some("org.apache.hadoop" % "hadoop-client" % HADOOP_VERSION) else None).toSeq, |
| unmanagedSourceDirectories in Compile <+= baseDirectory{ _ / ("src/hadoop" + HADOOP_MAJOR_VERSION + "/scala") } |
| ) ++ assemblySettings ++ extraAssemblySettings ++ Seq(test in assembly := {}) |
| |
| def rootSettings = sharedSettings ++ Seq( |
| publish := {} |
| ) |
| |
| def replSettings = sharedSettings ++ Seq( |
| name := "spark-repl", |
| libraryDependencies <+= scalaVersion("org.scala-lang" % "scala-compiler" % _) |
| ) |
| |
| def examplesSettings = sharedSettings ++ Seq( |
| name := "spark-examples" |
| ) |
| |
| def bagelSettings = sharedSettings ++ Seq(name := "spark-bagel") |
| |
| def extraAssemblySettings() = Seq(test in assembly := {}) ++ Seq( |
| mergeStrategy in assembly := { |
| case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard |
| case "reference.conf" => MergeStrategy.concat |
| case _ => MergeStrategy.first |
| } |
| ) |
| } |