[TOREE-412] Update AddDeps magic
* Removes non-Jar files (POM files) from downloaded
artifacts before adding dependencies to the kernel
* Updates to Coursier 1.0.0-M15-7
* Uses "default" configuration by default, instead
of "compile" for Ivy dependencies
* Adds --classifier for maven artifacts
* Adds --ivy-configuration for Ivy artifacts
Closes #120
diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala
index c17da43..a75ae1e 100644
--- a/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala
@@ -20,7 +20,6 @@
import java.net.{URI, URL}
import java.util.Properties
import java.util.concurrent.ConcurrentHashMap
-
import coursier.core.Authentication
import coursier.Cache.Logger
import coursier.Dependency
@@ -29,7 +28,7 @@
import coursier.ivy.{IvyRepository, IvyXml}
import coursier.maven.MavenRepository
import org.springframework.core.io.support.PathMatchingResourcePatternResolver
-
+import scala.util.Try
import scalaz.\/
import scalaz.concurrent.Task
@@ -74,7 +73,10 @@
ignoreResolutionErrors: Boolean,
extraRepositories: Seq[(URL, Option[Credentials])] = Nil,
verbose: Boolean,
- trace: Boolean
+ trace: Boolean,
+ configuration: Option[String] = None,
+ artifactType: Option[String] = None,
+ artifactClassifier: Option[String] = None
): Seq[URI] = {
assert(localDirectory != null)
import coursier._
@@ -90,7 +92,12 @@
module = Module(organization = groupId, name = artifactId),
version = version,
transitive = transitive,
- exclusions = exclusions // NOTE: Source/Javadoc not downloaded by default
+ exclusions = exclusions, // NOTE: Source/Javadoc not downloaded by default
+ configuration = configuration.getOrElse("default"),
+ attributes = Attributes(
+ artifactType.getOrElse(""),
+ artifactClassifier.getOrElse("")
+ )
)
))
@@ -118,8 +125,8 @@
// Report any resolution errors
val errors: Seq[(ModuleVersion, Seq[String])] = resolution.metadataErrors
- errors.foreach { case (d, e) =>
- printStream.println(s"-> Failed to resolve ${d._1.toString()}:${d._2}")
+ errors.foreach { case (dep, e) =>
+ printStream.println(s"-> Failed to resolve ${dep._1.toString()}:${dep._2}")
e.foreach(s => printStream.println(s" -> $s"))
}
@@ -313,10 +320,11 @@
* @param repositories The repositories to convert
* @return The resulting URIs
*/
- private def repositoriesToURIs(repositories: Seq[Repository]) = repositories.map {
- case ivy: IvyRepository => ivy.pattern.string
- case maven: MavenRepository => maven.root
- }.map(new URI(_))
+ private def repositoriesToURIs(repositories: Seq[Repository]) =
+ repositories.map {
+ case ivy: IvyRepository => ivy.pattern.string
+ case maven: MavenRepository => maven.root
+ }.map(s => Try(new URI(s))).filter(_.isSuccess).map(_.get)
/** Creates new Ivy2 local repository using base home URI. */
private def ivy2Local(ivy2HomeUri: URI) = IvyRepository.parse(
diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
index e2ffe61..2f83966 100644
--- a/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala
@@ -54,7 +54,10 @@
ignoreResolutionErrors: Boolean = true,
extraRepositories: Seq[(URL, Option[Credentials])] = Nil,
verbose: Boolean = false,
- trace: Boolean = false
+ trace: Boolean = false,
+ configuration: Option[String] = None,
+ artifactType: Option[String] = None,
+ artifactClassifier: Option[String] = None
): Seq[URI]
/**
diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala
index 56940c9..3ae6de6 100644
--- a/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala
+++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala
@@ -91,7 +91,10 @@
ignoreResolutionErrors: Boolean,
extraRepositories: Seq[(URL, Option[Credentials])] = Nil,
verbose: Boolean,
- trace: Boolean
+ trace: Boolean,
+ configuration: Option[String] = None,
+ artifactType: Option[String] = None,
+ artifactClassifier: Option[String] = None
): Seq[URI] = {
// Start building the ivy.xml file
val ivyFile = File.createTempFile("ivy-custom", ".xml")
diff --git a/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
index 9ef9359..90f97c2 100644
--- a/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
+++ b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala
@@ -60,6 +60,14 @@
"credential", "Adds a credential file to be used to the list"
).withRequiredArg().ofType(classOf[String])
+ private val _configuration = parser.accepts(
+ "ivy-configuration", "Sets the Ivy configuration for the dependency; defaults to \"default\""
+ ).withRequiredArg().ofType(classOf[String])
+
+ private val _classifier = parser.accepts(
+ "classifier", "Sets the dependency's classifier"
+ ).withRequiredArg().ofType(classOf[String])
+
/**
* Execute a magic representing a line magic.
*
@@ -86,11 +94,13 @@
ignoreResolutionErrors = !_abortOnResolutionErrors,
extraRepositories = repositoriesWithCreds,
verbose = _verbose,
- trace = _trace
+ trace = _trace,
+ configuration = get(_configuration),
+ artifactClassifier = get(_classifier)
)
// pass the new Jars to the kernel
- kernel.addJars(uris:_*)
+ kernel.addJars(uris.filter(_.getPath.endsWith(".jar")): _*)
} else {
printHelp(printStream, """%AddDeps my.company artifact-id version""")
}
diff --git a/scala-interpreter/src/main/scala-2.11/org/apache/toree/kernel/interpreter/scala/ScalaInterpreterSpecific.scala b/scala-interpreter/src/main/scala-2.11/org/apache/toree/kernel/interpreter/scala/ScalaInterpreterSpecific.scala
index 0b838fc..fa667b9 100644
--- a/scala-interpreter/src/main/scala-2.11/org/apache/toree/kernel/interpreter/scala/ScalaInterpreterSpecific.scala
+++ b/scala-interpreter/src/main/scala-2.11/org/apache/toree/kernel/interpreter/scala/ScalaInterpreterSpecific.scala
@@ -21,11 +21,9 @@
import java.net.URL
import org.apache.toree.global.StreamState
-import org.apache.toree.interpreter.InterpreterTypes.ExecuteOutput
import org.apache.toree.interpreter.imports.printers.{WrapperConsole, WrapperSystem}
-import org.apache.toree.interpreter.{ExecuteError, ExecuteFailure, Interpreter, Results}
-
-import scala.tools.nsc.interpreter.{InputStream => _, OutputStream => _, _}
+import org.apache.toree.interpreter.{ExecuteError, Interpreter}
+import scala.tools.nsc.interpreter._
import scala.concurrent.Future
import scala.tools.nsc.{Global, Settings, util}
import scala.util.Try
@@ -107,23 +105,14 @@
* @param jars The list of jar locations
*/
override def addJars(jars: URL*): Unit = {
- //jars.foreach(_runtimeClassloader.addJar)
-
- // Enable Scala class support
- reinitializeSymbols()
-
-// jars.foreach(_runtimeClassloader.addJar)
-// updateCompilerClassPath(jars : _*)
-
-
- iMain.addUrlsToClassPath(jars: _*)
-// iMain.
-// _runtimeClassloader =
-
- // Refresh all of our variables
+ iMain.addUrlsToClassPath(jars:_*)
+ // the Scala interpreter will invalidate definitions for any package defined in
+ // the new Jars. This can easily include org.* and make the kernel inaccessible
+ // because it is bound using the previous package definition. To avoid problems,
+ // it is necessary to refresh variable definitions to use the new packages and
+ // to rebind the global definitions.
refreshDefinitions()
-
-
+ bindVariables()
}
/**
@@ -367,6 +356,8 @@
List(
"-Yrepl-class-based",
"-Yrepl-outdir", s"$dir"
+ // useful for debugging compiler classpath or package issues
+ // "-uniqid", "-explaintypes", "-usejavacp", "-Ylog-classpath"
), processAll = true)
s
}
diff --git a/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala b/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
index 250d984..0380410 100644
--- a/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
+++ b/scala-interpreter/src/main/scala/org/apache/toree/kernel/interpreter/scala/ScalaInterpreter.scala
@@ -18,7 +18,6 @@
package org.apache.toree.kernel.interpreter.scala
import java.io.ByteArrayOutputStream
-import java.nio.charset.Charset
import java.util.concurrent.{ExecutionException, TimeoutException, TimeUnit}
import com.typesafe.config.{Config, ConfigFactory}
import jupyter.Displayers
@@ -93,16 +92,20 @@
settings = appendClassPath(settings)
start()
- bindKernelVariable(kernel)
// ensure bindings are defined before allowing user code to run
- bindSparkSession()
- bindSparkContext()
- defineImplicits()
+ bindVariables()
this
}
+ protected def bindVariables(): Unit = {
+ bindKernelVariable(kernel)
+ bindSparkSession()
+ bindSparkContext()
+ defineImplicits()
+ }
+
protected[scala] def buildClasspath(classLoader: ClassLoader): String = {
def toClassLoaderList( classLoader: ClassLoader ): Seq[ClassLoader] = {
@@ -367,7 +370,7 @@
doQuietly(interpret(code))
}
- override def classLoader: ClassLoader = _runtimeClassloader
+ override def classLoader: ClassLoader = _runtimeClassloader
/**
* Returns the language metadata for syntax highlighting
