Merge pull request #366 from sebastian-nagel/NUTCH-2622-unbundle-lgpl-licensed-jars

NUTCH-2622 Unbundle LGPL-licensed jars from binary release
diff --git a/ivy/ivy.xml b/ivy/ivy.xml
index 9b8d667..06bb919 100644
--- a/ivy/ivy.xml
+++ b/ivy/ivy.xml
@@ -23,26 +23,26 @@
 			database etc.
 		</description>
 	</info>
-	
+
 	<configurations>
 		<include file="${basedir}/ivy/ivy-configurations.xml" />
 	</configurations>
-	
+
 	<publications>
 		<!--get the artifact from our module name -->
 		<artifact conf="master" />
 	</publications>
-	
+
 	<dependencies>
 		<dependency org="org.slf4j" name="slf4j-api" rev="1.7.25" conf="*->master" />
 		<dependency org="org.slf4j" name="slf4j-log4j12" rev="1.7.25" conf="*->master" />
-		
+
 		<!--dependency org="log4j" name="log4j" rev="1.2.15" conf="*->default">
 			<exclude org="javax.jms" name="jms" />
 			<exclude org="com.sun.jdmk" name="jmxtools" />
 			<exclude org="com.sun.jmx" name="jmxri" />
 		</dependency-->
-		
+
 		<dependency org="org.apache.commons" name="commons-lang3" rev="3.7" conf="*->default" />
 		<dependency org="org.apache.commons" name="commons-collections4" rev="4.1" conf="*->master" />
 		<dependency org="org.apache.httpcomponents" name="httpclient" rev="4.5.5" conf="*->master" />
@@ -50,7 +50,7 @@
 		<dependency org="org.apache.commons" name="commons-compress" rev="1.16.1" conf="*->default" />
 		<dependency org="org.apache.commons" name="commons-jexl" rev="2.1.1" />
 		<dependency org="com.tdunning" name="t-digest" rev="3.2" />
-		
+
 		<!-- Hadoop Dependencies -->
 		<dependency org="org.apache.hadoop" name="hadoop-common" rev="2.7.4" conf="*->default">
 			<exclude org="hsqldb" name="hsqldb" />
@@ -77,23 +77,31 @@
 		<dependency org="com.github.crawler-commons" name="crawler-commons" rev="0.10" />
 
 		<dependency org="com.martinkl.warc" name="warc-hadoop" rev="0.1.0" />
-		
-        <!--dependency org="org.apache.cxf" name="cxf" rev="3.0.4" conf="*->default"/-->
-        <dependency org="org.apache.cxf" name="cxf-rt-frontend-jaxws" rev="3.1.15" conf="*->default"/>
-        <dependency org="org.apache.cxf" name="cxf-rt-frontend-jaxrs" rev="3.1.15" conf="*->default"/>
-        <dependency org="org.apache.cxf" name="cxf-rt-transports-http" rev="3.1.15" conf="*->default"/>
-        <dependency org="org.apache.cxf" name="cxf-rt-transports-http-jetty" rev="3.1.15" conf="*->default"/>
-        <dependency org="org.apache.cxf" name="cxf-rt-rs-client" rev="3.1.15" conf="test->default"/>
-        <dependency org="com.fasterxml.jackson.core" name="jackson-databind" rev="2.9.5"  conf="*->default"/> 
-        <dependency org="com.fasterxml.jackson.dataformat" name="jackson-dataformat-cbor" rev="2.9.5" conf="*->default"/>
-        <dependency org="com.fasterxml.jackson.jaxrs" name="jackson-jaxrs-json-provider" rev="2.9.5" conf="*->default"/>	
-        
-		<!-- WARC artifacts needed  -->
+
+		<!--dependency org="org.apache.cxf" name="cxf" rev="3.0.4" conf="*->default"/-->
+		<dependency org="org.apache.cxf" name="cxf-rt-frontend-jaxws" rev="3.1.15" conf="*->default"/>
+		<dependency org="org.apache.cxf" name="cxf-rt-frontend-jaxrs" rev="3.1.15" conf="*->default"/>
+		<dependency org="org.apache.cxf" name="cxf-rt-transports-http" rev="3.1.15" conf="*->default"/>
+		<dependency org="org.apache.cxf" name="cxf-rt-transports-http-jetty" rev="3.1.15" conf="*->default"/>
+		<dependency org="org.apache.cxf" name="cxf-rt-rs-client" rev="3.1.15" conf="test->default"/>
+		<dependency org="com.fasterxml.jackson.core" name="jackson-databind" rev="2.9.5" conf="*->default"/>
+		<dependency org="com.fasterxml.jackson.dataformat" name="jackson-dataformat-cbor" rev="2.9.5" conf="*->default"/>
+		<dependency org="com.fasterxml.jackson.jaxrs" name="jackson-jaxrs-json-provider" rev="2.9.5" conf="*->default"/>
+
+		<!-- WARC artifacts needed -->
 		<dependency org="org.netpreserve.commons" name="webarchive-commons" rev="1.1.5" conf="*->default">
 			<exclude module="hadoop-core"/>
 			<exclude org="com.google.guava"/>
 			<exclude org="junit"/>
-			<exclude org="org.json"/>
+			<!-- Exclude dependencies with incompatible license (see https://www.apache.org/legal/resolved.html#category-x) -->
+			<exclude org="org.json"/><!-- JSON License -->
+			<!--
+				Exclusion of the following dependencies disables support of WARC generation by
+				"bin/nutch commoncrawldump -warc ..."
+				Please remove these exclusion and recompile Nutch to generate WARC files using the tool "commoncrawldump".
+			-->
+			<exclude org="it.unimi.dsi" module="dsiutils"/><!-- LGPL 2.1 -->
+			<exclude org="org.gnu.inet" module="libidn"/><!-- LGPL 2.1 -->
 		</dependency>
 
 		<!--artifacts needed for testing -->
@@ -112,26 +120,27 @@
 		<!-- end of test artifacts -->
 
 		<!-- web app dependencies -->
+		<dependency org="org.mortbay.jetty" name="jetty" rev="6.1.26" />
 
-    	<dependency org="org.apache.commons" name="commons-collections4" rev="4.1" conf="*->default" />
-    	<dependency org="org.springframework" name="spring-core" rev="4.0.9.RELEASE" conf="*->default" />
-    	<dependency org="org.springframework" name="spring-context" rev="4.0.9.RELEASE" conf="*->default" />
-    	<dependency org="org.springframework" name="spring-web" rev="4.0.9.RELEASE" conf="*->default" />
+		<dependency org="org.apache.commons" name="commons-collections4" rev="4.1" conf="*->default" />
+		<dependency org="org.springframework" name="spring-core" rev="4.0.9.RELEASE" conf="*->default" />
+		<dependency org="org.springframework" name="spring-context" rev="4.0.9.RELEASE" conf="*->default" />
+		<dependency org="org.springframework" name="spring-web" rev="4.0.9.RELEASE" conf="*->default" />
 
-    	<dependency org="com.sun.jersey" name="jersey-client" rev="1.19.4" conf="*->default" />
-	
-    	<dependency org="com.j256.ormlite" name="ormlite-jdbc" rev="5.1" conf="*->default" />
-    	<dependency org="com.h2database" name="h2" rev="1.4.197" conf="*->default" />
-    	<dependency org="org.eclipse.persistence" name="javax.persistence" rev="2.2.0" conf="*->default" />
-	
-    	<dependency org="org.apache.wicket" name="wicket-core" rev="6.17.0" conf="*->default" />
-    	<dependency org="org.apache.wicket" name="wicket-spring" rev="6.17.0" conf="*->default" />
-    	<dependency org="de.agilecoders.wicket" name="wicket-bootstrap-core" rev="0.9.2" conf="*->default" />
+		<dependency org="com.sun.jersey" name="jersey-client" rev="1.19.4" conf="*->default" />
+
+		<dependency org="com.j256.ormlite" name="ormlite-jdbc" rev="5.1" conf="*->default" />
+		<dependency org="com.h2database" name="h2" rev="1.4.197" conf="*->default" />
+		<dependency org="org.eclipse.persistence" name="javax.persistence" rev="2.2.0" conf="*->default" />
+
+		<dependency org="org.apache.wicket" name="wicket-core" rev="6.17.0" conf="*->default" />
+		<dependency org="org.apache.wicket" name="wicket-spring" rev="6.17.0" conf="*->default" />
+		<dependency org="de.agilecoders.wicket" name="wicket-bootstrap-core" rev="0.9.2" conf="*->default" />
 		<dependency org="de.agilecoders.wicket" name="wicket-bootstrap-extensions" rev="0.9.2" conf="*->default">
 			<exclude org="org.json"/>
 		</dependency>
 
-		
+
 		<!-- RabbitMQ dependencies -->
 		<dependency org="com.rabbitmq" name="amqp-client" rev="5.2.0" conf="*->default" />