Pull in new LDAP mapping connector
git-svn-id: https://svn.apache.org/repos/asf/manifoldcf/trunk@1900034 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/CHANGES.txt b/CHANGES.txt
index 2ea84ab..eb53706 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,9 +1,19 @@
ManifoldCF Change Log
$Id$
-======================= 2.22-dev =====================
+======================= 2.23-dev =====================
+======================= Release 2.22 =====================
+
+CONNECTORS-1694: Enforce preemptive basic auth for Solr output connector.
+(Markus Günther, Markus Schuch)
+
+
+CONNECTORS-1693: Fix for the solr output connector to support
+basic authentication without realm scope.
+(Markus Günther, Markus Schuch)
+
======================= Release 2.21 =====================
CONNECTORS-1685: Fix to update the Icu4j dependencies in the Alfresco WARs
diff --git a/build.xml b/build.xml
index ab98f7b..f843d98 100644
--- a/build.xml
+++ b/build.xml
@@ -32,34 +32,33 @@
<property name="javax-activation.version" value="1.2.0"/>
<property name="hsqldb.version" value="2.3.2"/>
<property name="jetty.version" value="9.4.25.v20191220"/>
- <property name="httpclient.version" value="4.5.8"/>
- <property name="httpmime.version" value="4.5.8"/>
- <property name="httpcore.version" value="4.4.10"/>
+ <property name="httpclient.version" value="4.5.13"/>
+ <property name="httpmime.version" value="4.5.13"/>
+ <property name="httpcore.version" value="4.4.15"/>
<property name="xerces.version" value="2.10.0"/>
<property name="zookeeper.version" value="3.4.10"/>
<property name="mongodb.version" value="2.11.3"/>
<property name="postgresql.version" value="42.1.3"/>
<property name="axis.version" value="1.4"/>
<property name="saaj-api.version" value="1.3"/>
- <property name="saaj-impl.version" value="1.3.3"/>
+ <property name="saaj-impl.version" value="1.5.1"/>
<property name="castor.version" value="1.0.5"/>
<property name="wsdl4j.version" value="1.6.2"/>
<property name="commons-codec.version" value="1.10"/>
<property name="commons-collections.version" value="3.2.2"/>
- <property name="commons-collections4.version" value="4.2"/>
+ <property name="commons-collections4.version" value="4.4"/>
<property name="commons-discovery.version" value="0.5"/>
<property name="commons-el.version" value="1.0"/>
<property name="commons-fileupload.version" value="1.3.3"/>
- <property name="commons-io.version" value="2.5"/>
+ <property name="commons-io.version" value="2.11.0"/>
<property name="commons-lang.version" value="2.6"/>
- <property name="commons-lang3.version" value="3.9"/>
+ <property name="commons-lang3.version" value="3.12.0"/>
<property name="commons-logging.version" value="1.2"/>
<property name="commons-configuration.version" value="1.6"/>
<property name="commons-cli.version" value="1.2"/>
- <property name="slf4j.version" value="1.7.25"/>
+ <property name="slf4j.version" value="1.7.36"/>
<property name="xalan.version" value="2.7.1"/>
- <property name="poi.version" value="4.0.1"/>
- <property name="activation.version" value="1.1.1"/>
+ <property name="poi.version" value="4.1.2"/>
<property name="avalon.version" value="4.3.1"/>
<property name="ecj.version" value="4.3.1"/>
<property name="fop.version" value="1.0"/>
@@ -67,14 +66,24 @@
<property name="tomcat.version" value="6.0.35"/>
<property name="jaxb-api.version" value="2.3.0"/>
<property name="jaxb-impl.version" value="2.3.0"/>
- <property name="jaxb-glassfish.version" value="2.3.2"/>
+ <property name="jaxb-glassfish.version" value="2.3.5"/>
<property name="sun-bind.version" value="2.3.2"/>
<property name="dtd-parser.version" value="1.4.1"/>
- <property name="istack-commons.version" value="3.0.8"/>
+ <property name="istack-commons.version" value="3.0.12"/>
<property name="ant-jar.version" value="1.10.5"/>
<property name="stax-ex.version" value="1.8.1"/>
<property name="fastinfoset.version" value="1.2.16"/>
- <property name="jaxws.version" value="2.2.6"/>
+ <property name="jaxws.version" value="2.3.2"/>
+ <property name="jaxws-api.version" value="2.2.6"/>
+ <property name="jakarta.activation.version" value="1.2.2"/>
+ <property name="jakarta.activation-api.version" value="1.2.1"/>
+ <property name="jakarta.ws.rs-api.version" value="2.1.6"/>
+ <property name="jakarta.xml.bind-api.version" value = "2.3.2"/>
+ <property name="jakarta.xml.ws-api.version" value="2.3.2"/>
+ <property name="jakarta.xml.soap-api.version" value="1.4.1"/>
+ <property name="jakarta.annotation-api.version" value="1.3.4"/>
+ <property name="jakarta.jws-api.version" value="1.1.1"/>
+ <property name="ha-api.version" value="3.1.12"/>
<property name="solrj.version" value="8.3.1"/>
<property name="noggit.version" value="0.8"/>
<property name="commons-math3.version" value="3.6.1"/>
@@ -84,21 +93,21 @@
<property name="selenium.version" value="3.13.0"/>
<property name="hamcrest.version" value="1.3"/>
<property name="log4j12.version" value="1.2.17"/>
- <property name="log4j.version" value="2.17.0"/>
+ <property name="log4j.version" value="2.15.0"/>
<property name="mail.version" value="1.6.2"/>
- <property name="mimepull.version" value="1.3"/>
+ <property name="mimepull.version" value="1.9.11"/>
<property name="opensaml.version" value="1.1"/>
<property name="osgi.version" value="5.0.0"/>
<property name="xml-resolver.version" value="1.2"/>
<property name="servlet-api.version" value="3.1.0"/>
<property name="stax-api.version" value="1.0.1"/>
- <property name="stax-ex.version" value="1.2"/>
- <property name="streambuffer.version" value="0.9"/>
+ <property name="streambuffer.version" value="1.5.7"/>
<property name="velocity.version" value="1.7"/>
+ <property name="velocity-engine-core.version" value="2.3"/>
<property name="wss4j.version" value="1.5.12"/>
<property name="xmlgraphics-commons.version" value="1.4"/>
<property name="woodstox-core-asl.version" value="4.4.0"/>
- <property name="woodstox-core.version" value="5.1.0"/>
+ <property name="woodstox-core.version" value="6.2.8"/>
<property name="stax2-api.version" value="4.1"/>
<property name="xmlsec.version" value="1.4.0"/>
<property name="xml-apis.version" value="1.4.01"/>
@@ -110,7 +119,7 @@
<property name="google-http-client.version" value="1.25.0"/>
<property name="findbugs.version" value="3.0.2"/>
<property name="j2objc.version" value="1.1"/>
- <property name="jackson-core.version" value="2.9.8"/>
+ <property name="jackson-core.version" value="2.13.1"/>
<property name="jetty-jsp-jdt.version" value="2.3.3"/>
<property name="jetty-schemas.version" value="3.1.M0"/>
<property name="alfresco-indexer.version" value="0.8.1"/>
@@ -121,57 +130,61 @@
<property name="objenesis.version" value="2.1"/>
<property name="tika.version" value="1.21"/>
<property name="tagsoup.version" value="1.2.1"/>
- <property name="james.version" value="0.8.3"/>
+ <property name="james.version" value="0.8.4"/>
<property name="vorbis.version" value="0.8"/>
- <property name="metadata-extractor.version" value="2.11.0"/>
<property name="aspectjrt.version" value="1.8.6"/>
<property name="jhighlight.version" value="1.0.3"/>
- <property name="xmlbeans.version" value="3.0.1"/>
- <property name="bc.version" value="1.61"/>
- <property name="tukaani.version" value="1.8"/>
- <property name="parso.version" value="2.0.11"/>
+ <property name="xmlbeans.version" value="3.1.0"/>
+ <property name="bc.version" value="1.70"/>
+ <property name="SparseBitSet.version" value="1.2"/>
+ <property name="HikariCP-java7.version" value="2.4.13"/>
+ <property name="tukaani.version" value="1.9"/>
+ <property name="parso.version" value="2.0.14"/>
<property name="dec.version" value="0.1.2"/>
- <property name="pdfbox.version" value="2.0.15"/>
+ <property name="pdfbox.version" value="2.0.25"/>
<property name="jempbox.version" value="1.8.16"/>
- <property name="commons-compress.version" value="1.18"/>
- <property name="commons-csv.version" value="1.6"/>
+ <property name="commons-compress.version" value="1.21"/>
+ <property name="commons-csv.version" value="1.9.0"/>
<property name="commons-exec.version" value="1.3"/>
<property name="juniversalchardet.version" value="1.0.3"/>
<property name="ucar.version" value="4.5.5"/>
- <property name="jna.version" value="5.3.1"/>
+ <property name="jna.version" value="5.10.0"/>
<property name="cglib.version" value="3.2.4"/>
- <property name="jcommander.version" value="1.35"/>
- <property name="protobuf.version" value="3.7.1"/>
+ <property name="jcommander.version" value="1.82"/>
+ <property name="protobuf.version" value="3.19.4"/>
<property name="jcip-annotations.version" value="1.0"/>
<property name="ehcache.version" value="2.6.2"/>
<property name="bzip2.version" value="0.9.1"/>
- <property name="jdom2.version" value="2.0.6"/>
- <property name="jbig2-imageio.version" value="3.0.2"/>
+ <property name="jdom2.version" value="2.0.6.1"/>
+ <property name="jbig2-imageio.version" value="3.0.3"/>
<property name="jai-imageio.version" value="1.4.0"/>
- <property name="quartz.version" value="2.2.0"/>
+ <property name="quartz.version" value="2.3.2"/>
<property name="boilerpipe.version" value="1.1.0"/>
<property name="dom4j.version" value="1.6.1"/>
- <property name="asm.version" value="7.1"/>
- <property name="xmpcore.version" value="5.1.3"/>
- <property name="rome.version" value="1.12.0"/>
- <property name="mp4parser.version" value="1.1.22"/>
+ <property name="asm.version" value="9.2"/>
+ <property name="xmpcore.version" value="6.1.11"/>
+ <property name="mchange-commons-java.version" value="0.2.19"/>
+ <property name="dd-plist.version" value="1.23"/>
+ <property name="rome.version" value="1.18.0"/>
<property name="jmatio.version" value="1.5"/>
- <property name="java-libpst.version" value="0.8.1"/>
+ <property name="metadata-extractor.version" value="2.15.0.1"/>
+ <property name="isoparser.version" value="1.9.41.7"/>
+ <property name="java-libpst.version" value="0.9.3"/>
<property name="commons-httpclient.version" value="3.1"/>
- <property name="jackson-databind.version" value="2.9.8"/>
- <property name="jackson-annotations.version" value="2.9.8"/>
+ <property name="jackson-databind.version" value="2.13.1"/>
+ <property name="jackson-annotations.version" value="2.13.1"/>
<property name="resteasy.client.version" value="3.0.16.Final"/>
<property name="jsoup.version" value="1.11.3"/>
<property name="jj2000.version" value="5.2"/>
<property name="hadoop.version" value="2.6.0"/>
<property name="aws-sdk.version" value="1.9.0"/>
<property name="joda-time.version" value="2.9.7"/>
- <property name="junrar.version" value="4.0.0"/>
- <property name="jackcess.version" value="3.0.1"/>
- <property name="jackcess-encrypt.version" value="3.0.0"/>
- <property name="opennlp.version" value="1.9.0"/>
- <property name="openjson.version" value="1.0.10"/>
- <property name="sis.version" value="0.8"/>
+ <property name="junrar.version" value="7.4.1"/>
+ <property name="jackcess.version" value="4.0.1"/>
+ <property name="jackcess-encrypt.version" value="4.0.1"/>
+ <property name="opennlp.version" value="1.9.4"/>
+ <property name="openjson.version" value="1.0.12"/>
+ <property name="sis.version" value="1.1"/>
<property name="measure.version" value="1.0"/>
<property name="opengis.version" value="3.0.1"/>
<property name="jsr-275.version" value="0.9.3"/>
@@ -179,16 +192,19 @@
<property name="maven-scm.version" value="1.4"/>
<property name="plexus-utils.version" value="1.5.6"/>
<property name="regexp.version" value="1.3"/>
- <property name="c3p0.version" value="0.9.1.1"/>
+ <property name="c3p0.version" value="0.9.5.5"/>
<property name="sentiment-analysis.version" value="0.1"/>
<property name="uimafit.version" value="2.2.0"/>
<property name="uimaj.version" value="2.9.0"/>
- <property name="cxf.version" value="3.3.1"/>
+ <property name="cxf.version" value="3.5.0"/>
+ <property name="policy.version" value="2.7.6"/>
+ <property name="gmbal.version" value="4.0.0"/>
+ <property name="management-api.version" value="3.2.1"/>
+ <property name="pfl.version" value="4.0.1"/>
<property name="neethi.version" value="3.0.3"/>
- <property name="xmlschema.version" value="2.2.4"/>
- <property name="annotation-api.version" value="1.3.2"/>
- <property name="javax.ws.rs-api.version" value="2.1"/>
- <property name="virtuald.version" value="1.05"/>
+ <property name="xmlschema.version" value="2.3.0"/>
+ <property name="annotation-api.version" value="1.3.5"/>
+ <property name="virtuald.version" value="1.06"/>
<property name="rat.bootstrap.version" value="0.12"/>
<property name="rat_checksum_sha1" value="16398550402b27f81cd0d508cef54b3e47a4a6da"/>
<property name="rat_install_path" location="${user.home}/.ant/lib" />
@@ -579,7 +595,7 @@
<target name="doc" depends="deliver-site-doc"/>
<target name="set-version">
- <property name="release-version" value="2.22-dev"/>
+ <property name="release-version" value="2.23-dev"/>
</target>
<target name="create-source-zip" depends="set-version">
@@ -1024,6 +1040,12 @@
<target name="download-jdk11-materials">
<mkdir dir="lib"/>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="javax/annotation"/>
+ <param name="artifact-name" value="javax.annotation-api"/>
+ <param name="artifact-version" value="1.3.2"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
<antcall target="download-via-maven">
<param name="project-path" value="javax/activation"/>
<param name="artifact-version" value="${javax-activation.version}"/>
@@ -1495,16 +1517,6 @@
</antcall>
</target>
- <target name="download-activation">
- <mkdir dir="lib"/>
- <antcall target="download-via-maven"><param name="target" value="lib"/>
- <param name="project-path" value="javax/activation"/>
- <param name="artifact-version" value="${activation.version}"/>
- <param name="artifact-name" value="activation"/>
- <param name="artifact-type" value="jar"/>
- </antcall>
- </target>
-
<target name="download-avalon-framework">
<property name="avalon.path" value="org/apache/avalon/framework"/>
<mkdir dir="lib"/>
@@ -1682,28 +1694,78 @@
<param name="artifact-name" value="FastInfoset"/>
<param name="artifact-type" value="jar"/>
</antcall>
- <antcall target="download-via-maven"><param name="target" value="lib"/>
- <param name="project-path" value="org/jvnet/staxex"/>
- <param name="artifact-version" value="${stax-ex.version}"/>
- <param name="artifact-name" value="stax-ex"/>
- <param name="artifact-type" value="jar"/>
- </antcall>
</target>
<target name="download-jaxws">
<mkdir dir="lib"/>
<antcall target="download-via-maven"><param name="target" value="lib"/>
- <param name="project-path" value="javax/xml/ws"/>
- <param name="artifact-version" value="${jaxws.version}"/>
- <param name="artifact-name" value="jaxws-api"/>
- <param name="artifact-type" value="jar"/>
- </antcall>
- <antcall target="download-via-maven"><param name="target" value="lib"/>
<param name="project-path" value="com/sun/xml/ws"/>
<param name="artifact-version" value="${jaxws.version}"/>
<param name="artifact-name" value="jaxws-rt"/>
<param name="artifact-type" value="jar"/>
</antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="javax/xml/ws"/>
+ <param name="artifact-version" value="${jaxws-api.version}"/>
+ <param name="artifact-name" value="jaxws-api"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ </target>
+
+ <target name="download-jakarta-runtimes">
+ <mkdir dir="lib"/>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="com/sun/activation"/>
+ <param name="artifact-name" value="jakarta.activation"/>
+ <param name="artifact-version" value="${jakarta.activation.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ </target>
+
+ <target name="download-jakarta-apis">
+ <mkdir dir="lib"/>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="jakarta/activation"/>
+ <param name="artifact-name" value="jakarta.activation-api"/>
+ <param name="artifact-version" value="${jakarta.activation-api.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="jakarta/ws/rs"/>
+ <param name="artifact-name" value="jakarta.ws.rs-api"/>
+ <param name="artifact-version" value="${jakarta.ws.rs-api.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="jakarta/xml/bind"/>
+ <param name="artifact-version" value="${jakarta.xml.bind-api.version}"/>
+ <param name="artifact-name" value="jakarta.xml.bind-api"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="jakarta/xml/ws"/>
+ <param name="artifact-version" value="${jakarta.xml.ws-api.version}"/>
+ <param name="artifact-name" value="jakarta.xml.ws-api"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="jakarta/xml/soap"/>
+ <param name="artifact-version" value="${jakarta.xml.soap-api.version}"/>
+ <param name="artifact-name" value="jakarta.xml.soap-api"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="jakarta/annotation"/>
+ <param name="artifact-version" value="${jakarta.annotation-api.version}"/>
+ <param name="artifact-name" value="jakarta.annotation-api"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="jakarta/jws"/>
+ <param name="artifact-version" value="${jakarta.jws-api.version}"/>
+ <param name="artifact-name" value="jakarta.jws-api"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
</target>
<target name="download-chemistry">
@@ -1918,7 +1980,7 @@
<target name="download-mimepull">
<mkdir dir="lib"/>
<antcall target="download-via-maven"><param name="target" value="lib"/>
- <param name="project-path" value="org/jvnet"/>
+ <param name="project-path" value="org/jvnet/mimepull"/>
<param name="artifact-version" value="${mimepull.version}"/>
<param name="artifact-name" value="mimepull"/>
<param name="artifact-type" value="jar"/>
@@ -1991,6 +2053,16 @@
</antcall>
</target>
+ <target name="download-ha-api">
+ <mkdir dir="lib"/>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/glassfish/ha"/>
+ <param name="artifact-version" value="${ha-api.version}"/>
+ <param name="artifact-name" value="ha-api"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ </target>
+
<target name="download-velocity">
<mkdir dir="lib"/>
<antcall target="download-via-maven"><param name="target" value="lib"/>
@@ -1999,6 +2071,12 @@
<param name="artifact-name" value="velocity"/>
<param name="artifact-type" value="jar"/>
</antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/apache/velocity"/>
+ <param name="artifact-version" value="${velocity-engine-core.version}"/>
+ <param name="artifact-name" value="velocity-engine-core"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
</target>
<target name="download-wss4j">
@@ -2420,7 +2498,79 @@
</antcall>
</target>
- <target name="download-cxf" depends="download-xml-resolver,download-asm,download-jaxb">
+ <target name="download-policy">
+ <mkdir dir="lib"/>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="com/sun/xml/ws"/>
+ <param name="artifact-name" value="policy"/>
+ <param name="artifact-version" value="${policy.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ </target>
+
+ <target name="download-management-api">
+ <mkdir dir="lib"/>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/glassfish/external"/>
+ <param name="artifact-name" value="management-api"/>
+ <param name="artifact-version" value="${management-api.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ </target>
+
+ <target name="download-pfl">
+ <mkdir dir="lib"/>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/glassfish/pfl"/>
+ <param name="artifact-name" value="pfl-basic"/>
+ <param name="artifact-version" value="${pfl.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/glassfish/pfl"/>
+ <param name="artifact-name" value="pfl-tf"/>
+ <param name="artifact-version" value="${pfl.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/glassfish/pfl"/>
+ <param name="artifact-name" value="pfl-asm"/>
+ <param name="artifact-version" value="${pfl.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/glassfish/pfl"/>
+ <param name="artifact-name" value="pfl-dynamic"/>
+ <param name="artifact-version" value="${pfl.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/glassfish/pfl"/>
+ <param name="artifact-name" value="pfl-basic-tools"/>
+ <param name="artifact-version" value="${pfl.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/glassfish/pfl"/>
+ <param name="artifact-name" value="pfl-tf-tools"/>
+ <param name="artifact-version" value="${pfl.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+
+ </target>
+
+ <target name="download-gmbal" depends="download-management-api,download-pfl">
+ <mkdir dir="lib"/>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/glassfish/gmbal"/>
+ <param name="artifact-name" value="gmbal"/>
+ <param name="artifact-version" value="${gmbal.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ </target>
+
+ <target name="download-cxf" depends="download-jakarta-apis,download-policy,download-gmbal,download-stax,download-xml-resolver,download-asm,download-jaxb,download-ha-api,download-saaj">
+ <mkdir dir="lib"/>
<antcall target="download-via-maven"><param name="target" value="lib"/>
<param name="project-path" value="org/apache/cxf"/>
<param name="artifact-name" value="cxf-rt-transports-http-hc"/>
@@ -2558,7 +2708,7 @@
<param name="artifact-type" value="jar"/>
</antcall>
<antcall target="download-via-maven"><param name="target" value="lib"/>
- <param name="project-path" value="c3p0"/>
+ <param name="project-path" value="com/mchange"/>
<param name="artifact-version" value="${c3p0.version}"/>
<param name="artifact-name" value="c3p0"/>
<param name="artifact-type" value="jar"/>
@@ -2660,6 +2810,12 @@
<param name="artifact-type" value="jar"/>
</antcall>
<antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/apache/sis/core"/>
+ <param name="artifact-version" value="${sis.version}"/>
+ <param name="artifact-name" value="sis-feature"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
<param name="project-path" value="org/apache/sis/storage"/>
<param name="artifact-version" value="${sis.version}"/>
<param name="artifact-name" value="sis-netcdf"/>
@@ -2816,12 +2972,6 @@
<param name="artifact-type" value="jar"/>
</antcall>
<antcall target="download-via-maven"><param name="target" value="lib"/>
- <param name="project-path" value="com/drewnoakes"/>
- <param name="artifact-name" value="metadata-extractor"/>
- <param name="artifact-version" value="${metadata-extractor.version}"/>
- <param name="artifact-type" value="jar"/>
- </antcall>
- <antcall target="download-via-maven"><param name="target" value="lib"/>
<param name="project-path" value="org/aspectj"/>
<param name="artifact-name" value="aspectjrt"/>
<param name="artifact-version" value="${aspectjrt.version}"/>
@@ -2847,6 +2997,12 @@
</antcall>
<antcall target="download-via-maven"><param name="target" value="lib"/>
<param name="project-path" value="org/bouncycastle"/>
+ <param name="artifact-name" value="bcutil-jdk15on"/>
+ <param name="artifact-version" value="${bc.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/bouncycastle"/>
<param name="artifact-name" value="bcprov-jdk15on"/>
<param name="artifact-version" value="${bc.version}"/>
<param name="artifact-type" value="jar"/>
@@ -2858,6 +3014,18 @@
<param name="artifact-type" value="jar"/>
</antcall>
<antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="com/zaxxer"/>
+ <param name="artifact-name" value="SparseBitSet"/>
+ <param name="artifact-version" value="${SparseBitSet.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="com/zaxxer"/>
+ <param name="artifact-name" value="HikariCP-java7"/>
+ <param name="artifact-version" value="${HikariCP-java7.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
<param name="project-path" value="org/tukaani"/>
<param name="artifact-name" value="xz"/>
<param name="artifact-version" value="${tukaani.version}"/>
@@ -2895,6 +3063,18 @@
</antcall>
<antcall target="download-via-maven"><param name="target" value="lib"/>
<param name="project-path" value="org/apache/pdfbox"/>
+ <param name="artifact-name" value="preflight"/>
+ <param name="artifact-version" value="${pdfbox.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/apache/pdfbox"/>
+ <param name="artifact-name" value="xmpbox"/>
+ <param name="artifact-version" value="${pdfbox.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/apache/pdfbox"/>
<param name="artifact-name" value="pdfbox-debugger"/>
<param name="artifact-version" value="${pdfbox.version}"/>
<param name="artifact-type" value="jar"/>
@@ -2930,12 +3110,24 @@
<param name="artifact-type" value="jar"/>
</antcall>
<antcall target="download-via-maven"><param name="target" value="lib"/>
- <param name="project-path" value="com/adobe/xmp"/>
- <param name="artifact-name" value="xmpcore"/>
+ <param name="project-path" value="org/tallison/xmp"/>
+ <param name="artifact-name" value="xmpcore-shaded"/>
<param name="artifact-version" value="${xmpcore.version}"/>
<param name="artifact-type" value="jar"/>
</antcall>
<antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="com/mchange"/>
+ <param name="artifact-name" value="mchange-commons-java"/>
+ <param name="artifact-version" value="${mchange-commons-java.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="com/googlecode/plist"/>
+ <param name="artifact-name" value="dd-plist"/>
+ <param name="artifact-version" value="${dd-plist.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
<param name="project-path" value="org/gagravarr"/>
<param name="artifact-name" value="vorbis-java-core"/>
<param name="artifact-version" value="${vorbis.version}"/>
@@ -2972,9 +3164,9 @@
<param name="artifact-type" value="jar"/>
</antcall>
<antcall target="download-via-maven"><param name="target" value="lib"/>
- <param name="project-path" value="com/googlecode/mp4parser"/>
- <param name="artifact-name" value="isoparser"/>
- <param name="artifact-version" value="${mp4parser.version}"/>
+ <param name="project-path" value="org/apache/cxf"/>
+ <param name="artifact-name" value="cxf-rt-security"/>
+ <param name="artifact-version" value="${cxf.version}"/>
<param name="artifact-type" value="jar"/>
</antcall>
<antcall target="download-via-maven"><param name="target" value="lib"/>
@@ -3008,24 +3200,30 @@
<param name="artifact-type" value="jar"/>
</antcall>
<antcall target="download-via-maven"><param name="target" value="lib"/>
- <param name="project-path" value="javax/annotation"/>
- <param name="artifact-name" value="javax.annotation-api"/>
+ <param name="project-path" value="jakarta/annotation"/>
+ <param name="artifact-name" value="jakarta.annotation-api"/>
<param name="artifact-version" value="${annotation-api.version}"/>
<param name="artifact-type" value="jar"/>
</antcall>
<antcall target="download-via-maven"><param name="target" value="lib"/>
- <param name="project-path" value="javax/ws/rs"/>
- <param name="artifact-name" value="javax.ws.rs-api"/>
- <param name="artifact-version" value="${javax.ws.rs-api.version}"/>
- <param name="artifact-type" value="jar"/>
- </antcall>
- <antcall target="download-via-maven"><param name="target" value="lib"/>
<param name="project-path" value="org/tallison"/>
<param name="artifact-name" value="jmatio"/>
<param name="artifact-version" value="${jmatio.version}"/>
<param name="artifact-type" value="jar"/>
</antcall>
<antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/tallison"/>
+ <param name="artifact-name" value="metadata-extractor"/>
+ <param name="artifact-version" value="${metadata-extractor.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/tallison"/>
+ <param name="artifact-name" value="isoparser"/>
+ <param name="artifact-version" value="${isoparser.version}"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
<param name="project-path" value="com/pff"/>
<param name="artifact-name" value="java-libpst"/>
<param name="artifact-version" value="${java-libpst.version}"/>
@@ -3066,7 +3264,7 @@
</antcall>
</target>
- <target name="make-core-deps" depends="download-less-compiler,download-forbidden-checks,download-nuxeo-client,download-json-simple,download-selenium,download-kafka-client,download-joda-time,download-aws-sdk,download-resteasy,download-jsoup,download-mockito,download-alfresco-webscript-plugin,download-alfresco-indexer-client,download-mongo-java-driver,download-jira-client,download-google-api-client,download-dropbox-client,download-solrj,download-zookeeper,download-httpcomponents,download-jdk11-materials,download-hsqldb,download-xerces,download-commons,download-elasticsearch-plugin,download-solr-plugins,download-sharepoint-plugins,download-jstl,download-xmlgraphics-commons,download-woodstox,download-xmlsec,download-xml-apis,download-wss4j,download-velocity,download-streambuffer,download-stax,download-servlet-api,download-xml-resolver,download-osgi,download-opensaml,download-mimepull,download-mail,download-log4j,download-junit,download-jaxws,download-glassfish,download-jaxb,download-tomcat,download-h2,download-h2-support,download-geronimo-specs,download-fop,download-postgresql,download-axis,download-saaj,download-wsdl4j,download-castor,download-jetty,download-slf4j,download-xalan,download-activation,download-avalon-framework,download-poi,download-chemistry,download-ecj,download-hadoop,download-htrace,download-protobuf,download-tika,download-jackson">
+ <target name="make-core-deps" depends="download-jakarta-apis,download-jakarta-runtimes,download-less-compiler,download-forbidden-checks,download-nuxeo-client,download-json-simple,download-selenium,download-kafka-client,download-joda-time,download-aws-sdk,download-resteasy,download-jsoup,download-mockito,download-alfresco-webscript-plugin,download-alfresco-indexer-client,download-mongo-java-driver,download-jira-client,download-google-api-client,download-dropbox-client,download-solrj,download-zookeeper,download-httpcomponents,download-jdk11-materials,download-hsqldb,download-xerces,download-commons,download-elasticsearch-plugin,download-solr-plugins,download-sharepoint-plugins,download-jstl,download-xmlgraphics-commons,download-woodstox,download-xmlsec,download-xml-apis,download-wss4j,download-velocity,download-streambuffer,download-ha-api,download-stax,download-servlet-api,download-xml-resolver,download-osgi,download-opensaml,download-mimepull,download-mail,download-log4j,download-junit,download-jaxws,download-glassfish,download-jaxb,download-tomcat,download-h2,download-h2-support,download-geronimo-specs,download-fop,download-postgresql,download-axis,download-saaj,download-wsdl4j,download-castor,download-jetty,download-slf4j,download-xalan,download-avalon-framework,download-poi,download-chemistry,download-ecj,download-hadoop,download-htrace,download-protobuf,download-tika,download-jackson">
<copy todir="lib">
<fileset dir="lib-license" includes="*.txt"/>
</copy>
diff --git a/connectors/activedirectory/pom.xml b/connectors/activedirectory/pom.xml
index fd92aac..8a92ca2 100644
--- a/connectors/activedirectory/pom.xml
+++ b/connectors/activedirectory/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/alfresco-webscript/pom.xml b/connectors/alfresco-webscript/pom.xml
index bb798bb..0ba7549 100644
--- a/connectors/alfresco-webscript/pom.xml
+++ b/connectors/alfresco-webscript/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/alfresco/pom.xml b/connectors/alfresco/pom.xml
index 4d224fc..400cdc3 100644
--- a/connectors/alfresco/pom.xml
+++ b/connectors/alfresco/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/amazoncloudsearch/pom.xml b/connectors/amazoncloudsearch/pom.xml
index 2ea594a..fe44756 100644
--- a/connectors/amazoncloudsearch/pom.xml
+++ b/connectors/amazoncloudsearch/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/amazons3/pom.xml b/connectors/amazons3/pom.xml
index c844734..24b4106 100644
--- a/connectors/amazons3/pom.xml
+++ b/connectors/amazons3/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/cmis/pom.xml b/connectors/cmis/pom.xml
index 652ce0f..973d62f 100644
--- a/connectors/cmis/pom.xml
+++ b/connectors/cmis/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/confluence-v6/build.xml b/connectors/confluence-v6/build.xml
index b3493d6..da3c2a2 100644
--- a/connectors/confluence-v6/build.xml
+++ b/connectors/confluence-v6/build.xml
@@ -15,7 +15,7 @@
limitations under the License.
-->
-<project name="confluence v6" default="all">
+<project name="confluence-v6" default="all">
<property environment="env" />
<condition property="mcf-dist" value="${env.MCFDISTPATH}">
diff --git a/connectors/confluence-v6/pom.xml b/connectors/confluence-v6/pom.xml
index 3eec515..9a4e065 100755
--- a/connectors/confluence-v6/pom.xml
+++ b/connectors/confluence-v6/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<name>ManifoldCF - Connectors - Confluence v6 Connector</name>
diff --git a/connectors/confluence/pom.xml b/connectors/confluence/pom.xml
index 0ecdd1b..9d27154 100644
--- a/connectors/confluence/pom.xml
+++ b/connectors/confluence/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<name>ManifoldCF - Connectors - Confluence Connector</name>
diff --git a/connectors/contentlimiter/pom.xml b/connectors/contentlimiter/pom.xml
index 0c8fd61..1eff719 100644
--- a/connectors/contentlimiter/pom.xml
+++ b/connectors/contentlimiter/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/csws/pom.xml b/connectors/csws/pom.xml
index 7d51cd4..7c45b39 100755
--- a/connectors/csws/pom.xml
+++ b/connectors/csws/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -238,6 +238,11 @@
<version>${cxf.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.cxf</groupId>
+ <artifactId>cxf-tools-wsdlto-core</artifactId>
+ <version>${cxf.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>${httpcomponent.httpclient.version}</version>
diff --git a/connectors/documentfilter/pom.xml b/connectors/documentfilter/pom.xml
index a57ec90..a7ea910 100644
--- a/connectors/documentfilter/pom.xml
+++ b/connectors/documentfilter/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/dropbox/pom.xml b/connectors/dropbox/pom.xml
index bd134bb..5e76236 100644
--- a/connectors/dropbox/pom.xml
+++ b/connectors/dropbox/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/elasticsearch/pom.xml b/connectors/elasticsearch/pom.xml
index 4e4c6c0..012896f 100644
--- a/connectors/elasticsearch/pom.xml
+++ b/connectors/elasticsearch/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/email/pom.xml b/connectors/email/pom.xml
index 0d9a4ee..64eaee2 100644
--- a/connectors/email/pom.xml
+++ b/connectors/email/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/filesystem/pom.xml b/connectors/filesystem/pom.xml
index 0391377..a481546 100644
--- a/connectors/filesystem/pom.xml
+++ b/connectors/filesystem/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/forcedmetadata/pom.xml b/connectors/forcedmetadata/pom.xml
index 364345c..9789a85 100644
--- a/connectors/forcedmetadata/pom.xml
+++ b/connectors/forcedmetadata/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/generic/pom.xml b/connectors/generic/pom.xml
index fea3f98..562c739 100644
--- a/connectors/generic/pom.xml
+++ b/connectors/generic/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/googledrive/pom.xml b/connectors/googledrive/pom.xml
index 5728385..4199952 100644
--- a/connectors/googledrive/pom.xml
+++ b/connectors/googledrive/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/gridfs/pom.xml b/connectors/gridfs/pom.xml
index a93eea0..327c083 100644
--- a/connectors/gridfs/pom.xml
+++ b/connectors/gridfs/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/gts/pom.xml b/connectors/gts/pom.xml
index ae77e98..4fd791a 100644
--- a/connectors/gts/pom.xml
+++ b/connectors/gts/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/hdfs/pom.xml b/connectors/hdfs/pom.xml
index 52d584b..452e87d 100644
--- a/connectors/hdfs/pom.xml
+++ b/connectors/hdfs/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/html-extractor/pom.xml b/connectors/html-extractor/pom.xml
index 9249fe2..916c026 100644
--- a/connectors/html-extractor/pom.xml
+++ b/connectors/html-extractor/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/jcifs/pom.xml b/connectors/jcifs/pom.xml
index 529db37..f3e4859 100644
--- a/connectors/jcifs/pom.xml
+++ b/connectors/jcifs/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/jdbc/pom.xml b/connectors/jdbc/pom.xml
index b8fbe4c..8154dd0 100644
--- a/connectors/jdbc/pom.xml
+++ b/connectors/jdbc/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/jira/pom.xml b/connectors/jira/pom.xml
index af20805..1145799 100644
--- a/connectors/jira/pom.xml
+++ b/connectors/jira/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/kafka/pom.xml b/connectors/kafka/pom.xml
index 82b2b88..1972092 100644
--- a/connectors/kafka/pom.xml
+++ b/connectors/kafka/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/ldap/pom.xml b/connectors/ldap/pom.xml
index 541ccf3..94b131f 100644
--- a/connectors/ldap/pom.xml
+++ b/connectors/ldap/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/ldapmapper/pom.xml b/connectors/ldapmapper/pom.xml
index 9d69d61..a10f239 100755
--- a/connectors/ldapmapper/pom.xml
+++ b/connectors/ldapmapper/pom.xml
@@ -2,7 +2,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/mongodb/pom.xml b/connectors/mongodb/pom.xml
index 6f70f63..c6ee498 100644
--- a/connectors/mongodb/pom.xml
+++ b/connectors/mongodb/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/nullauthority/pom.xml b/connectors/nullauthority/pom.xml
index e6433b0..8600937 100644
--- a/connectors/nullauthority/pom.xml
+++ b/connectors/nullauthority/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/nulloutput/pom.xml b/connectors/nulloutput/pom.xml
index ef2a483..f243521 100644
--- a/connectors/nulloutput/pom.xml
+++ b/connectors/nulloutput/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/nulltransformation/pom.xml b/connectors/nulltransformation/pom.xml
index f656b46..9f80fd5 100644
--- a/connectors/nulltransformation/pom.xml
+++ b/connectors/nulltransformation/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/nuxeo/pom.xml b/connectors/nuxeo/pom.xml
index 06d1933..5e233c5 100755
--- a/connectors/nuxeo/pom.xml
+++ b/connectors/nuxeo/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<name>ManifoldCF - Connectors - Nuxeo Connector</name>
diff --git a/connectors/opennlp/pom.xml b/connectors/opennlp/pom.xml
index f52bbc5..fe58dd3 100644
--- a/connectors/opennlp/pom.xml
+++ b/connectors/opennlp/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/opensearchserver/pom.xml b/connectors/opensearchserver/pom.xml
index 1b02f3a..3f499e2 100644
--- a/connectors/opensearchserver/pom.xml
+++ b/connectors/opensearchserver/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/pom.xml b/connectors/pom.xml
index 26b88e9..ef63d8b 100644
--- a/connectors/pom.xml
+++ b/connectors/pom.xml
@@ -20,13 +20,13 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-parent</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
<name>ManifoldCF - Connectors</name>
<packaging>pom</packaging>
diff --git a/connectors/regexpmapper/pom.xml b/connectors/regexpmapper/pom.xml
index 1c71cc7..9bc49d2 100644
--- a/connectors/regexpmapper/pom.xml
+++ b/connectors/regexpmapper/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/rocketchat/pom.xml b/connectors/rocketchat/pom.xml
index 7bbf23c..bcbfc92 100644
--- a/connectors/rocketchat/pom.xml
+++ b/connectors/rocketchat/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/rss/pom.xml b/connectors/rss/pom.xml
index 3bca91d..2022ee1 100644
--- a/connectors/rss/pom.xml
+++ b/connectors/rss/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/searchblox/pom.xml b/connectors/searchblox/pom.xml
index f38b554..963b089 100644
--- a/connectors/searchblox/pom.xml
+++ b/connectors/searchblox/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<artifactId>mcf-searchblox-connector</artifactId>
diff --git a/connectors/sharepoint/pom.xml b/connectors/sharepoint/pom.xml
index 9e0d9af..52c18c8 100644
--- a/connectors/sharepoint/pom.xml
+++ b/connectors/sharepoint/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/slack/pom.xml b/connectors/slack/pom.xml
index d1b5fc4..514307e 100644
--- a/connectors/slack/pom.xml
+++ b/connectors/slack/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java b/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
index 134e652..31e174d 100644
--- a/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
+++ b/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
@@ -94,12 +94,12 @@
// Solrj connection-associated objects
protected PoolingHttpClientConnectionManager connectionManager = null;
protected SolrClient solrServer = null;
-
+
// Action URI pieces
private final String postUpdateAction;
private final String postRemoveAction;
private final String postStatusAction;
-
+
// Attribute names
private final String allowAttributeName;
private final String denyAttributeName;
@@ -111,17 +111,17 @@
private final String fileNameAttributeName;
private final String mimeTypeAttributeName;
private final String contentAttributeName;
-
+
// Whether we use extract/update handler or not
private final boolean useExtractUpdateHandler;
-
+
// Document max length
private final Long maxDocumentLength;
// Included and excluded mime types
private final Set<String> includedMimeTypes;
private final Set<String>excludedMimeTypes;
-
+
// Commit-within flag
private final String commitWithin;
@@ -130,7 +130,7 @@
private static final String NOTHING = "__NOTHING__";
private static final String ID_METADATA = "lcf_metadata_id";
private static final String COMMITWITHIN_METADATA = "commitWithin";
-
+
/** How long to wait before retrying a failed ingestion */
private static final long interruptionRetryTime = 60000L;
@@ -152,9 +152,9 @@
this.postUpdateAction = updatePath;
this.postRemoveAction = removePath;
this.postStatusAction = statusPath;
-
+
this.commitWithin = commitWithin;
-
+
this.allowAttributeName = allowAttributeName;
this.denyAttributeName = denyAttributeName;
this.idAttributeName = idAttributeName;
@@ -168,11 +168,11 @@
this.useExtractUpdateHandler = useExtractUpdateHandler;
this.includedMimeTypes = includedMimeTypes;
this.excludedMimeTypes = excludedMimeTypes;
-
+
this.maxDocumentLength = maxDocumentLength;
-
+
initializeKerberos();
-
+
try
{
CloudSolrClient cloudSolrServer = new CloudSolrClient.Builder()
@@ -210,9 +210,9 @@
this.postUpdateAction = updatePath;
this.postRemoveAction = removePath;
this.postStatusAction = statusPath;
-
+
this.commitWithin = commitWithin;
-
+
this.allowAttributeName = allowAttributeName;
this.denyAttributeName = denyAttributeName;
this.idAttributeName = idAttributeName;
@@ -226,11 +226,11 @@
this.useExtractUpdateHandler = useExtractUpdateHandler;
this.includedMimeTypes = includedMimeTypes;
this.excludedMimeTypes = excludedMimeTypes;
-
+
this.maxDocumentLength = maxDocumentLength;
initializeKerberos();
-
+
String location = "";
if (webapp != null)
location = "/" + webapp;
@@ -242,7 +242,7 @@
}
// Initialize standard solr-j.
-
+
SSLConnectionSocketFactory myFactory;
if (keystoreManager != null)
{
@@ -265,7 +265,7 @@
.setTcpNoDelay(true)
.setSoTimeout(socketTimeout)
.build());
-
+
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
@@ -285,10 +285,17 @@
{
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
Credentials credentials = new UsernamePasswordCredentials(userID, password);
- if (realm != null)
- credentialsProvider.setCredentials(new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, realm), credentials);
+ if (realm != null && realm.trim().length() > 0)
+ {
+ final AuthScope scope = new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, realm);
+ credentialsProvider.setCredentials(scope, credentials);
+ clientBuilder.addInterceptorFirst(new PreemptiveBasicAuthInterceptor(scope));
+ }
else
+ {
credentialsProvider.setCredentials(AuthScope.ANY, credentials);
+ clientBuilder.addInterceptorFirst(new PreemptiveBasicAuthInterceptor(AuthScope.ANY));
+ }
clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
}
@@ -314,7 +321,7 @@
}
}
-
+
/** Shut down the poster.
*/
public void shutdown()
@@ -335,7 +342,7 @@
connectionManager.shutdown();
connectionManager = null;
}
-
+
/** Cause a commit to happen.
*/
public void commitPost()
@@ -381,7 +388,7 @@
return;
}
}
-
+
/** Handle a RuntimeException.
* Unfortunately, SolrCloud 4.6.x throws RuntimeExceptions whenever ZooKeeper is not happy.
* We have to catch these too. I've logged a ticket: SOLR-5678.
@@ -399,12 +406,12 @@
currentTime + 2L * 60L * 60000L,
-1,
true);
- }
+ }
// Solr was not able to parse the request because it is malformed: skip the document
Logging.ingest.warn("Solr was unable to parse request during "+context+": "+e.getMessage(),e);
return;
}
-
+
/** Handle a SolrServerException.
* These exceptions seem to be catch-all exceptions having to do with misconfiguration or
* underlying IO exceptions, or request parsing exceptions.
@@ -457,7 +464,7 @@
throw new ManifoldCFException("Unexpected error: "+e2.getMessage());
}
}
-
+
// Use the exception text to determine the proper result.
if (code == 500 && e.getMessage().indexOf("org.apache.tika.exception.TikaException") != -1)
// Can't process the document, so don't keep trying.
@@ -470,17 +477,17 @@
Logging.ingest.error(message);
throw new ManifoldCFException(message);
}
-
+
// If the code is in the 400 range, the document will never be accepted, so indicate that.
if (code >= 400 && code < 500)
return;
-
+
// The only other kind of return code we know how to handle is 50x.
// For these, we should retry for a while.
if (code == 500)
{
long currentTime = System.currentTimeMillis();
-
+
// Log the error
String message = "Solr exception during "+context+" ("+e.code()+"): "+e.getMessage();
Logging.ingest.warn(message,e);
@@ -491,11 +498,11 @@
-1,
true);
}
-
+
// Unknown code: end the job.
throw new ManifoldCFException("Unhandled Solr exception during "+context+" ("+e.code()+"): "+e.getMessage());
}
-
+
/** Handle an IOException.
* I'm not actually sure where these exceptions come from in SolrJ, but we handle them
* as real I/O errors, meaning they should be retried.
@@ -507,7 +514,7 @@
throw new ManifoldCFException(e.getMessage(), ManifoldCFException.INTERRUPTED);
long currentTime = System.currentTimeMillis();
-
+
if (e instanceof java.net.ConnectException)
{
// Server isn't up at all. Try for a brief time then give up.
@@ -520,7 +527,7 @@
3,
true);
}
-
+
if (e instanceof java.net.SocketTimeoutException)
{
String message2 = "Socket timeout exception during "+context+": "+e.getMessage();
@@ -532,7 +539,7 @@
-1,
false);
}
-
+
if (e.getClass().getName().equals("java.net.SocketException"))
{
// In the past we would have treated this as a straight document rejection, and
@@ -559,8 +566,8 @@
3,
false);
}
-
- // Other socket exceptions are service interruptions - but if we keep getting them, it means
+
+ // Other socket exceptions are service interruptions - but if we keep getting them, it means
// that a socket timeout is probably set too low to accept this particular document. So
// we retry for a while, then skip the document.
String message2 = "Socket exception during "+context+": "+e.getMessage();
@@ -583,7 +590,7 @@
3,
false);
}
-
+
/**
* Post the input stream to ingest
*
@@ -616,7 +623,7 @@
activities.recordActivity(null,SolrConnector.INGEST_ACTIVITY,null,documentURI,activities.EXCLUDED_MIMETYPE,"Solr connector rejected document due to mime type restrictions: ("+document.getMimeType()+")");
return false;
}
-
+
// Convert the incoming acls that we know about to qualified forms, and reject the document if
// we don't know how to deal with its acls
Map<String,String[]> aclsMap = new HashMap<String,String[]>();
@@ -628,7 +635,7 @@
String aclType = aclTypes.next();
aclsMap.put(aclType,convertACL(document.getSecurityACL(aclType),authorityNameString,activities));
denyAclsMap.put(aclType,convertACL(document.getSecurityDenyACL(aclType),authorityNameString,activities));
-
+
// Reject documents that have security we don't know how to deal with in the Solr plugin!! Only safe thing to do.
if (!aclType.equals(RepositoryDocument.SECURITY_TYPE_DOCUMENT) &&
!aclType.equals(RepositoryDocument.SECURITY_TYPE_SHARE) &&
@@ -769,7 +776,7 @@
{
t.start();
t.finishUp();
-
+
if (t.getActivityCode() != null)
activities.recordActivity(t.getActivityStart(),SolrConnector.REMOVE_ACTIVITY,null,documentURI,t.getActivityCode(),t.getActivityDetails());
@@ -891,7 +898,7 @@
{
out.add(fieldName, fieldValues);
}
-
+
/** Write a field */
protected static void writeField(ModifiableSolrParams out, String fieldName, List<String> fieldValues)
{
@@ -902,7 +909,7 @@
}
writeField(out, fieldName, values);
}
-
+
/** Write a field */
protected static void writeField(ModifiableSolrParams out, String fieldName, String fieldValue)
{
@@ -923,7 +930,7 @@
writeField(out,metadataDenyACLName,denyAcl[i]);
}
}
-
+
/**
* Output an acl level in a SolrInputDocument
*/
@@ -949,7 +956,7 @@
protected final Map<String,List<String>> arguments;
protected final Map<String,String[]> aclsMap;
protected final Map<String,String[]> denyAclsMap;
-
+
protected Long activityStart = null;
protected Long activityBytes = null;
protected String activityCode = null;
@@ -1043,7 +1050,7 @@
activityBytes = new Long(length);
activityDetails = e.getMessage() +
((e.getCause() != null)?": "+e.getCause().getMessage():"");
-
+
// Broken pipe exceptions we log specially because they usually mean
// Solr has rejected the document, and the user will want to know that.
if (e.getCause() != null && e.getCause().getClass().getName().equals("java.net.SocketException") &&
@@ -1065,7 +1072,7 @@
activityCode = Integer.toString(e.code());
activityDetails = e.getMessage() +
((e.getCause() != null)?": "+e.getCause().getMessage():"");
-
+
// Rethrow; we'll interpret at the next level
throw e;
}
@@ -1074,7 +1081,7 @@
{
if ((ioe instanceof InterruptedIOException) && (!(ioe instanceof java.net.SocketTimeoutException)))
return;
-
+
activityStart = new Long(fullStartTime);
activityCode = ioe.getClass().getSimpleName().toUpperCase(Locale.ROOT);
activityDetails = ioe.getMessage();
@@ -1098,7 +1105,7 @@
// Write the id field
outputDoc.addField( idAttributeName, documentURI );
-
+
if (contentAttributeName != null)
{
// Copy the content into a string. This is a bad thing to do, but we have no choice given SolrJ architecture at this time.
@@ -1115,7 +1122,7 @@
}
outputDoc.addField( contentAttributeName, sb.toString() );
}
-
+
// Write the rest of the attributes
if ( originalSizeAttributeName != null )
{
@@ -1187,7 +1194,7 @@
{
ModifiableSolrParams out = new ModifiableSolrParams();
Logging.ingest.debug("Solr: Writing document '"+documentURI);
-
+
// Write the id field
writeField(out,LITERAL+idAttributeName,documentURI);
// Write the rest of the attributes
@@ -1231,7 +1238,7 @@
if (!StringUtils.isBlank(mimeType))
writeField(out,LITERAL+mimeTypeAttributeName,mimeType);
}
-
+
// Write the access token information
// Both maps have the same keys.
Iterator<String> typeIterator = aclsMap.keySet().iterator();
@@ -1250,23 +1257,23 @@
// Write the metadata, each in a field by itself
buildSolrParamsFromMetadata(out);
-
+
// These are unnecessary now in the case of non-solrcloud setups, because we overrode the SolrJ posting method to use multipart.
//writeField(out,LITERAL+"stream_size",String.valueOf(length));
//writeField(out,LITERAL+"stream_name",document.getFileName());
-
+
// General hint for Tika
if (!StringUtils.isBlank(document.getFileName()))
writeField(out,"resource.name",document.getFileName());
-
+
// Write the commitWithin parameter
if (commitWithin != null)
writeField(out,COMMITWITHIN_METADATA,commitWithin);
contentStreamUpdateRequest.setParams(out);
-
+
contentStreamUpdateRequest.addContentStream(new RepositoryDocumentStream(is,length,contentType,contentName));
-
+
Logging.ingest.debug("Solr: Done writing '"+documentURI+"'");
}
@@ -1405,7 +1412,7 @@
try
{
UpdateResponse response = new UpdateRequest(postRemoveAction).deleteById(documentURI).process(solrServer);
-
+
// Success
activityStart = new Long(fullStartTime);
activityCode = "OK";
@@ -1487,7 +1494,7 @@
return activityDetails;
}
}
-
+
/** Killable thread that does a commit.
* Java 1.5 stopped permitting thread interruptions to abort socket waits. As a result, it is impossible to get threads to shutdown cleanly that are doing
* such waits. So, the places where this happens are segregated in their own threads so that they can be just abandoned.
@@ -1630,7 +1637,7 @@
protected final long length;
protected final String contentType;
protected final String contentName;
-
+
public RepositoryDocumentStream(InputStream is, long length, String contentType, String contentName)
{
this.is = is;
@@ -1638,19 +1645,19 @@
this.contentType = contentType;
this.contentName = contentName;
}
-
+
@Override
public Long getSize()
{
return new Long(length);
}
-
+
@Override
public InputStream getStream() throws IOException
{
return is;
}
-
+
@Override
public Reader getReader() throws IOException
{
@@ -1676,7 +1683,7 @@
{
/** Request parameters. */
private ModifiableSolrParams params;
-
+
/**
* Create a new SolrPing object.
*/
@@ -1684,7 +1691,7 @@
super(METHOD.GET, "/admin/ping");
params = new ModifiableSolrParams();
}
-
+
public SolrPing(String url)
{
super( METHOD.GET, url );
@@ -1705,45 +1712,45 @@
public ModifiableSolrParams getParams() {
return params;
}
-
+
/**
* Remove the action parameter from this request. This will result in the same
* behavior as {@code SolrPing#setActionPing()}. For Solr server version 4.0
* and later.
- *
+ *
* @return this
*/
public SolrPing removeAction() {
params.remove(CommonParams.ACTION);
return this;
}
-
+
/**
* Set the action parameter on this request to enable. This will delete the
* health-check file for the Solr core. For Solr server version 4.0 and later.
- *
+ *
* @return this
*/
public SolrPing setActionDisable() {
params.set(CommonParams.ACTION, CommonParams.DISABLE);
return this;
}
-
+
/**
* Set the action parameter on this request to enable. This will create the
* health-check file for the Solr core. For Solr server version 4.0 and later.
- *
+ *
* @return this
*/
public SolrPing setActionEnable() {
params.set(CommonParams.ACTION, CommonParams.ENABLE);
return this;
}
-
+
/**
* Set the action parameter on this request to ping. This is the same as not
* including the action at all. For Solr server version 4.0 and later.
- *
+ *
* @return this
*/
public SolrPing setActionPing() {
@@ -1783,6 +1790,6 @@
}
return sb.toString();
}
-
+
}
diff --git a/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/PreemptiveBasicAuthInterceptor.java b/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/PreemptiveBasicAuthInterceptor.java
new file mode 100644
index 0000000..2d58538
--- /dev/null
+++ b/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/PreemptiveBasicAuthInterceptor.java
@@ -0,0 +1,68 @@
+/* $Id$ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.manifoldcf.agents.output.solr;
+
+import org.apache.http.HttpException;
+import org.apache.http.HttpRequest;
+import org.apache.http.HttpRequestInterceptor;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.AuthState;
+import org.apache.http.auth.Credentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.protocol.HttpContext;
+
+import java.io.IOException;
+
+public class PreemptiveBasicAuthInterceptor implements HttpRequestInterceptor {
+
+ private final AuthScope scope;
+
+ public PreemptiveBasicAuthInterceptor(final AuthScope scope) {
+ this.scope = scope;
+ }
+
+ @Override
+ public void process(final HttpRequest request,
+ final HttpContext context) throws HttpException, IOException {
+ if (!alreadyAppliesAuthScheme(context)) {
+ final CredentialsProvider provider = getCredentialsProvider(context);
+ final Credentials credentials = provider.getCredentials(scope);
+ if (credentials == null) {
+ throw new HttpException("Missing credentials for preemptive basic authentication.");
+ }
+ final AuthState state = getAuthState(context);
+ state.update(new BasicScheme(), credentials);
+ }
+ }
+
+ private boolean alreadyAppliesAuthScheme(final HttpContext context) {
+ final AuthState authState = getAuthState(context);
+ return authState.getAuthScheme() != null;
+ }
+
+ private AuthState getAuthState(final HttpContext context) {
+ return (AuthState) context.getAttribute(HttpClientContext.TARGET_AUTH_STATE);
+ }
+
+ private CredentialsProvider getCredentialsProvider(final HttpContext context) {
+ return (CredentialsProvider) context.getAttribute(HttpClientContext.CREDS_PROVIDER);
+ }
+}
diff --git a/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/tests/PreemptiveBasicAuthInterceptorTest.java b/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/tests/PreemptiveBasicAuthInterceptorTest.java
new file mode 100644
index 0000000..cd95f2c
--- /dev/null
+++ b/connectors/solr/connector/src/test/java/org/apache/manifoldcf/agents/output/solr/tests/PreemptiveBasicAuthInterceptorTest.java
@@ -0,0 +1,124 @@
+/* $Id$ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.manifoldcf.agents.output.solr.tests;
+
+import org.apache.http.HttpException;
+import org.apache.http.HttpRequest;
+import org.apache.http.HttpRequestInterceptor;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.AuthState;
+import org.apache.http.auth.Credentials;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.message.BasicHttpRequest;
+import org.apache.http.protocol.HttpContext;
+import org.apache.manifoldcf.agents.output.solr.PreemptiveBasicAuthInterceptor;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class PreemptiveBasicAuthInterceptorTest {
+
+ @Test
+ public void shouldAddBasicAuthenticationToRequestIfNotAlreadySet() throws Exception {
+ final HttpRequestInterceptor interceptor = new PreemptiveBasicAuthInterceptor(AuthScope.ANY);
+ final HttpContext context = contextWithoutBasicAuth(new UsernamePasswordCredentials("user", "secret"));
+ interceptor.process(get(), context);
+ final AuthState authState = (AuthState) context.getAttribute(HttpClientContext.TARGET_AUTH_STATE);
+ assertTrue(authState.getAuthScheme() instanceof BasicScheme);
+ assertEquals("user", authState.getCredentials().getUserPrincipal().getName());
+ assertEquals("secret", authState.getCredentials().getPassword());
+ }
+
+ @Test
+ public void shouldThrowHttpExceptionIfNoCredentialsWereProvided() {
+ final HttpRequestInterceptor interceptor = new PreemptiveBasicAuthInterceptor(AuthScope.ANY);
+ final HttpContext context = contextWithoutBasicAuth(null);
+ try {
+ interceptor.process(get(), context);
+ fail("Expected an HttpException, but none was raised.");
+ } catch (HttpException e) {
+ assertEquals("Missing credentials for preemptive basic authentication.", e.getMessage());
+ } catch (IOException e) {
+ fail("Expected an HttpException, but an IOException was raised instead.");
+ }
+ }
+
+ private HttpRequest get() {
+ return new BasicHttpRequest("GET", "https://manifoldcf.apache.org/");
+ }
+
+ private HttpContext contextWithoutBasicAuth(final Credentials credentials) {
+ final CredentialsProvider credentialsProvider = new FakeCredentialsProvider();
+ credentialsProvider.setCredentials(AuthScope.ANY, credentials);
+ final AuthState authState = new AuthState();
+ final HttpContext context = new FakeHttpContext();
+ context.setAttribute(HttpClientContext.CREDS_PROVIDER, credentialsProvider);
+ context.setAttribute(HttpClientContext.TARGET_AUTH_STATE, authState);
+ return context;
+ }
+
+ static class FakeHttpContext implements HttpContext {
+
+ private final Map<String, Object> context = new HashMap<>();
+
+ @Override
+ public Object getAttribute(final String id) {
+ return context.get(id);
+ }
+
+ @Override
+ public void setAttribute(final String id, final Object obj) {
+ context.put(id, obj);
+ }
+
+ @Override
+ public Object removeAttribute(final String id) {
+ return context.remove(id);
+ }
+ }
+
+ static class FakeCredentialsProvider implements CredentialsProvider {
+
+ private final Map<AuthScope, Credentials> credentialsByAuthScope = new HashMap<>();
+
+ @Override
+ public void setCredentials(final AuthScope authScope, final Credentials credentials) {
+ credentialsByAuthScope.put(authScope, credentials);
+ }
+
+ @Override
+ public Credentials getCredentials(final AuthScope authScope) {
+ return credentialsByAuthScope.get(authScope);
+ }
+
+ @Override
+ public void clear() {
+ credentialsByAuthScope.clear();
+ }
+ }
+}
diff --git a/connectors/solr/pom.xml b/connectors/solr/pom.xml
index b06860e..83efac6 100644
--- a/connectors/solr/pom.xml
+++ b/connectors/solr/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/tika/pom.xml b/connectors/tika/pom.xml
index 425ffbf..9a71b20 100644
--- a/connectors/tika/pom.xml
+++ b/connectors/tika/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/tikaservice-rmeta/connector/src/main/java/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/TikaConfig.java b/connectors/tikaservice-rmeta/connector/src/main/java/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/TikaConfig.java
index 5893b05..1caec69 100644
--- a/connectors/tikaservice-rmeta/connector/src/main/java/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/TikaConfig.java
+++ b/connectors/tikaservice-rmeta/connector/src/main/java/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/TikaConfig.java
@@ -47,6 +47,10 @@
public static final String NODE_WRITELIMIT = "writeLimit";
public static final String NODE_EXTRACTARCHIVES = "extractArchives";
public static final String NODE_MAXEMBEDDEDRESOURCES = "maxEmbeddedResources";
+ public static final String NODE_MAXMETADATAVALUELENGTH = "maxMetadataValueLength";
+ public static final String NODE_TOTALMETADATALIMIT = "totalMetadataLimit";
+ public static final String MAXMETADATAVALUELENGTH_DEFAULT = "250000";
+ public static final String TOTALMETADATALIMIT_DEFAULT = "500000";
public static final int WRITELIMIT_DEFAULT = -1;
public static final int MAXEMBEDDEDRESOURCES_DEFAULT = -1;
public static final String NODE_IGNORETIKAEXCEPTION = "ignoreException";
diff --git a/connectors/tikaservice-rmeta/connector/src/main/java/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/TikaExtractor.java b/connectors/tikaservice-rmeta/connector/src/main/java/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/TikaExtractor.java
index 769efe2..fe3f7d3 100644
--- a/connectors/tikaservice-rmeta/connector/src/main/java/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/TikaExtractor.java
+++ b/connectors/tikaservice-rmeta/connector/src/main/java/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/TikaExtractor.java
@@ -109,6 +109,9 @@
/** We handle up to 64K in memory; after that we go to disk. */
protected static final long inMemoryMaximumFile = 65536;
+ // Metadata name exceeding 8k chars may trigger exceptions when the Solr output connector is used
+ private static final int maxMetadataNameLength = 8000;
+
// Raw parameters
/** Tika host name */
@@ -628,7 +631,7 @@
*
* @param documentURI is the URI of the document. The URI is presumed to be the unique identifier which the output data store will use to process and serve the document. This URI is
* constructed by the repository connector which fetches the document, and is thus universal across all output connectors.
- * @param pipelineDescription is the description string that was constructed for this document by the getOutputDescription() method.
+ * @param pipelineDescription is the description string that was constructed for this document by the getOutputDescription() method.
* @param document is the document data to be processed (handed to the output data store).
* @param authorityNameString is the name of the authority responsible for authorizing any access tokens passed in with the repository document. May be null.
* @param activities is the handle to an object that the implementer of a pipeline connector may use to perform operations, such as logging processing activity, or sending a modified
@@ -772,42 +775,122 @@
final JsonFactory jfactory = new JsonFactory();
final JsonParser jParser = jfactory.createParser(is);
- JsonToken token;
- int cptObj = 0;
- while ((token = jParser.nextToken()) != null) {
+ JsonToken token = null;
+
+ // Go to beginning of metadata
+ boolean inMetadata = false;
+ while (!inMetadata && (token = jParser.nextToken()) != null) {
if (token == JsonToken.START_OBJECT) {
- cptObj++;
- }
- final String fieldName = jParser.getCurrentName();
- if (fieldName != null) {
- if (fieldName.contentEquals("X-TIKA:content")) {
- jParser.nextToken();
- length += jParser.getText(w);
- } else if (cptObj == 1 && !fieldName.startsWith("X-TIKA") && !fieldName.startsWith("X-Parsed-By")) {
- token = jParser.nextToken();
- if (!metadata.containsKey(fieldName)) {
- metadata.put(fieldName, new ArrayList<>());
- }
- if (token == JsonToken.START_ARRAY) {
- while (jParser.nextToken() != JsonToken.END_ARRAY) {
- metadata.get(fieldName).add(jParser.getText());
- }
- } else {
- metadata.get(fieldName).add(jParser.getText());
- }
- } else if (fieldName.contentEquals("X-TIKA:EXCEPTION:write_limit_reached")) {
- resultCode = "TRUNCATEDOK";
- truncated = true;
- } else if (fieldName.contentEquals("X-TIKA:EXCEPTION:embedded_resource_limit_reached")) {
- resources_limit = true;
- } else if (fieldName.startsWith("X-TIKA:EXCEPTION:")) {
- resultCode = "TIKAEXCEPTION";
- jParser.nextToken();
- description += fieldName + ": " + jParser.getText() + System.lineSeparator();
- }
+ inMetadata = true;
}
}
- jParser.close();
+
+ int totalMetadataLength = 0;
+ boolean maxMetadataReached = false;
+ boolean metadataSkipped = false;
+ boolean metadataTruncated = false;
+
+ if (token != null) {
+ while ((token = jParser.nextToken()) != null && token != JsonToken.END_OBJECT) {
+
+ final int fieldNameLength = jParser.getTextLength();
+ if (fieldNameLength <= maxMetadataNameLength) {
+ final String fieldName = jParser.getCurrentName();
+ if (fieldName != null) {
+ if (fieldName.startsWith("X-Parsed-By")) {
+ skipMetadata(jParser);
+ } else if (fieldName.contentEquals("X-TIKA:content")) {
+ // Consume content
+ jParser.nextToken();
+ length += jParser.getText(w);
+ } else if (!fieldName.startsWith("X-TIKA")) {
+ token = jParser.nextToken();
+ if (!metadata.containsKey(fieldName)) {
+ totalMetadataLength += fieldName.length();
+ metadata.put(fieldName, new ArrayList<>());
+ }
+ if (token == JsonToken.START_ARRAY) {
+ while (jParser.nextToken() != JsonToken.END_ARRAY) {
+ if (jParser.getTextLength() <= sp.maxMetadataValueLength) {
+ final int totalMetadataLengthPreview = totalMetadataLength + jParser.getTextLength();
+ if (totalMetadataLengthPreview <= sp.totalMetadataLimit) {
+ metadata.get(fieldName).add(jParser.getText());
+ totalMetadataLength = totalMetadataLengthPreview;
+ } else {
+ maxMetadataReached = true;
+ }
+ } else {
+ metadataSkipped = true;
+ if (Logging.ingest.isDebugEnabled()) {
+ Logging.ingest
+ .debug("Skip value of metadata " + fieldName + " of document " + documentURI + " because it exceeds the max value limit of " + sp.maxMetadataValueLength);
+ }
+ }
+ }
+ } else {
+ if (jParser.getTextLength() <= sp.maxMetadataValueLength) {
+ final int totalMetadataLengthPreview = totalMetadataLength + jParser.getTextLength();
+ if (totalMetadataLengthPreview <= sp.totalMetadataLimit) {
+ metadata.get(fieldName).add(jParser.getText());
+ } else {
+ maxMetadataReached = true;
+ }
+ } else {
+ metadataSkipped = true;
+ if (Logging.ingest.isDebugEnabled()) {
+ Logging.ingest
+ .debug("Skip value of metadata " + fieldName + " of document " + documentURI + " because it exceeds the max value limit of " + sp.maxMetadataValueLength);
+ }
+ }
+ }
+ // Remove metadata if no data has been gathered
+ if (metadata.get(fieldName).isEmpty()) {
+ totalMetadataLength -= fieldName.length();
+ metadata.remove(fieldName);
+ }
+ } else if (fieldName.startsWith("X-TIKA:EXCEPTION:")) {
+ boolean unknownException = false;
+ if (fieldName.contentEquals("X-TIKA:EXCEPTION:write_limit_reached")) {
+ resultCode = "TRUNCATEDOK";
+ truncated = true;
+ } else if (fieldName.contentEquals("X-TIKA:EXCEPTION:embedded_resource_limit_reached")) {
+ resources_limit = true;
+ } else {
+ unknownException = true;
+ resultCode = "TIKAEXCEPTION";
+ jParser.nextToken();
+ description += fieldName + ": " + jParser.getText() + System.lineSeparator();
+ }
+ if (!unknownException) {
+ skipMetadata(jParser);
+ }
+ } else if (fieldName.startsWith("X-TIKA:WARN:truncated_metadata")) {
+ metadataTruncated = true;
+ skipMetadata(jParser);
+ } else {
+ skipMetadata(jParser);
+ }
+ }
+ } else {
+ metadataSkipped = true;
+ if (Logging.ingest.isDebugEnabled()) {
+ Logging.ingest.debug("Skip a metadata of document " + documentURI + " because its name exceeds the max allowed length of " + maxMetadataNameLength);
+ }
+ skipMetadata(jParser);
+ }
+ }
+ jParser.close();
+ }
+
+ if (maxMetadataReached) {
+ description += "Some metadata have been skipped because the total metadata limit of " + sp.totalMetadataLimit + " has been reached" + System.lineSeparator();
+ } else if (metadataSkipped) {
+ description += "Some metadata have been skipped because their names or values exceeded the limits" + System.lineSeparator();
+ }
+
+ if (metadataTruncated) {
+ description += "Some metadata have been truncated by Tika because they exceeded the limits specified in the Tika conf" + System.lineSeparator();
+ }
}
} else if (responseCode == 503) {
// Service interruption; Tika trying to come up.
@@ -880,7 +963,7 @@
String[] values = metadata.get(mName).toArray(new String[0]);
// Only keep metadata if its name does not exceed 8k chars to avoid HTTP header error
- if (mName.length() < 8000) {
+ if (mName.length() < maxMetadataNameLength) {
if (sp.lowerNames()) {
final StringBuilder sb = new StringBuilder();
for (int i = 0; i < mName.length(); i++) {
@@ -950,6 +1033,20 @@
}
+ private void skipMetadata(final JsonParser jParser) throws IOException {
+ JsonToken token = jParser.nextToken();
+ if (token == JsonToken.START_OBJECT) {
+ while (token != JsonToken.END_OBJECT) {
+ token = jParser.nextToken();
+ }
+ }
+ if (token == JsonToken.START_ARRAY) {
+ while (token != JsonToken.END_ARRAY) {
+ token = jParser.nextToken();
+ }
+ }
+ }
+
private void removeField(final RepositoryDocument document, final String fieldName) {
final Iterator<String> fields = document.getFields();
while (fields.hasNext()) {
@@ -1035,15 +1132,15 @@
tabsArray.add(Messages.getString(locale, "TikaExtractor.TikaServerTabName"));
// Fill in the specification header map, using data from all tabs.
- fillInFieldMappingSpecificationMap(paramMap, os);
+ fillInTikaSpecificationMap(paramMap, os);
Messages.outputResourceWithVelocity(out, locale, EDIT_SPECIFICATION_JS, paramMap);
}
/**
* Output the specification body section. This method is called in the body section of a job page which has selected a pipeline connection of the current type. Its purpose is to present the required
- * form elements for editing. The coder can presume that the HTML that is output from this configuration will be within appropriate <html>, <body>, and <form> tags. The name of the form is
- * "editjob".
+ * form elements for editing. The coder can presume that the HTML that is output from this configuration will be within appropriate <html>, <body>, and <form> tags. The name of the
+ * form is "editjob".
*
* @param out is the output to which any HTML should be sent.
* @param locale is the preferred local of the output.
@@ -1063,7 +1160,7 @@
paramMap.put("SELECTEDNUM", Integer.toString(actualSequenceNumber));
// Fill in the field mapping tab data
- fillInFieldMappingSpecificationMap(paramMap, os);
+ fillInTikaSpecificationMap(paramMap, os);
Messages.outputResourceWithVelocity(out, locale, EDIT_SPECIFICATION_TIKASERVER_HTML, paramMap);
}
@@ -1160,6 +1257,24 @@
}
os.addChild(os.getChildCount(), node3);
+ final SpecificationNode maxMetadataValueNode = new SpecificationNode(TikaConfig.NODE_MAXMETADATAVALUELENGTH);
+ final String maxMetadataValue = variableContext.getParameter(seqPrefix + "maxmetadatavaluelength");
+ if (maxMetadataValue != null) {
+ maxMetadataValueNode.setAttribute(TikaConfig.ATTRIBUTE_VALUE, maxMetadataValue);
+ } else {
+ maxMetadataValueNode.setAttribute(TikaConfig.ATTRIBUTE_VALUE, TikaConfig.MAXMETADATAVALUELENGTH_DEFAULT);
+ }
+ os.addChild(os.getChildCount(), maxMetadataValueNode);
+
+ final SpecificationNode metadataLimitNode = new SpecificationNode(TikaConfig.NODE_TOTALMETADATALIMIT);
+ final String metadataLimit = variableContext.getParameter(seqPrefix + "totalmetadatalimit");
+ if (metadataLimit != null) {
+ metadataLimitNode.setAttribute(TikaConfig.ATTRIBUTE_VALUE, metadataLimit);
+ } else {
+ metadataLimitNode.setAttribute(TikaConfig.ATTRIBUTE_VALUE, TikaConfig.TOTALMETADATALIMIT_DEFAULT);
+ }
+ os.addChild(os.getChildCount(), metadataLimitNode);
+
final SpecificationNode node4 = new SpecificationNode(TikaConfig.NODE_EXTRACTARCHIVES);
final String extractArch = variableContext.getParameter(seqPrefix + TikaConfig.NODE_EXTRACTARCHIVES);
if (extractArch != null) {
@@ -1197,18 +1312,20 @@
paramMap.put("SEQNUM", Integer.toString(connectionSequenceNumber));
// Fill in the map with data from all tabs
- fillInFieldMappingSpecificationMap(paramMap, os);
+ fillInTikaSpecificationMap(paramMap, os);
Messages.outputResourceWithVelocity(out, locale, VIEW_SPECIFICATION_HTML, paramMap);
}
- protected static void fillInFieldMappingSpecificationMap(final Map<String, Object> paramMap, final Specification os) {
+ protected static void fillInTikaSpecificationMap(final Map<String, Object> paramMap, final Specification os) {
// Prep for field mappings
final List<Map<String, String>> fieldMappings = new ArrayList<>();
String keepAllMetadataValue = "true";
String lowernamesValue = "true";
String writeLimitValue = "1000000"; // 1Mo by default
+ String maxMetadataValueLength = "250000"; // 250ko by default
+ String totalMetadataLimit = "500000"; // 500ko by default
String extractArchives = "false";
String maxEmbeddedResources = "";
for (int i = 0; i < os.getChildCount(); i++) {
@@ -1238,12 +1355,18 @@
extractArchives = sn.getAttributeValue(TikaConfig.ATTRIBUTE_VALUE);
} else if (sn.getType().equals(TikaConfig.NODE_MAXEMBEDDEDRESOURCES)) {
maxEmbeddedResources = sn.getAttributeValue(TikaConfig.ATTRIBUTE_VALUE);
+ } else if (sn.getType().equals(TikaConfig.NODE_MAXMETADATAVALUELENGTH)) {
+ maxMetadataValueLength = sn.getAttributeValue(TikaConfig.ATTRIBUTE_VALUE);
+ } else if (sn.getType().equals(TikaConfig.NODE_TOTALMETADATALIMIT)) {
+ totalMetadataLimit = sn.getAttributeValue(TikaConfig.ATTRIBUTE_VALUE);
}
}
paramMap.put("FIELDMAPPINGS", fieldMappings);
paramMap.put("KEEPALLMETADATA", keepAllMetadataValue);
paramMap.put("LOWERNAMES", lowernamesValue);
paramMap.put("WRITELIMIT", writeLimitValue);
+ paramMap.put("MAXMETADATAVALUELENGTH", maxMetadataValueLength);
+ paramMap.put("TOTALMETADATALIMIT", totalMetadataLimit);
paramMap.put("EXTRACTARCHIVES", extractArchives);
paramMap.put("MAXEMBEDDEDRESOURCES", maxEmbeddedResources);
}
@@ -1402,6 +1525,8 @@
private final int writeLimit;
private final boolean extractArchives;
private final int maxEmbeddedResources;
+ private final int maxMetadataValueLength;
+ private final int totalMetadataLimit;
public SpecPacker(final Specification os) {
boolean keepAllMetadata = true;
@@ -1409,6 +1534,9 @@
boolean extractArchives = true;
int writeLimit = TikaConfig.WRITELIMIT_DEFAULT;
int maxEmbeddedResources = TikaConfig.MAXEMBEDDEDRESOURCES_DEFAULT;
+ int maxMetadataValueLength = Integer.parseInt(TikaConfig.MAXMETADATAVALUELENGTH_DEFAULT);
+ int totalMetadataLimit = Integer.parseInt(TikaConfig.TOTALMETADATALIMIT_DEFAULT);
+
for (int i = 0; i < os.getChildCount(); i++) {
final SpecificationNode sn = os.getChild(i);
@@ -1425,6 +1553,20 @@
} else {
writeLimit = Integer.parseInt(value);
}
+ } else if (sn.getType().equals(TikaConfig.NODE_MAXMETADATAVALUELENGTH)) {
+ final String value = sn.getAttributeValue(TikaConfig.ATTRIBUTE_VALUE);
+ if (value.length() == 0) {
+ maxMetadataValueLength = Integer.parseInt(TikaConfig.MAXMETADATAVALUELENGTH_DEFAULT);
+ } else {
+ maxMetadataValueLength = Integer.parseInt(value);
+ }
+ } else if (sn.getType().equals(TikaConfig.NODE_TOTALMETADATALIMIT)) {
+ final String value = sn.getAttributeValue(TikaConfig.ATTRIBUTE_VALUE);
+ if (value.length() == 0) {
+ totalMetadataLimit = Integer.parseInt(TikaConfig.TOTALMETADATALIMIT_DEFAULT);
+ } else {
+ totalMetadataLimit = Integer.parseInt(value);
+ }
} else if (sn.getType().equals(TikaConfig.NODE_EXTRACTARCHIVES)) {
final String value = sn.getAttributeValue(TikaConfig.ATTRIBUTE_VALUE);
extractArchives = Boolean.parseBoolean(value);
@@ -1448,6 +1590,11 @@
this.keepAllMetadata = keepAllMetadata;
this.lowerNames = lowerNames;
this.writeLimit = writeLimit;
+ if (maxMetadataValueLength > totalMetadataLimit) {
+ maxMetadataValueLength = totalMetadataLimit;
+ }
+ this.maxMetadataValueLength = maxMetadataValueLength;
+ this.totalMetadataLimit = totalMetadataLimit;
this.extractArchives = extractArchives;
this.maxEmbeddedResources = maxEmbeddedResources;
}
@@ -1493,6 +1640,16 @@
sb.append(writeLimit);
}
+ if (maxMetadataValueLength != Integer.parseInt(TikaConfig.MAXMETADATAVALUELENGTH_DEFAULT)) {
+ sb.append('+');
+ sb.append(maxMetadataValueLength);
+ }
+
+ if (totalMetadataLimit != Integer.parseInt(TikaConfig.TOTALMETADATALIMIT_DEFAULT)) {
+ sb.append('+');
+ sb.append(totalMetadataLimit);
+ }
+
if (extractArchives) {
sb.append('+');
} else {
diff --git a/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_en_US.properties b/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_en_US.properties
index 8447ebf..941acc0 100644
--- a/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_en_US.properties
+++ b/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_en_US.properties
@@ -29,7 +29,7 @@
TikaExtractor.NoFieldMappingSpecified=No field mapping specified
TikaExtractor.KeepAllMetadata=Keep all metadata:
TikaExtractor.LowerNames=Normalize metadata names:
-TikaExtractor.WriteLimit=Content write limit:
+TikaExtractor.WriteLimit=Content write limit (bytes):
TikaExtractor.MaxEmbeddedResources=Max embedded resources process limit:
TikaExtractor.Add=Add
TikaExtractor.AddFieldMapping=Add field mapping
@@ -39,3 +39,5 @@
TikaExtractor.PleaseSupplyAValidNumber=Please supply a valid number
TikaExtractor.PleaseDontSpecifyAnyServerPathInformation=Please don't include server path information in host name
TikaExtractor.HostNameCannotBeNull=Host name cannot be null
+TikaExtractor.MaxMetadataValueLength=Maximum length per metadata value (bytes):
+TikaExtractor.TotalMetadataLimit=Total metadata limit (bytes):
diff --git a/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_es_ES.properties b/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_es_ES.properties
index 8447ebf..941acc0 100644
--- a/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_es_ES.properties
+++ b/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_es_ES.properties
@@ -29,7 +29,7 @@
TikaExtractor.NoFieldMappingSpecified=No field mapping specified
TikaExtractor.KeepAllMetadata=Keep all metadata:
TikaExtractor.LowerNames=Normalize metadata names:
-TikaExtractor.WriteLimit=Content write limit:
+TikaExtractor.WriteLimit=Content write limit (bytes):
TikaExtractor.MaxEmbeddedResources=Max embedded resources process limit:
TikaExtractor.Add=Add
TikaExtractor.AddFieldMapping=Add field mapping
@@ -39,3 +39,5 @@
TikaExtractor.PleaseSupplyAValidNumber=Please supply a valid number
TikaExtractor.PleaseDontSpecifyAnyServerPathInformation=Please don't include server path information in host name
TikaExtractor.HostNameCannotBeNull=Host name cannot be null
+TikaExtractor.MaxMetadataValueLength=Maximum length per metadata value (bytes):
+TikaExtractor.TotalMetadataLimit=Total metadata limit (bytes):
diff --git a/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_fr_FR.properties b/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_fr_FR.properties
index 7b25be2..50c83b6 100644
--- a/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_fr_FR.properties
+++ b/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_fr_FR.properties
@@ -29,7 +29,7 @@
TikaExtractor.NoFieldMappingSpecified=No field mapping specified
TikaExtractor.KeepAllMetadata=Conserver toutes les métadonnées:
TikaExtractor.LowerNames=Normaliser les noms des métadonnées:
-TikaExtractor.WriteLimit=Limite d'écriture du contenu:
+TikaExtractor.WriteLimit=Limite d'écriture du contenu (bytes):
TikaExtractor.MaxEmbeddedResources=Limite du nombre maximal de ressources intégrées à traiter:
TikaExtractor.Add=Ajouter
TikaExtractor.AddFieldMapping=Add field mapping
@@ -39,3 +39,5 @@
TikaExtractor.PleaseSupplyAValidNumber=Veuillez renseigner un nombre valide
TikaExtractor.PleaseDontSpecifyAnyServerPathInformation=Please don't include server path information in host name
TikaExtractor.HostNameCannotBeNull=Le nom d'hôte doit être renseigné
+TikaExtractor.MaxMetadataValueLength=Longueur maximum par valeur de métadonnée (bytes):
+TikaExtractor.TotalMetadataLimit=Limite du total de métadonnées (bytes):
diff --git a/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_ja_JP.properties b/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_ja_JP.properties
index 3a3bafc..1e3f7e7 100644
--- a/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_ja_JP.properties
+++ b/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_ja_JP.properties
@@ -39,3 +39,5 @@
TikaExtractor.PleaseSupplyAValidNumber=Please supply a valid number
TikaExtractor.PleaseDontSpecifyAnyServerPathInformation=Please don't include server path information in host name
TikaExtractor.HostNameCannotBeNull=Host name cannot be null
+TikaExtractor.MaxMetadataValueLength=Maximum length per metadata value (bytes):
+TikaExtractor.TotalMetadataLimit=Total metadata limit (bytes):
diff --git a/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_zh_CN.properties b/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_zh_CN.properties
index 6ea20b4..9c7e2f8 100644
--- a/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_zh_CN.properties
+++ b/connectors/tikaservice-rmeta/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/common_zh_CN.properties
@@ -39,3 +39,5 @@
TikaExtractor.PleaseSupplyAValidNumber=Please supply a valid number
TikaExtractor.PleaseDontSpecifyAnyServerPathInformation=Please don't include server path information in host name
TikaExtractor.HostNameCannotBeNull=Host name cannot be null
+TikaExtractor.MaxMetadataValueLength=Maximum length per metadata value (bytes):
+TikaExtractor.TotalMetadataLimit=Total metadata limit (bytes):
diff --git a/connectors/tikaservice-rmeta/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/editSpecification_TikaServer.html b/connectors/tikaservice-rmeta/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/editSpecification_TikaServer.html
index 74c1873..3afbaad 100644
--- a/connectors/tikaservice-rmeta/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/editSpecification_TikaServer.html
+++ b/connectors/tikaservice-rmeta/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/editSpecification_TikaServer.html
@@ -111,6 +111,20 @@
</tr>
<tr>
+ <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('TikaExtractor.MaxMetadataValueLength'))</nobr></td>
+ <td class="value"><input name="s${SEQNUM}_maxmetadatavaluelength" type="text"
+ value="$Encoder.attributeEscape($MAXMETADATAVALUELENGTH)" size="16" />
+ </td>
+ </tr>
+
+ <tr>
+ <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('TikaExtractor.TotalMetadataLimit'))</nobr></td>
+ <td class="value"><input name="s${SEQNUM}_totalmetadatalimit" type="text"
+ value="$Encoder.attributeEscape($TOTALMETADATALIMIT)" size="16" />
+ </td>
+ </tr>
+
+ <tr>
<td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('TikaExtractor.ExtractArchives'))</nobr></td>
<td class="value">
#if($EXTRACTARCHIVES == 'true')
@@ -141,6 +155,8 @@
<input type="hidden" name="s${SEQNUM}_keepallmetadata" value="$Encoder.bodyEscape($KEEPALLMETADATA)"/>
<input type="hidden" name="s${SEQNUM}_lowernames" value="$Encoder.bodyEscape($LOWERNAMES)"/>
<input type="hidden" name="s${SEQNUM}_writelimit" value="$Encoder.attributeEscape($WRITELIMIT)" />
+<input type="hidden" name="s${SEQNUM}_maxmetadatavaluelength" value="$Encoder.attributeEscape($MAXMETADATAVALUELENGTH)" />
+<input type="hidden" name="s${SEQNUM}_totalmetadatalimit" value="$Encoder.attributeEscape($TOTALMETADATALIMIT)" />
<input type="hidden" name="s${SEQNUM}_extractArchives" value="$Encoder.bodyEscape($EXTRACTARCHIVES)" />
<input type="hidden" name="s${SEQNUM}_maxEmbeddedResources" value="$Encoder.bodyEscape($MAXEMBEDDEDRESOURCES)" />
diff --git a/connectors/tikaservice-rmeta/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/viewSpecification.html b/connectors/tikaservice-rmeta/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/viewSpecification.html
index c8cc87a..bebbd50 100644
--- a/connectors/tikaservice-rmeta/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/viewSpecification.html
+++ b/connectors/tikaservice-rmeta/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tikaservice/rmeta/viewSpecification.html
@@ -61,6 +61,14 @@
<td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('TikaExtractor.WriteLimit'))</nobr></td>
<td class="value"><nobr>$Encoder.bodyEscape($WRITELIMIT)</nobr></td>
</tr>
+ <tr>
+ <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('TikaExtractor.MaxMetadataValueLength'))</nobr></td>
+ <td class="value"><nobr>$Encoder.bodyEscape($MAXMETADATAVALUELENGTH)</nobr></td>
+ </tr>
+ <tr>
+ <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('TikaExtractor.TotalMetadataLimit'))</nobr></td>
+ <td class="value"><nobr>$Encoder.bodyEscape($TOTALMETADATALIMIT)</nobr></td>
+ </tr>
<tr><td class="separator" colspan="2"><hr/></td></tr>
<tr>
<td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('TikaExtractor.ExtractArchives'))</nobr></td>
diff --git a/connectors/tikaservice-rmeta/pom.xml b/connectors/tikaservice-rmeta/pom.xml
index a835e62..1b14347 100755
--- a/connectors/tikaservice-rmeta/pom.xml
+++ b/connectors/tikaservice-rmeta/pom.xml
@@ -4,7 +4,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -86,11 +86,6 @@
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-core</artifactId>
- <version>${tika.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>${httpcomponent.httpclient.version}</version>
diff --git a/connectors/tikaservice/pom.xml b/connectors/tikaservice/pom.xml
index cc8b273..e52a5be 100644
--- a/connectors/tikaservice/pom.xml
+++ b/connectors/tikaservice/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java b/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java
index ad97f46..af51017 100644
--- a/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java
+++ b/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java
@@ -545,7 +545,7 @@
while (index < list.size())
{
String urlCandidate = (String)list.get(index++);
- String documentIdentifier = makeDocumentIdentifier(null,urlCandidate,filter);
+ String documentIdentifier = makeDocumentIdentifier(null,urlCandidate,filter, activities);
if (documentIdentifier == null)
{
// Bad seed. Log it, and continue!
@@ -616,7 +616,7 @@
for (String documentIdentifier : documentIdentifiers)
{
// Verify that the url is legal
- if (!filter.isDocumentAndHostLegal(documentIdentifier))
+ if (!filter.isDocumentAndHostLegal(documentIdentifier,activities))
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("WEB: Removing url '"+documentIdentifier+"' because it's not in the set of allowed ones");
@@ -1320,7 +1320,7 @@
return;
}
- String ingestURL = filter.isDocumentIndexable(documentIdentifier);
+ String ingestURL = filter.isDocumentIndexable(documentIdentifier,activities);
if (ingestURL == null)
{
if (Logging.connectors.isDebugEnabled())
@@ -3400,7 +3400,7 @@
*@param filter the filter object, used to remove unmatching URLs.
*@return the canonical URL (the document identifier), or null if the url was illegal.
*/
- protected String makeDocumentIdentifier(String parentIdentifier, String rawURL, DocumentURLFilter filter)
+ protected String makeDocumentIdentifier(String parentIdentifier, String rawURL, DocumentURLFilter filter, IHistoryActivity activities)
throws ManifoldCFException
{
try
@@ -3462,7 +3462,7 @@
}
// Check to be sure the canonicalized URL is in fact one of the ones we want to include
- if (!filter.isDocumentLegal(id))
+ if (!filter.isDocumentLegal(id, activities))
return null;
return id;
@@ -3897,7 +3897,7 @@
public void noteDiscoveredBase(String rawURL)
throws ManifoldCFException
{
- String newIdentifier = makeDocumentIdentifier(baseDocumentIdentifier,rawURL,filter);
+ String newIdentifier = makeDocumentIdentifier(baseDocumentIdentifier,rawURL,filter,activities);
if (newIdentifier != null)
baseDocumentIdentifier = newIdentifier;
}
@@ -3909,7 +3909,7 @@
public void noteDiscoveredLink(String rawURL)
throws ManifoldCFException
{
- String newIdentifier = makeDocumentIdentifier(baseDocumentIdentifier,rawURL,filter);
+ String newIdentifier = makeDocumentIdentifier(baseDocumentIdentifier,rawURL,filter,activities);
if (newIdentifier != null)
{
if (Logging.connectors.isDebugEnabled())
@@ -5908,10 +5908,11 @@
}
/** Check if both a document and host are legal.
+ * @throws ManifoldCFException
*/
- public boolean isDocumentAndHostLegal(String url)
+ public boolean isDocumentAndHostLegal(String url, IHistoryActivity activities) throws ManifoldCFException
{
- if (!isDocumentLegal(url))
+ if (!isDocumentLegal(url, activities))
return false;
if (seedHosts == null)
return true;
@@ -5944,8 +5945,9 @@
}
/** Check if the document identifier is legal.
+ * @throws ManifoldCFException
*/
- public boolean isDocumentLegal(String url)
+ public boolean isDocumentLegal(String url, IHistoryActivity activities) throws ManifoldCFException
{
// First, verify that the url matches one of the patterns in the include list.
int i = 0;
@@ -5961,6 +5963,7 @@
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("WEB: Url '"+url+"' is illegal because no include patterns match it");
+ activities.recordActivity(System.currentTimeMillis(), ACTIVITY_FETCH, null, url, "EXCLUDED", "URL has been excluded as it does not match any include filter", null);
return false;
}
@@ -5974,6 +5977,7 @@
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("WEB: Url '"+url+"' is illegal because exclude pattern '"+p.toString()+"' matched it");
+ activities.recordActivity(System.currentTimeMillis(), ACTIVITY_FETCH, null, url, "EXCLUDED", "URL has been excluded as the exclude pattern " + p.toString() + " matched it", null);
return false;
}
i++;
@@ -5985,7 +5989,7 @@
/** Check if the document identifier is indexable, and return the indexing URL if found.
* @return null if the url doesn't match or should not be ingested, or the new string if it does.
*/
- public String isDocumentIndexable(String url)
+ public String isDocumentIndexable(String url, IProcessActivity activities)
throws ManifoldCFException
{
// First, verify that the url matches one of the patterns in the include list.
@@ -6002,6 +6006,7 @@
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("WEB: Url '"+url+"' is not indexable because no include patterns match it");
+ activities.recordActivity(System.currentTimeMillis(), ACTIVITY_FETCH, null, url, "EXCLUDED", "URL has been excluded as it does not match any include filter", null);
return null;
}
@@ -6015,6 +6020,7 @@
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("WEB: Url '"+url+"' is not indexable because exclude pattern '"+p.toString()+"' matched it");
+ activities.recordActivity(System.currentTimeMillis(), ACTIVITY_FETCH, null, url, "EXCLUDED", "URL has been excluded as the exclude pattern " + p.toString() + " matched it", null);
return null;
}
i++;
@@ -6025,6 +6031,7 @@
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("WEB: Url '"+url+"' is not indexable because it did not match a mapping rule");
+ activities.recordActivity(System.currentTimeMillis(), ACTIVITY_FETCH, null, url, "EXCLUDED", "URL has been excluded because it did not match a mapping rule", null);
}
return rval;
diff --git a/connectors/webcrawler/pom.xml b/connectors/webcrawler/pom.xml
index 8b3ad02..08284ab 100644
--- a/connectors/webcrawler/pom.xml
+++ b/connectors/webcrawler/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/wiki/pom.xml b/connectors/wiki/pom.xml
index c763628..a9a16fb 100644
--- a/connectors/wiki/pom.xml
+++ b/connectors/wiki/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/agents/pom.xml b/framework/agents/pom.xml
index e3ef7ae..856117d 100644
--- a/framework/agents/pom.xml
+++ b/framework/agents/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/api-service/pom.xml b/framework/api-service/pom.xml
index 3794ea9..3d5ed3f 100644
--- a/framework/api-service/pom.xml
+++ b/framework/api-service/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/api-servlet/pom.xml b/framework/api-servlet/pom.xml
index 7b66103..2465535 100644
--- a/framework/api-servlet/pom.xml
+++ b/framework/api-servlet/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/authority-service/pom.xml b/framework/authority-service/pom.xml
index d2a4a37..cae7d6d 100644
--- a/framework/authority-service/pom.xml
+++ b/framework/authority-service/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/authority-servlet/pom.xml b/framework/authority-servlet/pom.xml
index 5cfc28d..7c157a7 100644
--- a/framework/authority-servlet/pom.xml
+++ b/framework/authority-servlet/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/build.xml b/framework/build.xml
index 14eff01..8f93f4e 100644
--- a/framework/build.xml
+++ b/framework/build.xml
@@ -120,6 +120,7 @@
<include name="sis-utility*.jar"/>
<include name="sis-netcdf*.jar"/>
<include name="sis-storage*.jar"/>
+ <include name="sis-feature*.jar"/>
<include name="unit-api*.jar"/>
<include name="opennlp-tools*.jar"/>
<include name="openjson*.jar"/>
@@ -155,15 +156,21 @@
<include name="pdfbox*.jar"/>
<include name="fontbox*.jar"/>
<include name="jempbox*.jar"/>
+ <include name="xmpbox*.jar"/>
+ <include name="preflight*.jar"/>
<include name="bcmail-jdk15on*.jar"/>
+ <include name="bcutil-jdk15on*.jar"/>
<include name="bcprov-jdk15on*.jar"/>
<include name="bcpkix-jdk15on*.jar"/>
+ <include name="SparseBitSet*.jar"/>
+ <include name="HikariCP-java7*.jar"/>
<include name="geronimo-stax-api_1.0_spec*.jar"/>
<include name="asm*.jar"/>
<include name="isoparser*.jar"/>
<include name="aspectjrt*.jar"/>
- <include name="metadata-extractor*.jar"/>
<include name="xmpcore*.jar"/>
+ <include name="mchange-commons-java*.jar"/>
+ <include name="dd-plist*.jar"/>
<include name="xml-apis*.jar"/>
<include name="boilerpipe*.jar"/>
<include name="rome*.jar"/>
@@ -173,16 +180,22 @@
<include name="juniversalchardet*.jar"/>
<include name="jhighlight*.jar"/>
<include name="jmatio*.jar"/>
+ <include name="metadata-extractor*.jar"/>
<include name="java-libpst*.jar"/>
<include name="xz*.jar"/>
<include name="parso*.jar"/>
<include name="dec*.jar"/>
<include name="cxf-*.jar"/>
+ <include name="policy*.jar"/>
+ <include name="gmbal*.jar"/>
+ <include name="pfl-*.jar"/>
+ <include name="management-api*.jar"/>
+ <include name="stax-ex*.jar"/>
+ <include name="ha-api*.jar"/>
<include name="neethi*.jar"/>
<include name="xml-resolver*.jar"/>
<include name="xmlschema-core*.jar"/>
- <include name="javax.annotation-api*.jar"/>
- <include name="javax.ws.rs-api*.jar"/>
+ <include name="jakarta.*.jar"/>
<include name="jaxb*.jar"/>
<include name="xsom*.jar"/>
<include name="codemodel*.jar"/>
@@ -193,7 +206,10 @@
<include name="txw2*.jar"/>
<include name="istack-commons*.jar"/>
<include name="FastInfoset*.jar"/>
- <include name="stax-ex*.jar"/>
+ <include name="velocity-engine-core*.jar"/>
+ <include name="jaxws-api*.jar"/>
+ <include name="javax.annotation-api*.jar"/>
+ <include name="javax.jws-api*.jar"/>
</fileset>
<fileset dir="../lib">
<include name="castor*.jar"/>
@@ -205,7 +221,7 @@
<include name="wsdl4j*.jar"/>
<include name="xmlsec*.jar"/>
<include name="opensaml*.jar"/>
- <include name="activation*.jar"/>
+ <!--include name="activation*.jar"/-->
</fileset>
</path>
@@ -1599,6 +1615,7 @@
<include name="sis-utility*.jar"/>
<include name="sis-netcdf*.jar"/>
<include name="sis-storage*.jar"/>
+ <include name="sis-feature*.jar"/>
<include name="unit-api*.jar"/>
<include name="opennlp-tools*.jar"/>
<include name="openjson*.jar"/>
@@ -1638,17 +1655,23 @@
<include name="pdfbox*.jar"/>
<include name="fontbox*.jar"/>
<include name="jempbox*.jar"/>
+ <include name="xmpbox*.jar"/>
+ <include name="preflight*.jar"/>
<include name="kafka*.jar"/>
<include name="zookeeper*.jar"/>
<include name="bcmail-jdk15*.jar"/>
+ <include name="bcutil-jdk15*.jar"/>
<include name="bcprov-jdk15*.jar"/>
<include name="bcpkix-jdk15*.jar"/>
+ <include name="SparseBitSet*.jar"/>
+ <include name="HikariCP-java7*.jar"/>
<include name="geronimo-stax-api_1.0_spec*.jar"/>
<include name="asm*.jar"/>
<include name="isoparser*.jar"/>
<include name="aspectjrt*.jar"/>
- <include name="metadata-extractor*.jar"/>
<include name="xmpcore*.jar"/>
+ <include name="mchange-commons-java*.jar"/>
+ <include name="dd-plist*.jar"/>
<include name="xml-apis*.jar"/>
<include name="boilerpipe*.jar"/>
<include name="rome*.jar"/>
@@ -1658,16 +1681,22 @@
<include name="juniversalchardet*.jar"/>
<include name="jhighlight*.jar"/>
<include name="jmatio*.jar"/>
+ <include name="metadata-extractor*.jar"/>
<include name="java-libpst*.jar"/>
<include name="xz*.jar"/>
<include name="parso*.jar"/>
<include name="dec*.jar"/>
<include name="cxf-*.jar"/>
+ <include name="policy*.jar"/>
+ <include name="gmbal*.jar"/>
+ <include name="pfl-*.jar"/>
+ <include name="management-api*.jar"/>
+ <include name="stax-ex*.jar"/>
+ <include name="ha-api*.jar"/>
<include name="neethi*.jar"/>
<include name="xml-resolver*.jar"/>
<include name="xmlschema-core*.jar"/>
- <include name="javax.annotation-api*.jar"/>
- <include name="javax.ws.rs-api*.jar"/>
+ <include name="jakarta.*.jar"/>
<include name="jaxb*.jar"/>
<include name="xsom*.jar"/>
<include name="codemodel*.jar"/>
@@ -1678,7 +1707,10 @@
<include name="txw2*.jar"/>
<include name="istack-commons*.jar"/>
<include name="FastInfoset*.jar"/>
- <include name="stax-ex*.jar"/>
+ <include name="velocity-engine-core*.jar"/>
+ <include name="jaxws-api*.jar"/>
+ <include name="javax.annotation-api*.jar"/>
+ <include name="javax.jws-api*.jar"/>
</fileset>
<fileset dir="../lib">
<include name="castor*.jar"/>
@@ -1690,11 +1722,7 @@
<include name="wsdl4j*.jar"/>
<include name="xmlsec*.jar"/>
<include name="opensaml*.jar"/>
- <include name="activation*.jar"/>
- </fileset>
- <fileset dir="../lib">
- <include name="jaxws-api*.jar"/>
- <include name="javax.jws-api*.jar"/>
+ <!--include name="activation*.jar"/-->
</fileset>
</copy>
</target>
diff --git a/framework/buildfiles/connector-build.xml b/framework/buildfiles/connector-build.xml
index 93119f6..ff4e461 100644
--- a/framework/buildfiles/connector-build.xml
+++ b/framework/buildfiles/connector-build.xml
@@ -227,8 +227,8 @@
<include name="cxf-rt-transports-http*.jar"/>
<include name="cxf-rt-rs-client*.jar"/>
<include name="xmlschema-core*.jar"/>
- <include name="javax.annotation-api*.jar"/>
- <include name="javax.ws.rs-api*.jar"/>
+ <include name="jakarta.annotation-api*.jar"/>
+ <include name="jakarta.ws.rs-api*.jar"/>
</fileset>
<fileset dir="${mcf-dist}/connector-common-lib">
<include name="axis*.jar"/>
@@ -427,6 +427,7 @@
</fileset>
<fileset dir="${mcf-dist}/connector-common-lib">
<include name="cxf*.jar"/>
+ <include name="pfl*.jar"/>
<include name="jaxb*.jar"/>
<include name="wsdl4j*.jar"/>
<include name="woodstox*.jar"/>
@@ -434,12 +435,20 @@
<include name="xmlschema*.jar"/>
<include name="neethi*.jar"/>
<include name="xml-resolver*.jar"/>
+ <include name="codemodel*.jar"/>
+ <include name="xsom*.jar"/>
+ <include name="relaxng-datatype*.jar"/>
+ <include name="rngom*.jar"/>
+ <include name="dtd-parser*.jar"/>
+ <include name="istack-commons-tools*.jar"/>
+ <include name="velocity-engine-core*.jar"/>
</fileset>
<fileset dir="${mcf-dist}/lib">
<include name="velocity*.jar"/>
<include name="commons-collections*.jar"/>
<include name="commons-lang*.jar"/>
<include name="javax.activation*.jar"/>
+ <include name="slf4j*.jar"/>
</fileset>
</classpath>
<jvmarg value="-Dorg.apache.cxf.stax.allowInsecureParser=1"/>
@@ -557,6 +566,9 @@
<include name="commons-discovery*.jar"/>
<include name="commons-logging*.jar"/>
</fileset>
+ <fileset dir="${mcf-dist}/lib">
+ <include name="javax.activation*.jar"/>
+ </fileset>
</classpath>
</javac>
</target>
diff --git a/framework/combined-service/pom.xml b/framework/combined-service/pom.xml
index cc5848f..c3c88aa 100644
--- a/framework/combined-service/pom.xml
+++ b/framework/combined-service/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/connector-common/pom.xml b/framework/connector-common/pom.xml
index 406156e..8d6c296 100644
--- a/framework/connector-common/pom.xml
+++ b/framework/connector-common/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/core/pom.xml b/framework/core/pom.xml
index 7f2d984..6db842e 100644
--- a/framework/core/pom.xml
+++ b/framework/core/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/crawler-ui/pom.xml b/framework/crawler-ui/pom.xml
index c32da9e..4ed5186 100644
--- a/framework/crawler-ui/pom.xml
+++ b/framework/crawler-ui/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/jetty-runner/pom.xml b/framework/jetty-runner/pom.xml
index 28e0696..28608bb 100644
--- a/framework/jetty-runner/pom.xml
+++ b/framework/jetty-runner/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/less-compiler/pom.xml b/framework/less-compiler/pom.xml
index 91421ce..50300de 100644
--- a/framework/less-compiler/pom.xml
+++ b/framework/less-compiler/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/pom.xml b/framework/pom.xml
index 54589ab..c051c72 100644
--- a/framework/pom.xml
+++ b/framework/pom.xml
@@ -20,13 +20,13 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-parent</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
<name>ManifoldCF - Framework</name>
<packaging>pom</packaging>
diff --git a/framework/pull-agent/pom.xml b/framework/pull-agent/pom.xml
index 7ce079e..0854039 100644
--- a/framework/pull-agent/pom.xml
+++ b/framework/pull-agent/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
index fee88ab..7bf1544 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
@@ -25,6 +25,7 @@
import java.util.*;
import java.io.*;
import java.lang.reflect.*;
+import java.text.SimpleDateFormat;
/** This class represents a worker thread. Hundreds of these threads are instantiated in order to
* perform crawling and extraction.
@@ -45,6 +46,8 @@
protected final QueueTracker queueTracker;
/** Process ID */
protected final String processID;
+
+ private static final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss,SSS");
/** Constructor.
*@param id is the worker thread id.
@@ -846,7 +849,7 @@
}
catch (OutOfMemoryError e)
{
- System.err.println("agents process ran out of memory - shutting down");
+ System.err.println(sdf.format(new Date()) + " agents process ran out of memory - shutting down");
e.printStackTrace(System.err);
ManifoldCF.systemExit(-200);
}
diff --git a/framework/script-engine/pom.xml b/framework/script-engine/pom.xml
index 4e55490..93dec20 100644
--- a/framework/script-engine/pom.xml
+++ b/framework/script-engine/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/ui-core/pom.xml b/framework/ui-core/pom.xml
index 926b6e0..e034ad2 100644
--- a/framework/ui-core/pom.xml
+++ b/framework/ui-core/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pom.xml b/pom.xml
index a55a1e3..3928234 100644
--- a/pom.xml
+++ b/pom.xml
@@ -28,7 +28,7 @@
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-parent</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
<name>ManifoldCF</name>
<packaging>pom</packaging>
@@ -84,7 +84,7 @@
<zookeeper.version>3.4.10</zookeeper.version>
<xmlbeans.version>2.6.0</xmlbeans.version>
<poi.version>3.17</poi.version>
- <tika.version>1.21</tika.version>
+ <tika.version>1.28.1</tika.version>
<boilerpipe.version>1.1.0</boilerpipe.version>
<hadoop.version>2.6.0</hadoop.version>
<tomcat.version>6.0.35</tomcat.version>
@@ -95,7 +95,7 @@
<gson.version>2.8.0</gson.version>
<guava.version>21.0</guava.version>
<jsoup.version>1.7.2</jsoup.version>
- <cxf.version>3.3.1</cxf.version>
+ <cxf.version>3.5.0</cxf.version>
<jaxws-ri.version>2.3.2</jaxws-ri.version>
</properties>
diff --git a/test-materials/alfresco-4-war/pom.xml b/test-materials/alfresco-4-war/pom.xml
index 4b0ca95..e08e493 100644
--- a/test-materials/alfresco-4-war/pom.xml
+++ b/test-materials/alfresco-4-war/pom.xml
@@ -14,7 +14,7 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>alfresco-4-war</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
<name>ManifoldCF - Test Materials - Alfresco WAR</name>
<description>Alfresco WAR builder</description>
<packaging>war</packaging>
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-test-materials</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<properties>
diff --git a/test-materials/pom.xml b/test-materials/pom.xml
index 8dfa950..1def6b5 100644
--- a/test-materials/pom.xml
+++ b/test-materials/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-parent</artifactId>
- <version>2.22-SNAPSHOT</version>
+ <version>2.23-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>mcf-test-materials</artifactId>
