Create a branch to work on CONNECTORS-1177
git-svn-id: https://svn.apache.org/repos/asf/manifoldcf/branches/CONNECTORS-1177@1679734 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/CHANGES.txt b/CHANGES.txt
index 75b5b50..4d822e8 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,7 +1,64 @@
ManifoldCF Change Log
$Id$
-======================= 2.1-dev =====================
+======================= 2.2-dev =====================
+
+CONNECTORS-1199: SearchBlox connector formatting non-standard.
+(Karl Wright)
+
+CONNECTORS-1198: SearchBlox connector session management not
+working properly.
+(Timo Selvaraj, Rafa Haro, Karl Wright)
+
+CONNECTORS-1197: Add "windows" file mode to file system output connector.
+(Karl Wright)
+
+CONNECTORS-1195: Fix for NumberFormatException exception thrown in Maximum Bandwidth Report
+(Kishore Kumar)
+
+CONNECTORS-1192: Fix a problem with login-page detection based on
+content. Last line was getting skipped, and infinite amounts of content
+could be buffered in memory. Adopted a compromise that scans an
+overlapping window of minimum size 16K.
+(Karl Wright)
+
+======================= Release 2.1 =====================
+
+CONNECTORS-1191: Unexpected job status 34, due to a race condition.
+Jobs shutting down and being aborted at the same time could wind up
+in a state where repeated exceptions would be thrown.
+(Arcadius Ahouansou, Karl Wright)
+
+CONNECTORS-1190: Broken dependencies in solr connector maven build
+caused tests to fail under maven.
+(Piergiorgio Lucidi, Karl Wright)
+
+CONNECTORS-1187: Handle "File in Use" error properly in JCIFS connector.
+(Karl Wright)
+
+CONNECTORS-1186: HttpClient 4.4 requires that you set the SSL socket
+factory at the connection manager level.
+(Karl Wright)
+
+CONNECTORS-1185: Fix HttpClient deprecation warnings, and make sure
+socket timeouts are actually honored in http connections.
+(Karl Wright)
+
+CONNECTORS-1183: Fix for GridFS connector broken UI server tab.
+(Kishore Kumar)
+
+CONNECTORS-1173: Option to change View URL for Livelink Connector
+(Kishore Kumar)
+
+CONNECTORS-1179: Include the collection name as part of the Solr
+version string.
+(Kamil Żyta, Karl Wright)
+
+CONNECTORS-1178: Script example broken.
+(Karl Wright)
+
+CONNECTORS-1177: Add session-based security for API.
+(Jan van Haarst, Karl Wright)
CONNECTORS-1176: Update SolrJ version.
(Karl Wright)
diff --git a/build.xml b/build.xml
index 97aba16..af834a5 100644
--- a/build.xml
+++ b/build.xml
@@ -31,9 +31,9 @@
<property name="hsqldb.version" value="2.3.2"/>
<property name="jetty.version" value="9.2.3.v20140905"/>
- <property name="httpclient.version" value="4.3.6"/>
- <property name="httpmime.version" value="4.3.6"/>
- <property name="httpcore.version" value="4.3.3"/>
+ <property name="httpclient.version" value="4.4.1"/>
+ <property name="httpmime.version" value="4.4"/>
+ <property name="httpcore.version" value="4.4.1"/>
<property name="xerces.version" value="2.10.0"/>
<property name="zookeeper.version" value="3.4.6"/>
<property name="mongodb.version" value="2.11.3"/>
@@ -51,7 +51,7 @@
<property name="commons-fileupload.version" value="1.2.2"/>
<property name="commons-io.version" value="2.4"/>
<property name="commons-lang.version" value="2.6"/>
- <property name="commons-logging.version" value="1.1.3"/>
+ <property name="commons-logging.version" value="1.2"/>
<property name="commons-configuration.version" value="1.6"/>
<property name="commons-cli.version" value="1.2"/>
<property name="slf4j.version" value="1.7.7"/>
@@ -473,7 +473,7 @@
<target name="doc" depends="deliver-site-doc"/>
<target name="set-version">
- <property name="release-version" value="2.1-dev"/>
+ <property name="release-version" value="2.2-dev"/>
</target>
<target name="create-source-zip" depends="set-version">
@@ -1671,7 +1671,7 @@
<antcall target="download-via-maven">
<param name="target" value="lib"/>
<param name="project-path" value="org/apache/solr"/>
- <param name="artifact-version" value="5.0.0"/>
+ <param name="artifact-version" value="5.1.0"/>
<param name="artifact-name" value="solr-solrj"/>
<param name="artifact-type" value="jar"/>
</antcall>
diff --git a/connectors/activedirectory/pom.xml b/connectors/activedirectory/pom.xml
index e89e667..acc969d 100644
--- a/connectors/activedirectory/pom.xml
+++ b/connectors/activedirectory/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/alfresco-webscript/pom.xml b/connectors/alfresco-webscript/pom.xml
index fe2af08..2dc2cbe 100644
--- a/connectors/alfresco-webscript/pom.xml
+++ b/connectors/alfresco-webscript/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/alfresco/pom.xml b/connectors/alfresco/pom.xml
index ee255c6..a280aad 100644
--- a/connectors/alfresco/pom.xml
+++ b/connectors/alfresco/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/amazoncloudsearch/pom.xml b/connectors/amazoncloudsearch/pom.xml
index 7959090..11f3ee8 100644
--- a/connectors/amazoncloudsearch/pom.xml
+++ b/connectors/amazoncloudsearch/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/cmis/pom.xml b/connectors/cmis/pom.xml
index b8f67e6..36e1378 100644
--- a/connectors/cmis/pom.xml
+++ b/connectors/cmis/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/documentfilter/pom.xml b/connectors/documentfilter/pom.xml
index df9043b..d1de4a4 100644
--- a/connectors/documentfilter/pom.xml
+++ b/connectors/documentfilter/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/dropbox/pom.xml b/connectors/dropbox/pom.xml
index cd4846f..21b9fd9 100644
--- a/connectors/dropbox/pom.xml
+++ b/connectors/dropbox/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java b/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
index 8f3f6b5..b89ca20 100644
--- a/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
+++ b/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
@@ -113,20 +113,24 @@
{
if (client == null)
{
- connectionManager = new PoolingHttpClientConnectionManager();
-
int socketTimeout = 900000;
int connectionTimeout = 60000;
// Set up connection manager
- connectionManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ connectionManager = poolingConnectionManager;
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
@@ -136,10 +140,6 @@
.setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- //.setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(credentialsProvider)
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
.build();
diff --git a/connectors/elasticsearch/pom.xml b/connectors/elasticsearch/pom.xml
index 7606850..a045327 100644
--- a/connectors/elasticsearch/pom.xml
+++ b/connectors/elasticsearch/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/email/pom.xml b/connectors/email/pom.xml
index 1c0c4fa..dabc9af 100644
--- a/connectors/email/pom.xml
+++ b/connectors/email/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java b/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
index 5739c48..2ab0daf 100644
--- a/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
+++ b/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/agents/output/filesystem/FileOutputConnector.java
@@ -28,6 +28,7 @@
import java.io.StringReader;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.SystemUtils;
import java.net.URI;
import java.net.URISyntaxException;
@@ -629,17 +630,94 @@
*/
final private String documentURItoFilePath(String documentURI) throws URISyntaxException, NullPointerException {
StringBuffer path = new StringBuffer();
- URI uri = null;
-
- uri = new URI(documentURI);
-
+ URI uri = new URI(documentURI);
+ if (SystemUtils.IS_OS_WINDOWS)
+ windowsFileName(path, uri);
+ else
+ unixFileName(path, uri);
+ return path.toString();
+ }
+
+ final private void windowsFileName(final StringBuffer path, final URI uri) {
+ // Illegal characters: \ / : * ? " < > |
+ boolean endsWithSlash = false;
if (uri.getScheme() != null) {
path.append(uri.getScheme());
path.append("/");
+ endsWithSlash = true;
}
if (uri.getHost() != null) {
path.append(uri.getHost());
+ endsWithSlash = false;
+ if (uri.getPort() != -1) {
+ path.append("_");
+ path.append(uri.getPort());
+ }
+ if (uri.getRawPath() != null) {
+ if (uri.getRawPath().length() == 0) {
+ path.append("/");
+ endsWithSlash = true;
+ } else if (uri.getRawPath().equals("/")) {
+ path.append(uri.getRawPath());
+ endsWithSlash = false;
+ } else {
+ for (String name : uri.getRawPath().split("/")) {
+ if (name.length() > 0) {
+ path.append("/");
+ path.append(convertWindowsString(name));
+ endsWithSlash = false;
+ }
+ }
+ }
+ }
+ if (uri.getRawQuery() != null) {
+ path.append("/");
+ path.append(convertWindowsString(uri.getRawQuery()));
+ endsWithSlash = false;
+ }
+ } else {
+ if (uri.getRawSchemeSpecificPart() != null) {
+ for (String name : uri.getRawSchemeSpecificPart().split("/")) {
+ if (name.length() > 0) {
+ path.append("/");
+ path.append(convertWindowsString(name));
+ endsWithSlash = false;
+ }
+ }
+ }
+ }
+
+ if (endsWithSlash) {
+ path.append(".content");
+ }
+ }
+
+ final private String convertWindowsString(final String input) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < input.length(); i++) {
+ char c = input.charAt(i);
+ // Handle filename disallowed special characters!
+ if (c == ':' || c == '/' || c == '\\' | c == '*' | c == '"' | c == '?' | c == '|' || c == '<' || c == '>') {
+ sb.append('_');
+ }
+ else
+ sb.append(c);
+ }
+ return sb.toString();
+ }
+
+ final private void unixFileName(final StringBuffer path, final URI uri) {
+ boolean endsWithSlash = false;
+ if (uri.getScheme() != null) {
+ path.append(uri.getScheme());
+ path.append("/");
+ endsWithSlash = true;
+ }
+
+ if (uri.getHost() != null) {
+ path.append(uri.getHost());
+ endsWithSlash = false;
if (uri.getPort() != -1) {
path.append(":");
path.append(uri.getPort());
@@ -647,39 +725,43 @@
if (uri.getRawPath() != null) {
if (uri.getRawPath().length() == 0) {
path.append("/");
+ endsWithSlash = true;
} else if (uri.getRawPath().equals("/")) {
path.append(uri.getRawPath());
+ endsWithSlash = false;
} else {
for (String name : uri.getRawPath().split("/")) {
if (name.length() > 0) {
path.append("/");
- path.append(convertString(name));
+ path.append(convertUnixString(name));
+ endsWithSlash = false;
}
}
}
}
if (uri.getRawQuery() != null) {
path.append("?");
- path.append(convertString(uri.getRawQuery()));
+ path.append(convertUnixString(uri.getRawQuery()));
+ endsWithSlash = false;
}
} else {
if (uri.getRawSchemeSpecificPart() != null) {
for (String name : uri.getRawSchemeSpecificPart().split("/")) {
if (name.length() > 0) {
path.append("/");
- path.append(convertString(name));
+ path.append(convertUnixString(name));
+ endsWithSlash = false;
}
}
}
}
- if (path.toString().endsWith("/")) {
+ if (endsWithSlash) {
path.append(".content");
}
- return path.toString();
}
- final private String convertString(final String input) {
+ final private String convertUnixString(final String input) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < input.length(); i++) {
char c = input.charAt(i);
diff --git a/connectors/filesystem/pom.xml b/connectors/filesystem/pom.xml
index 8b2e2d4..1e8e4ee 100644
--- a/connectors/filesystem/pom.xml
+++ b/connectors/filesystem/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/forcedmetadata/pom.xml b/connectors/forcedmetadata/pom.xml
index 30883ab..04cf736 100644
--- a/connectors/forcedmetadata/pom.xml
+++ b/connectors/forcedmetadata/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/generic/pom.xml b/connectors/generic/pom.xml
index 12d6128..80a27a8 100644
--- a/connectors/generic/pom.xml
+++ b/connectors/generic/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/googledrive/pom.xml b/connectors/googledrive/pom.xml
index 856ad22..bcaf4e6 100644
--- a/connectors/googledrive/pom.xml
+++ b/connectors/googledrive/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/GridFSRepositoryConnector.java b/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/GridFSRepositoryConnector.java
index 2054997..89d6379 100644
--- a/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/GridFSRepositoryConnector.java
+++ b/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/GridFSRepositoryConnector.java
@@ -34,7 +34,6 @@
import java.util.List;
import java.util.Locale;
import java.util.Map;
-import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
@@ -134,10 +133,6 @@
* GridFS server tab name.
*/
private static final String GRIDFS_SERVER_TAB_RESOURCE = "GridFSConnector.Server";
- /**
- * GridFS credentials tab name.
- */
- private static final String GRIDFS_CREDENTIALS_TAB_RESOURCE = "GridFSConnector.Credentials";
/**
* Tab name parameter for managing the view of the Web UI.
@@ -220,19 +215,19 @@
getSession();
if (session != null) {
Mongo currentMongoSession = session.getMongo();
- DBTCPConnector currentTCPConnection = currentMongoSession.getConnector();
- boolean status = currentTCPConnection.isOpen();
- if (status) {
- session.getMongo().close();
- session = null;
- return super.check();
- } else {
- session = null;
- }
+ currentMongoSession.getConnector()
+ .getDBPortPool(currentMongoSession.getAddress())
+ .get()
+ .ensureOpen();
+ session.getMongo().close();
+ session = null;
+ return super.check();
}
return "Not connected.";
} catch (ManifoldCFException e) {
return e.getMessage();
+ } catch (IOException ex) {
+ return ex.getMessage();
}
}
@@ -246,7 +241,7 @@
public void disconnect() throws ManifoldCFException {
if (session != null) {
try {
- session.getMongo().getConnector().close();
+ session.getMongo().close();
} catch (Exception e) {
Logging.connectors.error("GridFS: Error when trying to disconnect: " + e.getMessage());
throw new ManifoldCFException("GridFS: Error when trying to disconnect: " + e.getMessage(), e);
@@ -280,7 +275,7 @@
long currentTime = System.currentTimeMillis();
if (currentTime >= lastSessionFetch + SESSION_EXPIRATION_MILLISECONDS) {
if (session != null) {
- session.getMongo().getConnector().close();
+ session.getMongo().close();
session = null;
}
lastSessionFetch = -1L;
@@ -326,22 +321,22 @@
/** Queue "seed" documents. Seed documents are the starting places for crawling activity. Documents
* are seeded when this method calls appropriate methods in the passed in ISeedingActivity object.
- *
+ *
* This method can choose to find repository changes that happen only during the specified time interval.
* The seeds recorded by this method will be viewed by the framework based on what the
* getConnectorModel() method returns.
- *
+ *
* It is not a big problem if the connector chooses to create more seeds than are
* strictly necessary; it is merely a question of overall work required.
- *
+ *
* The end time and seeding version string passed to this method may be interpreted for greatest efficiency.
* For continuous crawling jobs, this method will
* be called once, when the job starts, and at various periodic intervals as the job executes.
- *
+ *
* When a job's specification is changed, the framework automatically resets the seeding version string to null. The
* seeding version string may also be set to null on each job run, depending on the connector model returned by
* getConnectorModel().
- *
+ *
* Note that it is always ok to send MORE documents rather than less to this method.
* The connector will be connected before this method can be called.
*@param activities is the interface this method should use to perform whatever framework actions are desired.
@@ -350,11 +345,11 @@
*@param lastSeedVersionString is the last seeding version string for this job, or null if the job has no previous seeding version string.
*@param jobMode is an integer describing how the job is being run, whether continuous or once-only.
*@return an updated seeding version string, to be stored with the job.
- */
+ */
@Override
public String addSeedDocuments(ISeedingActivity activities, Specification spec,
- String lastSeedVersion, long seedTime, int jobMode)
- throws ManifoldCFException, ServiceInterruption {
+ String lastSeedVersion, long seedTime, int jobMode)
+ throws ManifoldCFException, ServiceInterruption {
getSession();
DBCollection fsFiles = session.getCollection(
bucket + GridFSConstants.COLLECTION_SEPERATOR + GridFSConstants.FILES_COLLECTION_NAME
@@ -383,11 +378,11 @@
* and ingest documents.
*@param jobMode is an integer describing how the job is being run, whether continuous or once-only.
*@param usesDefaultAuthority will be true only if the authority in use for these documents is the default one.
- */
+ */
@Override
public void processDocuments(String[] documentIdentifiers, IExistingVersions statuses, Specification spec,
- IProcessActivity activities, int jobMode, boolean usesDefaultAuthority)
- throws ManifoldCFException, ServiceInterruption {
+ IProcessActivity activities, int jobMode, boolean usesDefaultAuthority)
+ throws ManifoldCFException, ServiceInterruption {
for (String documentIdentifier : documentIdentifiers) {
@@ -409,7 +404,7 @@
: StringUtils.EMPTY;
}
- if (versionString.length() == 0 || activities.checkDocumentNeedsReindexing(documentIdentifier,versionString)) {
+ if (versionString.length() == 0 || activities.checkDocumentNeedsReindexing(documentIdentifier, versionString)) {
long startTime = System.currentTimeMillis();
String errorCode = null;
String errorDesc = null;
@@ -533,7 +528,7 @@
handleIOException(e);
}
}
- gfs.getDB().getMongo().getConnector().close();
+ gfs.getDB().getMongo().close();
session = null;
errorCode = "OK";
} else {
@@ -564,7 +559,7 @@
throw new ManifoldCFException(e.getMessage(), e);
}
}
-
+
/**
* Output the configuration header section. This method is called in the
* head section of the connector's configuration page. Its purpose is to add
@@ -584,7 +579,6 @@
@Override
public void outputConfigurationHeader(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters, List<String> tabsArray) throws ManifoldCFException, IOException {
tabsArray.add(Messages.getString(locale, GRIDFS_SERVER_TAB_RESOURCE));
- tabsArray.add(Messages.getString(locale, GRIDFS_CREDENTIALS_TAB_RESOURCE));
Map<String, String> paramMap = new HashMap<String, String>();
fillInServerParameters(paramMap, out, parameters);
diff --git a/connectors/gridfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/gridfs/common_en_US.properties b/connectors/gridfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/gridfs/common_en_US.properties
index 8b739e9..78f7525 100644
--- a/connectors/gridfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/gridfs/common_en_US.properties
+++ b/connectors/gridfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/gridfs/common_en_US.properties
@@ -16,10 +16,10 @@
# Fill in this file with English words and phrases
GridFSConnector.Server=Server
-GridFSConnector.CredentialsColon=Credentials
+GridFSConnector.CredentialsColon=Credentials:
GridFSConnector.ParametersColon=Parameters:
-GridFSConnector.Credentials=Credentials:
+GridFSConnector.Credentials=Credentials
GridFSConnector.UserNameColon=Username:
GridFSConnector.PasswordColon=Password:
diff --git a/connectors/gridfs/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/gridfs/editConfiguration.js b/connectors/gridfs/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/gridfs/editConfiguration.js
index f14995e..197e586 100644
--- a/connectors/gridfs/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/gridfs/editConfiguration.js
+++ b/connectors/gridfs/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/gridfs/editConfiguration.js
@@ -33,13 +33,18 @@
return false;
}
- if (!isInteger(editconnection.port.value))
+ if(editconnection.port.value !== "")
{
- alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('GridFSConnector.PortValueMustBeAValidNumber'))");
- SelectTab("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('GridFSConnector.Server'))");
- editconnection.socketTimeout.focus();
- return false;
+ if (!isInteger(editconnection.port.value))
+ {
+ alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('GridFSConnector.PortValueMustBeAValidNumber'))");
+ SelectTab("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('GridFSConnector.Server'))");
+ editconnection.socketTimeout.focus();
+ return false;
+ }
}
+
+
return true;
}
// -->
diff --git a/connectors/gridfs/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/gridfs/editConfiguration_Server.html b/connectors/gridfs/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/gridfs/editConfiguration_Server.html
index 388c984..384f31b 100644
--- a/connectors/gridfs/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/gridfs/editConfiguration_Server.html
+++ b/connectors/gridfs/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/gridfs/editConfiguration_Server.html
@@ -99,7 +99,7 @@
</nobr>
</td>
<td class="value">
- <input id="url" name="url" type="text" size="10" value="$Encoder.attributeEscape($ACL)" />
+ <input id="acl" name="acl" type="text" size="10" value="$Encoder.attributeEscape($ACL)" />
</td>
</tr>
<tr>
@@ -109,7 +109,7 @@
</nobr>
</td>
<td class="value">
- <input id="url" name="url" type="text" size="10" value="$Encoder.attributeEscape($DENYACL)" />
+ <input id="denyAcl" name="denyAcl" type="text" size="10" value="$Encoder.attributeEscape($DENYACL)" />
</td>
</tr>
</table>
diff --git a/connectors/gridfs/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/gridfs/editSpecification_Credentials.html b/connectors/gridfs/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/gridfs/editSpecification_Credentials.html
deleted file mode 100644
index 1586129..0000000
--- a/connectors/gridfs/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/gridfs/editSpecification_Credentials.html
+++ /dev/null
@@ -1,44 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-#if($TABNAME == $ResourceBundle.getString('GridFSConnector.Credentials'))
-
-<table class="displaytable">
- <tr>
- <td class="separator" colspan="3">
- <hr />
- </td>
- </tr>
- <tr>
- <td class="description">
- <nobr>
- $Encoder.bodyEscape($ResourceBundle.getString('GridFSConnector.CredentialsColon'))
- </nobr>
- </td>
- <td class="value">
- <nobr>
- <input type="text" size="120" name="luceneQuery" value="$Encoder.attributeEscape($LUCENEQUERY)" />
- </nobr>
- </td>
- </tr>
-</table>
-
-#else
-
-<input type="hidden" name="luceneQuery" value="$Encoder.attributeEscape($LUCENEQUERY)" />
-
-#end
diff --git a/connectors/gridfs/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/gridfs/viewConfiguration.html b/connectors/gridfs/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/gridfs/viewConfiguration.html
index b68e5c2..9ae0447 100644
--- a/connectors/gridfs/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/gridfs/viewConfiguration.html
+++ b/connectors/gridfs/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/gridfs/viewConfiguration.html
@@ -51,6 +51,14 @@
$Encoder.bodyEscape($ResourceBundle.getString('GridFSConnector.BucketEquals'))$Encoder.bodyEscape($BUCKET)
</nobr>
<br />
+ <nobr>
+ $Encoder.bodyEscape($ResourceBundle.getString('GridFSConnector.ACLEquals'))$Encoder.bodyEscape($ACL)
+ </nobr>
+ <br />
+ <nobr>
+ $Encoder.bodyEscape($ResourceBundle.getString('GridFSConnector.DenyACLEquals'))$Encoder.bodyEscape($DENYACL)
+ </nobr>
+ <br />
</td>
</tr>
</table>
diff --git a/connectors/gridfs/pom.xml b/connectors/gridfs/pom.xml
index 37b07de..da3ba83 100644
--- a/connectors/gridfs/pom.xml
+++ b/connectors/gridfs/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/gts/pom.xml b/connectors/gts/pom.xml
index c23d7cd..cc76b0e 100644
--- a/connectors/gts/pom.xml
+++ b/connectors/gts/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/hdfs/pom.xml b/connectors/hdfs/pom.xml
index 61f7441..6de0a48 100644
--- a/connectors/hdfs/pom.xml
+++ b/connectors/hdfs/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/jcifs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharedrive/SharedDriveConnector.java b/connectors/jcifs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharedrive/SharedDriveConnector.java
index c215469..908bb47 100644
--- a/connectors/jcifs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharedrive/SharedDriveConnector.java
+++ b/connectors/jcifs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharedrive/SharedDriveConnector.java
@@ -16,57 +16,49 @@
*/
package org.apache.manifoldcf.crawler.connectors.sharedrive;
-import org.apache.manifoldcf.core.util.URLEncoder;
-import org.apache.manifoldcf.crawler.system.ManifoldCF;
-import java.io.FileOutputStream;
-import java.io.FileInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Date;
-
import jcifs.smb.ACE;
import jcifs.smb.NtlmPasswordAuthentication;
import jcifs.smb.SmbException;
import jcifs.smb.SmbFile;
import jcifs.smb.SmbFileFilter;
-
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
-import org.apache.manifoldcf.core.interfaces.IThreadContext;
-import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
-import org.apache.manifoldcf.core.interfaces.IPasswordMapperActivity;
-import org.apache.manifoldcf.core.interfaces.IPostParameters;
-import org.apache.manifoldcf.core.interfaces.ConfigParams;
-import org.apache.manifoldcf.core.interfaces.Specification;
-import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.connectorcommon.extmimemap.ExtensionMimeMap;
import org.apache.manifoldcf.connectorcommon.interfaces.IKeystoreManager;
import org.apache.manifoldcf.connectorcommon.interfaces.KeystoreManagerFactory;
+import org.apache.manifoldcf.core.common.DateParser;
+import org.apache.manifoldcf.core.interfaces.ConfigParams;
import org.apache.manifoldcf.core.interfaces.Configuration;
import org.apache.manifoldcf.core.interfaces.ConfigurationNode;
+import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
+import org.apache.manifoldcf.core.interfaces.IPostParameters;
+import org.apache.manifoldcf.core.interfaces.IThreadContext;
import org.apache.manifoldcf.core.interfaces.LockManagerFactory;
-import org.apache.manifoldcf.crawler.interfaces.ISeedingActivity;
-import org.apache.manifoldcf.crawler.interfaces.IProcessActivity;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.core.interfaces.Specification;
+import org.apache.manifoldcf.core.interfaces.SpecificationNode;
+import org.apache.manifoldcf.core.util.URLEncoder;
import org.apache.manifoldcf.crawler.interfaces.IExistingVersions;
import org.apache.manifoldcf.crawler.interfaces.IFingerprintActivity;
-import org.apache.manifoldcf.core.interfaces.SpecificationNode;
+import org.apache.manifoldcf.crawler.interfaces.IProcessActivity;
+import org.apache.manifoldcf.crawler.interfaces.ISeedingActivity;
import org.apache.manifoldcf.crawler.system.Logging;
import org.apache.manifoldcf.crawler.system.ManifoldCF;
-import org.apache.manifoldcf.connectorcommon.extmimemap.ExtensionMimeMap;
-import org.apache.manifoldcf.core.common.DateParser;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InterruptedIOException;
+import java.net.MalformedURLException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
/** This is the "repository connector" for a smb/cifs shared drive file system. It's a relative of the share crawler, and should have
* comparable basic functionality.
@@ -453,7 +445,7 @@
*@param activities is the interface this method should use to perform whatever framework actions are desired.
*@param spec is a document specification (that comes from the job).
*@param seedTime is the end of the time range of documents to consider, exclusive.
- *@param lastSeedVersionString is the last seeding version string for this job, or null if the job has no previous seeding version string.
+ *@param lastSeedVersion is the last seeding version string for this job, or null if the job has no previous seeding version string.
*@param jobMode is an integer describing how the job is being run, whether continuous or once-only.
*@return an updated seeding version string, to be stored with the job.
*/
@@ -700,8 +692,12 @@
catch (jcifs.smb.SmbAuthException e)
{
Logging.connectors.warn("JCIFS: Authorization exception reading version information for "+documentIdentifier+" - skipping");
- activities.deleteDocument(documentIdentifier);
- continue;
+ if(e.getMessage().equals("Logon failure: unknown user name or bad password."))
+ throw new ManifoldCFException( "SmbAuthException thrown: " + e.getMessage(), e );
+ else {
+ activities.deleteDocument(documentIdentifier );
+ continue;
+ }
}
catch (MalformedURLException mue)
{
@@ -987,9 +983,12 @@
Logging.connectors.warn("JCIFS: Authorization exception reading document/directory "+documentIdentifier+" - skipping");
errorCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
errorDesc = "Authorization: "+e.getMessage();
- // We call the delete even if it's a directory; this is harmless.
- activities.noDocument(documentIdentifier, versionString);
- continue;
+ if(e.getMessage().equals("Logon failure: unknown user name or bad password."))
+ throw new ManifoldCFException( "SmbAuthException thrown: " + e.getMessage(), e );
+ else {
+ activities.noDocument(documentIdentifier, versionString);
+ continue;
+ }
}
catch (SmbException se)
{
@@ -1267,6 +1266,13 @@
throw new ServiceInterruption("Timeout or other service interruption: "+se.getMessage(),se,currentTime + 300000L,
currentTime + 3 * 60 * 60000L,-1,false);
}
+ else if (se.getMessage().toLowerCase(Locale.ROOT).indexOf("file in use") != -1)
+ {
+ Logging.connectors.warn("JCIFS: 'File in Use' response when "+activity+" for "+documentIdentifier+": retrying...",se);
+ // 'File in Use' skip the document and keep going
+ throw new ServiceInterruption("Timeout or other service interruption: "+se.getMessage(),se,currentTime + 300000L,
+ currentTime + 3 * 60 * 60000L,-1,false);
+ }
else if (se.getMessage().indexOf("cannot find") != -1 || se.getMessage().indexOf("cannot be found") != -1)
{
return;
diff --git a/connectors/jcifs/pom.xml b/connectors/jcifs/pom.xml
index fc48c19..6043abe 100644
--- a/connectors/jcifs/pom.xml
+++ b/connectors/jcifs/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/jdbc/pom.xml b/connectors/jdbc/pom.xml
index 1b1218e..fb7a2a1 100644
--- a/connectors/jdbc/pom.xml
+++ b/connectors/jdbc/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraSession.java b/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraSession.java
index 513cf46..108fd73 100644
--- a/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraSession.java
+++ b/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraSession.java
@@ -63,7 +63,11 @@
import org.apache.http.util.EntityUtils;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.protocol.HttpContext;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.entity.ContentType;
import org.apache.http.client.AuthCache;
import org.apache.http.impl.client.BasicAuthCache;
@@ -124,9 +128,19 @@
javax.net.ssl.SSLSocketFactory httpsSocketFactory = KeystoreManagerFactory.getTrustingSecureSocketFactory();
SSLConnectionSocketFactory myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(httpsSocketFactory,connectionTimeout),
- SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ NoopHostnameVerifier.INSTANCE);
- connectionManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory>create()
+ .register("http", PlainConnectionSocketFactory.getSocketFactory())
+ .register("https", myFactory)
+ .build());
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ connectionManager = poolingConnectionManager;
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
@@ -141,7 +155,6 @@
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
@@ -169,15 +182,9 @@
httpClient = HttpClients.custom()
.setConnectionManager(connectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(credentialsProvider)
- .setSSLSocketFactory(myFactory)
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
.setRedirectStrategy(new DefaultRedirectStrategy())
.build();
diff --git a/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java b/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java
index 45907e5..8d40301 100644
--- a/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java
+++ b/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java
@@ -61,7 +61,11 @@
import org.apache.http.util.EntityUtils;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.protocol.HttpContext;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.entity.ContentType;
import org.apache.http.client.AuthCache;
import org.apache.http.impl.client.BasicAuthCache;
@@ -121,9 +125,19 @@
javax.net.ssl.SSLSocketFactory httpsSocketFactory = KeystoreManagerFactory.getTrustingSecureSocketFactory();
SSLConnectionSocketFactory myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(httpsSocketFactory,connectionTimeout),
- SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ NoopHostnameVerifier.INSTANCE);
- connectionManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory>create()
+ .register("http", PlainConnectionSocketFactory.getSocketFactory())
+ .register("https", myFactory)
+ .build());
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ connectionManager = poolingConnectionManager;
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
@@ -138,7 +152,6 @@
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
@@ -167,15 +180,9 @@
httpClient = HttpClients.custom()
.setConnectionManager(connectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(credentialsProvider)
- .setSSLSocketFactory(myFactory)
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
.setRedirectStrategy(new DefaultRedirectStrategy())
.build();
diff --git a/connectors/jira/pom.xml b/connectors/jira/pom.xml
index 7938f47..b962c2f 100644
--- a/connectors/jira/pom.xml
+++ b/connectors/jira/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/ldap/pom.xml b/connectors/ldap/pom.xml
index 996fc24..e1e8e42 100644
--- a/connectors/ldap/pom.xml
+++ b/connectors/ldap/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java b/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
index b6fd964..87ad3f1 100644
--- a/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
+++ b/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
@@ -52,8 +52,11 @@
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.NameValuePair;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
@@ -179,6 +182,7 @@
private String viewServerName = null;
private String viewPort = null;
private String viewCgiPath = null;
+ private String viewAction = null;
private String ingestNtlmDomain = null;
private String ingestNtlmUsername = null;
@@ -366,6 +370,7 @@
viewServerName = params.getParameter(LiveLinkParameters.viewServerName);
viewPort = params.getParameter(LiveLinkParameters.viewPort);
viewCgiPath = params.getParameter(LiveLinkParameters.viewCgiPath);
+ viewAction = params.getParameter(LiveLinkParameters.viewAction);
ingestNtlmDomain = params.getParameter(LiveLinkParameters.ingestNtlmDomain);
ingestNtlmUsername = params.getParameter(LiveLinkParameters.ingestNtlmUsername);
@@ -522,18 +527,32 @@
int socketTimeout = 900000;
int connectionTimeout = 300000;
- // Set up connection manager
- connectionManager = new PoolingHttpClientConnectionManager();
-
- CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-
// Set up ingest ssl if indicated
SSLConnectionSocketFactory myFactory = null;
if (ingestKeystoreManager != null)
{
myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(ingestKeystoreManager.getSecureSocketFactory(), connectionTimeout),
- new BrowserCompatHostnameVerifier());
+ NoopHostnameVerifier.INSTANCE);
}
+ else
+ {
+ myFactory = SSLConnectionSocketFactory.getSocketFactory();
+ }
+
+ // Set up connection manager
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory>create()
+ .register("http", PlainConnectionSocketFactory.getSocketFactory())
+ .register("https", myFactory)
+ .build());
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ connectionManager = poolingConnectionManager;
+
+ CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
// Set up authentication to use
if (ingestNtlmDomain != null)
@@ -544,27 +563,18 @@
HttpClientBuilder builder = HttpClients.custom()
.setConnectionManager(connectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout)
.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(credentialsProvider)
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
.setRedirectStrategy(new DefaultRedirectStrategy());
- if (myFactory != null)
- builder.setSSLSocketFactory(myFactory);
-
httpClient = builder.build();
// System.out.println("Connection server object = "+llServer.toString());
@@ -841,11 +851,28 @@
// The document identifier is the string form of the object ID for this connector.
if (!documentIdentifier.startsWith("D"))
return null;
+ String objectID = null;
int colonPosition = documentIdentifier.indexOf(":",1);
if (colonPosition == -1)
- return viewBasePath+"?func=ll&objID="+documentIdentifier.substring(1)+"&objAction=download";
+ objectID = documentIdentifier.substring(1);
else
- return viewBasePath+"?func=ll&objID="+documentIdentifier.substring(colonPosition+1)+"&objAction=download";
+ objectID = documentIdentifier.substring(colonPosition+1);
+ String viewURL = null;
+ switch(viewAction)
+ {
+ case "download":
+ viewURL = viewBasePath+"?func=ll&objAction=download&objID=" + objectID;
+ break;
+ case "open":
+ viewURL = viewBasePath+"/open/" + objectID;
+ break;
+ case "overview":
+ viewURL = viewBasePath+"?func=ll&objAction=overview&objID=" + objectID;
+ break;
+ default:
+ viewURL = viewBasePath+"?func=ll&objAction=download&objID=" + objectID;
+ }
+ return viewURL;
}
/** Request arbitrary connector information.
@@ -1751,11 +1778,15 @@
String viewCgiPath = parameters.getParameter(LiveLinkParameters.viewCgiPath);
if (viewCgiPath == null)
viewCgiPath = "/livelink/livelink.exe";
+ String viewAction = parameters.getParameter(LiveLinkParameters.viewAction);
+ if (viewAction == null)
+ viewAction = "download";
velocityContext.put("VIEWPROTOCOL",viewProtocol);
velocityContext.put("VIEWSERVERNAME",viewServerName);
velocityContext.put("VIEWPORT",viewPort);
velocityContext.put("VIEWCGIPATH",viewCgiPath);
+ velocityContext.put("VIEWACTION",viewAction);
}
/** Process a configuration post.
@@ -1785,6 +1816,9 @@
String viewCgiPath = variableContext.getParameter("viewcgipath");
if (viewCgiPath != null)
parameters.setParameter(LiveLinkParameters.viewCgiPath,viewCgiPath);
+ String viewAction = variableContext.getParameter("viewaction");
+ if (viewAction != null)
+ parameters.setParameter(LiveLinkParameters.viewAction,viewAction);
// Server parameters
String serverProtocol = variableContext.getParameter("serverprotocol");
diff --git a/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/livelink/LiveLinkParameters.java b/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/livelink/LiveLinkParameters.java
index 365c9ef..f423be9 100644
--- a/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/livelink/LiveLinkParameters.java
+++ b/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/livelink/LiveLinkParameters.java
@@ -49,6 +49,8 @@
public final static String viewPort = "View port";
/** View CGI path (path to use for viewing) */
public final static String viewCgiPath = "View CGI path";
+ /** Document View Action**/
+ public final static String viewAction = "View Action";
// These parameters are for LAPI
/** Connection options; choices are "internal", "http", "https" */
diff --git a/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_en_US.properties b/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_en_US.properties
index baecde2..a8a4ac0 100644
--- a/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_en_US.properties
+++ b/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_en_US.properties
@@ -67,6 +67,10 @@
LivelinkConnector.BlankSameAsFetchServer=(blank = same as fetch server)
LivelinkConnector.DocumentViewPort=Document view port:
LivelinkConnector.DocumentViewCGIPath=Document view CGI path
+LivelinkConnector.DocumentViewAction=Document view action
+LivelinkConnector.DocumentViewActionDownload=Download
+LivelinkConnector.DocumentViewActionOpen=Open
+LivelinkConnector.DocumentViewActionOverview=Overview
LivelinkConnector.Parameters=Parameters:
LivelinkConnector.certificates= certificate(s)
LivelinkConnector.SelectAFolderFirst=Select a folder first
diff --git a/connectors/livelink/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/livelink/editConfiguration_View.html b/connectors/livelink/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/livelink/editConfiguration_View.html
index 2c72ebc..614212a 100644
--- a/connectors/livelink/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/livelink/editConfiguration_View.html
+++ b/connectors/livelink/connector/src/main/resources/org/apache/manifoldcf/crawler/connectors/livelink/editConfiguration_View.html
@@ -20,7 +20,7 @@
<tr>
<td class="description">$Encoder.bodyEscape($ResourceBundle.getString('LivelinkConnector.DocumentViewProtocol'))</td>
<td class="value">
- <select name="viewprotocol" size="3">
+ <select name="viewprotocol">
<option value="" #if($VIEWPROTOCOL == '') selected="selected" #{else} "" #end >$Encoder.bodyEscape($ResourceBundle.getString('LivelinkConnector.SameAsFetchProtocol'))</option>
<option value="http" #if($VIEWPROTOCOL == 'http') selected="selected" #{else} "" #end >http</option>
<option value="https" #if($VIEWPROTOCOL == 'https') selected="selected" #{else} "" #end >https</option>
@@ -58,10 +58,23 @@
<input type="text" size="32" name="viewcgipath" value="$Encoder.attributeEscape($VIEWCGIPATH)"/>
</td>
</tr>
+ <tr>
+ <td class="description">
+ <nobr>$Encoder.bodyEscape($ResourceBundle.getString('LivelinkConnector.DocumentViewAction'))</nobr>
+ </td>
+ <td class="value">
+ <select name="viewaction">
+ <option value="download" #if($VIEWACTION == 'download') selected="selected" #{else} "" #end >$Encoder.bodyEscape($ResourceBundle.getString('LivelinkConnector.DocumentViewActionDownload'))</option>
+ <option value="open" #if($VIEWACTION == 'open') selected="selected" #{else} "" #end >$Encoder.bodyEscape($ResourceBundle.getString('LivelinkConnector.DocumentViewActionOpen'))</option>
+ <option value="overview" #if($VIEWACTION == 'overview') selected="selected" #{else} "" #end >$Encoder.bodyEscape($ResourceBundle.getString('LivelinkConnector.DocumentViewActionOverview'))</option>
+ </select>
+ </td>
+ </tr>
</table>
#else
<input type="hidden" name="viewprotocol" value="$!VIEWPROTOCOL"/>
<input type="hidden" name="viewservername" value="$!VIEWSERVERNAME"/>
<input type="hidden" name="viewport" value="$!VIEWPORT"/>
<input type="hidden" name="viewcgipath" value="$Encoder.attributeEscape($VIEWCGIPATH)"/>
+<input type="hidden" name="viewaction" value="$!VIEWACTION"/>
#end
\ No newline at end of file
diff --git a/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/meridio/MeridioWrapper.java b/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/meridio/MeridioWrapper.java
index bd70aa4..09f4a91 100644
--- a/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/meridio/MeridioWrapper.java
+++ b/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/meridio/MeridioWrapper.java
@@ -81,8 +81,11 @@
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.config.SocketConfig;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.impl.client.DefaultRedirectStrategy;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.NTCredentials;
@@ -218,7 +221,11 @@
SSLConnectionSocketFactory myFactory = null;
if (mySSLFactory != null)
{
- myFactory = new SSLConnectionSocketFactory(mySSLFactory, new BrowserCompatHostnameVerifier());
+ myFactory = new SSLConnectionSocketFactory(mySSLFactory, new NoopHostnameVerifier());
+ }
+ else
+ {
+ myFactory = SSLConnectionSocketFactory.getSocketFactory();
}
// Set up the pool.
@@ -257,9 +264,41 @@
int socketTimeout = 900000;
int connectionTimeout = 300000;
- dmwsConnectionManager = new PoolingHttpClientConnectionManager();
- rmwsConnectionManager = new PoolingHttpClientConnectionManager();
- mcwsConnectionManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingDmwsConnectionManager = new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory>create()
+ .register("http", PlainConnectionSocketFactory.getSocketFactory())
+ .register("https", myFactory)
+ .build());
+ poolingDmwsConnectionManager.setDefaultMaxPerRoute(1);
+ poolingDmwsConnectionManager.setValidateAfterInactivity(60000);
+ poolingDmwsConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ dmwsConnectionManager = poolingDmwsConnectionManager;
+
+ PoolingHttpClientConnectionManager poolingRmwsConnectionManager = new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory>create()
+ .register("http", PlainConnectionSocketFactory.getSocketFactory())
+ .register("https", myFactory)
+ .build());
+ poolingRmwsConnectionManager.setDefaultMaxPerRoute(1);
+ poolingRmwsConnectionManager.setValidateAfterInactivity(60000);
+ poolingRmwsConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ rmwsConnectionManager = poolingRmwsConnectionManager;
+
+ PoolingHttpClientConnectionManager poolingMcwsConnectionManager = new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory>create()
+ .register("http", PlainConnectionSocketFactory.getSocketFactory())
+ .register("https", myFactory)
+ .build());
+ poolingMcwsConnectionManager.setDefaultMaxPerRoute(1);
+ poolingMcwsConnectionManager.setValidateAfterInactivity(60000);
+ poolingMcwsConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ mcwsConnectionManager = poolingMcwsConnectionManager;
// Initialize the three httpclient objects
@@ -269,14 +308,12 @@
RequestConfig.Builder dmwsRequestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(false)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
RequestConfig.Builder rmwsRequestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
@@ -326,30 +363,18 @@
dmwsHttpClient = HttpClients.custom()
.setConnectionManager(dmwsConnectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(dmwsRequestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(dmwsCredentialsProvider)
- .setSSLSocketFactory(myFactory)
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
.setRedirectStrategy(new DefaultRedirectStrategy())
.build();
rmwsHttpClient = HttpClients.custom()
.setConnectionManager(rmwsConnectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(rmwsRequestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(rmwsCredentialsProvider)
- .setSSLSocketFactory(myFactory)
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
.setRedirectStrategy(new DefaultRedirectStrategy())
.build();
@@ -361,7 +386,6 @@
RequestConfig.Builder mcwsRequestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
@@ -391,15 +415,9 @@
mcwsHttpClient = HttpClients.custom()
.setConnectionManager(mcwsConnectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(mcwsRequestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(mcwsCredentialsProvider)
- .setSSLSocketFactory(myFactory)
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
.setRedirectStrategy(new DefaultRedirectStrategy())
.build();
diff --git a/connectors/nullauthority/pom.xml b/connectors/nullauthority/pom.xml
index 8163ba0..d784836 100644
--- a/connectors/nullauthority/pom.xml
+++ b/connectors/nullauthority/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/nulloutput/pom.xml b/connectors/nulloutput/pom.xml
index e811a15..17c4873 100644
--- a/connectors/nulloutput/pom.xml
+++ b/connectors/nulloutput/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/nulltransformation/pom.xml b/connectors/nulltransformation/pom.xml
index a25e4bf..7c1c0e6 100644
--- a/connectors/nulltransformation/pom.xml
+++ b/connectors/nulltransformation/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java b/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
index 4009bcc..1d56f9a 100644
--- a/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
+++ b/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
@@ -104,28 +104,31 @@
{
if (client == null)
{
- connectionManager = new PoolingHttpClientConnectionManager();
-
final int executorTimeout = 300000;
final int socketTimeout = 60000;
final int connectionTimeout = 60000;
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ connectionManager = poolingConnectionManager;
+
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true).setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true).setExpectContinueEnabled(true)
+ .setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
HttpClientBuilder clientBuilder = HttpClients
.custom()
.setConnectionManager(connectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setRequestExecutor(new HttpRequestExecutor(executorTimeout))
- .setDefaultSocketConfig(
- SocketConfig.custom().setTcpNoDelay(true)
- .setSoTimeout(socketTimeout).build());
+ .setRequestExecutor(new HttpRequestExecutor(executorTimeout));
client = clientBuilder.build();
diff --git a/connectors/opensearchserver/pom.xml b/connectors/opensearchserver/pom.xml
index 0f0205d..1adab81 100644
--- a/connectors/opensearchserver/pom.xml
+++ b/connectors/opensearchserver/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/pom.xml b/connectors/pom.xml
index 75e24f1..363adc2 100644
--- a/connectors/pom.xml
+++ b/connectors/pom.xml
@@ -20,13 +20,13 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-parent</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
<name>ManifoldCF - Connectors</name>
<packaging>pom</packaging>
diff --git a/connectors/regexpmapper/pom.xml b/connectors/regexpmapper/pom.xml
index d8e5712..30a587a 100644
--- a/connectors/regexpmapper/pom.xml
+++ b/connectors/regexpmapper/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java b/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
index b6f8a07..052dd4e 100644
--- a/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
+++ b/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
@@ -40,7 +40,7 @@
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.config.SocketConfig;
-import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
@@ -261,16 +261,22 @@
// Create the https scheme for this connection
javax.net.ssl.SSLSocketFactory httpsSocketFactory = KeystoreManagerFactory.getTrustingSecureSocketFactory();;
SSLConnectionSocketFactory myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(httpsSocketFactory,connectionTimeoutMilliseconds),
- SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ NoopHostnameVerifier.INSTANCE);
- connectionManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(connectionTimeoutMilliseconds)
+ .build());
+ connectionManager = poolingConnectionManager;
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(connectionTimeoutMilliseconds)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeoutMilliseconds)
.setConnectionRequestTimeout(connectionTimeoutMilliseconds);
@@ -302,10 +308,6 @@
.setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(connectionTimeoutMilliseconds)
- .build())
.setDefaultCredentialsProvider(credentialsProvider)
.setSSLSocketFactory(myFactory)
.setRequestExecutor(new HttpRequestExecutor(connectionTimeoutMilliseconds))
diff --git a/connectors/rss/pom.xml b/connectors/rss/pom.xml
index fba2f61..3b56d27 100644
--- a/connectors/rss/pom.xml
+++ b/connectors/rss/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxClient.java b/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxClient.java
index 88f8edd..49a6fdb 100644
--- a/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxClient.java
+++ b/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxClient.java
@@ -47,222 +47,215 @@
*/
public class SearchBloxClient {
- // TODO All this might need to be included in a configuration file
- public static final String DEFAULT_ENDPOINT = "http://localhost:8080/searchblox/rest/v1/api";
+ // TODO All this might need to be included in a configuration file
+ public static final String DEFAULT_ENDPOINT = "http://localhost:8080/searchblox/rest/v1/api";
- private static final String ADD_PATH = "add";
+ private static final String ADD_PATH = "add";
- private static final String DELETE_PATH = "delete";
+ private static final String DELETE_PATH = "delete";
- private static final String STATUS_PATH = "status";
+ private static final String STATUS_PATH = "status";
- private static final String CREATE_PATH = "coladd";
+ private static final String CREATE_PATH = "coladd";
- private static final String CLEAR_PATH = "clear";
+ private static final String CLEAR_PATH = "clear";
- private static final String STATUS_NODE = "statuscode";
+ private static final String STATUS_NODE = "statuscode";
+
+ private static final Pattern status_pattern = Pattern.compile("^status code\\s?:\\s([0-9]+)$");
+
+ public static enum ResponseCode {
+ DOCUMENT_INDEXED(100),
+ DOCUMENT_REJECTED(101),
+ DOCUMENT_DELETED(200, 2001),
+ DOCUMENT_NOT_EXIST(201, 2002),
+ DOCUMENT_NOT_FOUND(301),
+ COLLECTION_CLEARED(400),
+ ERROR_CLEARING_COLLECTION(401),
+ COLLECTION_CREATED(900),
+ INVALID_COLLECTION_NAME(500, 501),
+ INVALID_REQUEST(501),
+ INVALID_DOCUMENT_LOCATION(502),
+ NOT_CUSTOM_COLLECTION(503),
+ LIMIT_EXCEEDED(504),
+ INVALID_LICENSE_ID(601),
+ SERVER_UNREACHABLE(700);
+
+ private int code;
+ private int jsonCode;
- private static final Pattern status_pattern = Pattern.compile("^status code\\s?:\\s([0-9]+)$");
+ ResponseCode(int code) {
+ this.code = code;
+ }
+
+ ResponseCode(int code, int jsonCode) {
+ this.code = code;
+ this.jsonCode = jsonCode;
+ }
+
+ static ResponseCode getCodeFromValue(int value){
+ for(ResponseCode e:ResponseCode.values())
+ if(value == e.code)
+ return e;
+ return null;
+ }
+
+ static ResponseCode getCodeFromValue(int value, boolean json){
+ for(ResponseCode e:ResponseCode.values())
+ if((json && value == e.jsonCode) || (value == e.code)) {
+ return e;
+ }
+ return null;
+ }
+
+ int getCode(){
+ return code;
+ }
+
+ int getJsonCode() {
+ return jsonCode;
+ }
+ }
+
- public static enum ResponseCode {
- DOCUMENT_INDEXED(100),
- DOCUMENT_REJECTED(101),
- DOCUMENT_DELETED(200, 2001),
- DOCUMENT_NOT_EXIST(201, 2002),
- DOCUMENT_NOT_FOUND(301),
- COLLECTION_CLEARED(400),
- ERROR_CLEARING_COLLECTION(401),
- COLLECTION_CREATED(900),
- INVALID_COLLECTION_NAME(500, 501),
- INVALID_REQUEST(501),
- INVALID_DOCUMENT_LOCATION(502),
- NOT_CUSTOM_COLLECTION(503),
- LIMIT_EXCEEDED(504),
- INVALID_LICENSE_ID(601),
- SERVER_UNREACHABLE(700);
+ private String apikey;
+ private Client client;
+ private UriBuilder uriBuilder;
- private int code;
- private int jsonCode;
-
- ResponseCode(int code) {
- this.code = code;
- }
-
- ResponseCode(int code, int jsonCode) {
- this.code = code;
- this.jsonCode = jsonCode;
- }
-
- static ResponseCode getCodeFromValue(int value){
- for(ResponseCode e:ResponseCode.values())
- if(value == e.code)
- return e;
- return null;
- }
-
- static ResponseCode getCodeFromValue(int value, boolean json){
- for(ResponseCode e:ResponseCode.values())
- if((json && value == e.jsonCode) || (value == e.code)) {
- return e;
- }
- return null;
- }
-
- int getCode(){
- return code;
- }
-
- int getJsonCode() {
- return jsonCode;
- }
+ public SearchBloxClient(String apikey, ClientBuilder builder, String endpoint) {
+ this.apikey = apikey;
+ builder.register(StringTextStar.class);
+ this.client = builder.build();
+ if (endpoint != null && !endpoint.isEmpty()) {
+ uriBuilder = UriBuilder.fromUri(endpoint);
+ } else {
+ uriBuilder = UriBuilder.fromUri(DEFAULT_ENDPOINT);
+ }
+ }
+
+
+ public ResponseCode addUpdateDocument(SearchBloxDocument document, String format)
+ throws SearchBloxException {
+ return post(document, format, SearchBloxDocument.DocumentAction.ADD_UPDATE);
+ }
+
+ public ResponseCode deleteDocument(SearchBloxDocument document, String format)
+ throws SearchBloxException {
+ return post(document, format, SearchBloxDocument.DocumentAction.DELETE);
+ }
+
+ public ResponseCode createCollection(String colname, String format)
+ throws SearchBloxException {
+ SearchBloxDocument document = new SearchBloxDocument(apikey);
+ document.colName = colname;
+ return post(document, format, SearchBloxDocument.DocumentAction.CREATE);
+ }
+
+ public ResponseCode clearCollection(String colname, String format)
+ throws SearchBloxException {
+ SearchBloxDocument document = new SearchBloxDocument(apikey);
+ document.colName = colname;
+ return post(document, format, SearchBloxDocument.DocumentAction.CLEAR);
+ }
+
+ public boolean ping(String format)
+ throws SearchBloxException {
+ SearchBloxDocument document = new SearchBloxDocument(apikey);
+ document.colName = UUID.randomUUID().toString();
+ document.uid = UUID.randomUUID().toString();
+ ResponseCode result = post(document, format, SearchBloxDocument.DocumentAction.STATUS);
+ return result == ResponseCode.INVALID_COLLECTION_NAME;
+ }
+
+ private ResponseCode post(SearchBloxDocument document, String format, SearchBloxDocument.DocumentAction action)
+ throws SearchBloxException {
+
+ SearchBloxDocument.IndexingFormat iFormat = SearchBloxDocument.IndexingFormat.valueOf(format.toUpperCase());
+
+ if (iFormat == null) {
+ Logging.connectors.error("[Post request] Format not recognized " +format);
+ throw new SearchBloxException("Unknown Serialization Format " + format);
+ }
+
+ boolean isJson = iFormat.equals(SearchBloxDocument.IndexingFormat.JSON);
+
+
+
+ UriBuilder uri = uriBuilder.clone();
+ if (action == SearchBloxDocument.DocumentAction.ADD_UPDATE) {
+ uri = uri.path(ADD_PATH);
+ } else if (action == SearchBloxDocument.DocumentAction.DELETE) {
+ uri = uri.path(DELETE_PATH);
+ } else if (action == SearchBloxDocument.DocumentAction.STATUS) {
+ uri = uri.path(STATUS_PATH);
+ } else if (action == SearchBloxDocument.DocumentAction.CREATE) {
+ uri = uri.path(CREATE_PATH);
+ } else if (action == SearchBloxDocument.DocumentAction.CLEAR) {
+ uri = uri.path(CLEAR_PATH);
+ }
+
+ WebTarget target = client.target(uri.build());
+ Builder httpRequest = target.request();
+ if (iFormat == SearchBloxDocument.IndexingFormat.JSON) {
+ httpRequest.accept(MediaType.APPLICATION_JSON_TYPE);
+ }else{
+ httpRequest.accept(MediaType.APPLICATION_XML_TYPE);
}
- private String apikey;
- private Client client;
- private UriBuilder uriBuilder;
-
- public SearchBloxClient(String apikey, ClientBuilder builder, String endpoint) {
- this.apikey = apikey;
- builder.register(StringTextStar.class);
- this.client = builder.build();
- if (endpoint != null && !endpoint.isEmpty()) {
- uriBuilder = UriBuilder.fromUri(endpoint);
- } else {
- uriBuilder = UriBuilder.fromUri(DEFAULT_ENDPOINT);
- }
+ document.apiKey = this.apikey;
+
+ String body = document.toString(iFormat, action);
+ Logging.connectors.debug("Document for document: " + document.uid +":" + body);
+ MediaType type = MediaType.TEXT_PLAIN_TYPE;
+ if (iFormat == SearchBloxDocument.IndexingFormat.JSON) {
+ type = MediaType.APPLICATION_JSON_TYPE;
}
-
- public ResponseCode addUpdateDocument(SearchBloxDocument document, String format)
- throws SearchBloxException {
- return post(document, format, SearchBloxDocument.DocumentAction.ADD_UPDATE);
+
+ Entity<String> entity = Entity.entity(body, type);
+ Response response = null;
+ try {
+ response = httpRequest.post(entity);
}
-
- public ResponseCode deleteDocument(SearchBloxDocument document, String format)
- throws SearchBloxException {
- return post(document, format, SearchBloxDocument.DocumentAction.DELETE);
+ catch(Exception e) {
+ // return e.getCause() instanceof ConnectException ? ResponseCode.SERVER_UNREACHABLE : ResponseCode.INVALID_COLLECTION_NAME;
+ Logging.connectors.error("[No Connection] Error trying to connect ",e);
+ return ResponseCode.SERVER_UNREACHABLE;
}
-
- public ResponseCode createCollection(String colname, String format)
- throws SearchBloxException {
- SearchBloxDocument document = new SearchBloxDocument(apikey);
- document.colName = colname;
- return post(document, format, SearchBloxDocument.DocumentAction.CREATE);
+
+ String rawResponse = response.readEntity(String.class);
+ if(iFormat == IndexingFormat.XML){
+ DOMParser parser = new DOMParser();
+ try {
+ parser.parse(new InputSource(new StringReader(rawResponse)));
+ } catch (SAXException | IOException e) {
+ Logging.connectors.error("[Response parsing] Dom parsing error", e);
+ throw new SearchBloxException(e);
+ }
+ Document doc = parser.getDocument();
+ NodeList nodeList = doc.getElementsByTagName(STATUS_NODE);
+ if (nodeList == null || nodeList.getLength() == 0) {
+ String message = "[Response Parsing] Status code not found";
+ Logging.connectors.error(message);
+ throw new SearchBloxException(message);
+ }
+ String codeStr = nodeList.item(0).getTextContent();
+ int statusCode = Integer.parseInt(codeStr);
+ return ResponseCode.getCodeFromValue(statusCode, isJson);
+ }else{
+ Matcher matcher = status_pattern.matcher(rawResponse);
+ String codeStr = null;
+ if(matcher.find())
+ codeStr = matcher.group(1);
+ if(codeStr == null){
+ String message = "[Response parsing] Response code parsing error";
+ Logging.connectors.error(message);
+ throw new SearchBloxException(message);
+ }
+
+ int statusCode = Integer.parseInt(codeStr);
+ return ResponseCode.getCodeFromValue(statusCode, isJson);
}
-
- public ResponseCode clearCollection(String colname, String format)
- throws SearchBloxException {
- SearchBloxDocument document = new SearchBloxDocument(apikey);
- document.colName = colname;
- return post(document, format, SearchBloxDocument.DocumentAction.CLEAR);
- }
-
- public boolean ping(String format)
- throws SearchBloxException {
- SearchBloxDocument document = new SearchBloxDocument(apikey);
- document.colName = UUID.randomUUID().toString();
- document.uid = UUID.randomUUID().toString();
- ResponseCode result = post(document, format, SearchBloxDocument.DocumentAction.STATUS);
- return result == ResponseCode.INVALID_COLLECTION_NAME;
- }
-
- private ResponseCode post(SearchBloxDocument document, String format, SearchBloxDocument.DocumentAction action)
- throws SearchBloxException {
-
- SearchBloxDocument.IndexingFormat iFormat = SearchBloxDocument.IndexingFormat.valueOf(format.toUpperCase());
-
- if (iFormat == null) {
- Logging.connectors.error("[Post request] Format not recognized " +format);
- throw new SearchBloxException("Unknown Serialization Format " + format);
- }
-
- boolean isJson = iFormat.equals(SearchBloxDocument.IndexingFormat.JSON);
-
-
-
- UriBuilder uri = uriBuilder.clone();
- if (action == SearchBloxDocument.DocumentAction.ADD_UPDATE) {
- uri = uri.path(ADD_PATH);
- } else if (action == SearchBloxDocument.DocumentAction.DELETE) {
- uri = uri.path(DELETE_PATH);
- } else if (action == SearchBloxDocument.DocumentAction.STATUS) {
- uri = uri.path(STATUS_PATH);
- } else if (action == SearchBloxDocument.DocumentAction.CREATE) {
- uri = uri.path(CREATE_PATH);
- } else if (action == SearchBloxDocument.DocumentAction.CLEAR) {
- uri = uri.path(CLEAR_PATH);
- }
-
- WebTarget target = client.target(uri.build());
- Builder httpRequest = target.request();
- if (iFormat == SearchBloxDocument.IndexingFormat.JSON) {
- httpRequest.accept(MediaType.APPLICATION_JSON_TYPE);
- }else{
- httpRequest.accept(MediaType.APPLICATION_XML_TYPE);
- }
-
-
- document.apiKey = this.apikey;
-
- String body = document.toString(iFormat, action);
- Logging.connectors.debug("Document for document: " + document.uid +":" + body);
- MediaType type = MediaType.TEXT_PLAIN_TYPE;
- if (iFormat == SearchBloxDocument.IndexingFormat.JSON) {
- type = MediaType.APPLICATION_JSON_TYPE;
- }
-
-
- Entity<String> entity = Entity.entity(body, type);
- Response response = null;
- try {
- response = httpRequest.post(entity);
- }
- catch(Exception e) {
- // return e.getCause() instanceof ConnectException ? ResponseCode.SERVER_UNREACHABLE : ResponseCode.INVALID_COLLECTION_NAME;
- Logging.connectors.error("[No Connection] Error trying to connect ",e);
- return ResponseCode.SERVER_UNREACHABLE;
- }
-
- String rawResponse = response.readEntity(String.class);
- if(iFormat == IndexingFormat.XML){
- DOMParser parser = new DOMParser();
- try {
- parser.parse(new InputSource(new StringReader(rawResponse)));
- } catch (SAXException | IOException e) {
- Logging.connectors.error("[Response parsing] Dom parsing error", e);
- throw new SearchBloxException(e);
- }
- Document doc = parser.getDocument();
- NodeList nodeList = doc.getElementsByTagName(STATUS_NODE);
- if (nodeList == null || nodeList.getLength() == 0) {
- String message = "[Response Parsing] Status code not found";
- Logging.connectors.error(message);
- throw new SearchBloxException(message);
- }
- String codeStr = nodeList.item(0).getTextContent();
- int statusCode = Integer.parseInt(codeStr);
- return ResponseCode.getCodeFromValue(statusCode, isJson);
- }else{
-// try {
-// JSONObject json = new JSONObject(rawResponse);
-// String codeStr = json.getString(STATUS_NODE);
- Matcher matcher = status_pattern.matcher(rawResponse);
- String codeStr = null;
- if(matcher.find())
- codeStr = matcher.group(1);
- if(codeStr == null){
- String message = "[Response parsing] Response code parsing error";
- Logging.connectors.error(message);
- throw new SearchBloxException(message);
- }
-
- int statusCode = Integer.parseInt(codeStr);
- return ResponseCode.getCodeFromValue(statusCode, isJson);
-// } catch (JSONException e) {
-// Logging.connectors.error("[Response parsing] Response JSON parsing error", e);
-// throw new SearchBloxException(e);
-// }
- }
- }
+ }
}
diff --git a/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxConfig.java b/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxConfig.java
index 40ff11e..262fbc3 100644
--- a/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxConfig.java
+++ b/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxConfig.java
@@ -7,15 +7,12 @@
*/
class SearchBloxConfig {
- static final String NODE_CONFIGURATION = "configuration";
- static final String ATTRIBUTE_TITLEBOOST = "title_boost";
- static final String ATTRIBUTE_CONTENTBOOST = "content_boost";
- static final String ATTRIBUTE_KEYWORDSBOOST = "keywords_boost";
- static final String ATTRIBUTE_DESCRIPTIONBOOST = "description_boost";
- static final String ATTRIBUTE_POOLSIZE = "poolsize";
- static final String ATTRIBUTE_TIMEOUT_CONNECTION = "timeoutconnection";
- static final String ATTRIBUTE_TIMEOUT_SOCKET = "timeoutsocket";
- static final String ATTRIBUTE_INDEX_FORMAT = "indexformat";
- static final String ATTRIBUTE_COLLECTION_NAME = "collection";
+ static final String NODE_CONFIGURATION = "configuration";
+ static final String ATTRIBUTE_TITLEBOOST = "title_boost";
+ static final String ATTRIBUTE_CONTENTBOOST = "content_boost";
+ static final String ATTRIBUTE_KEYWORDSBOOST = "keywords_boost";
+ static final String ATTRIBUTE_DESCRIPTIONBOOST = "description_boost";
+ static final String ATTRIBUTE_INDEX_FORMAT = "indexformat";
+ static final String ATTRIBUTE_COLLECTION_NAME = "collection";
}
diff --git a/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxConnector.java b/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxConnector.java
index a4941c9..043ab8b 100644
--- a/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxConnector.java
+++ b/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxConnector.java
@@ -43,833 +43,799 @@
*/
public class SearchBloxConnector extends BaseOutputConnector {
- private final static String SEARCHBLOX_TAB_PARAMETERS = "SearchBloxConnector.Parameters";
+ private final static String SEARCHBLOX_TAB_PARAMETERS = "SearchBloxConnector.Parameters";
- private static final String EDIT_SPECIFICATION_JS = "editSpecification.js";
- private static final String EDIT_SPECIFICATION_CONFIGURATION_HTML = "editSpecification_Configuration.html";
- private static final String VIEW_SPECIFICATION_HTML = "viewSpecification.html";
+ private static final String EDIT_SPECIFICATION_JS = "editSpecification.js";
+ private static final String EDIT_SPECIFICATION_CONFIGURATION_HTML = "editSpecification_Configuration.html";
+ private static final String VIEW_SPECIFICATION_HTML = "viewSpecification.html";
- /** Forward to the javascript to check the configuration parameters */
- private static final String EDIT_CONFIG_HEADER_FORWARD = "editConfiguration.js";
+ /** Forward to the javascript to check the configuration parameters */
+ private static final String EDIT_CONFIG_HEADER_FORWARD = "editConfiguration.js";
- /** Forward to the HTML template to edit the configuration parameters */
- private static final String EDIT_CONFIG_FORWARD_PARAMETERS = "editConfiguration_Parameters.html";
+ /** Forward to the HTML template to edit the configuration parameters */
+ private static final String EDIT_CONFIG_FORWARD_PARAMETERS = "editConfiguration_Parameters.html";
- /** Forward to the HTML template to view the configuration parameters */
- private static final String VIEW_CONFIG_FORWARD = "viewConfiguration.html";
+ /** Forward to the HTML template to view the configuration parameters */
+ private static final String VIEW_CONFIG_FORWARD = "viewConfiguration.html";
- /**
- * Default collection name
- */
- private static final String DEFAULT_COLLECTION = "Custom";
-
- /**
- * Default apiKey
- */
- private static final String DEFAULT_APIKEY = "apiKey";
-
- /**
- * Ingestion activity
- */
- public final static String INGEST_ACTIVITY = "document ingest";
+ /**
+ * Default collection name
+ */
+ private static final String DEFAULT_COLLECTION = "Custom";
+
+ /**
+ * Default apiKey
+ */
+ private static final String DEFAULT_APIKEY = "apiKey";
+
+ /**
+ * Ingestion activity
+ */
+ public final static String INGEST_ACTIVITY = "document ingest";
- /**
- * Document removal activity
- */
- public final static String REMOVE_ACTIVITY = "document deletion";
+ /**
+ * Document removal activity
+ */
+ public final static String REMOVE_ACTIVITY = "document deletion";
- /**
- * Collection Creation activity
- */
- public final static String CREATION_ACTIVITY = "collection created";
+ /**
+ * Collection Creation activity
+ */
+ public final static String CREATION_ACTIVITY = "collection created";
-
- private static final String SEARCHBLOX_ENDPOINT = "endpoint";
- private static final String SEARCHBLOX_INDEXING_FORMAT = "indexformat";
+
+ private static final String SEARCHBLOX_ENDPOINT = "endpoint";
+ private static final String SEARCHBLOX_INDEXING_FORMAT = "indexformat";
+ private static final String SEARCHBLOX_SOCKET_TIMEOUT = "sockettimeout";
+ private static final String SEARCHBLOX_CONNECTION_TIMEOUT = "connectiontimeout";
+
+ private static final String BUILDER_DEFAULT_SOCKET_TIMEOUT = "60";
+ private static final String BUILDER_DEFAULT_CONNECTION_TIMEOUT = "60";
- private static final int BUILDER_DEFAULT_POOL_SIZE = 5;
- private static final long BUILDER_DEFAULT_SOCKET_TIMEOUT = 60;
- private static final long BUILDER_DEFAULT_CONNECTION_TIMEOUT = 60;
+ private SearchBloxClient client = null;
+ private String apiKey = null;
- private ResteasyClientBuilder restBuilder = null;
- private SearchBloxClient client = null;
- private String apiKey = null;
- private String lastVersion = null;
+ public SearchBloxConnector() {
- public SearchBloxConnector() {
+ }
- }
+ /** Connect.
+ */
+ @Override
+ public void connect(ConfigParams configParams) {
+ super.connect(configParams);
+ }
+
+ /**
+ * This method is called to assess whether to count this connector instance
+ * should actually be counted as being connected.
+ *
+ * @return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected() {
+ return client != null;
+ }
- /**
- * This method is called to assess whether to count this connector instance
- * should actually be counted as being connected.
- *
- * @return true if the connector instance is actually connected.
- */
- @Override
- public boolean isConnected() {
- return client != null;
- }
+ /**
+ * Close the connection. Call this before discarding the connection.
+ */
+ @Override
+ public void disconnect() throws ManifoldCFException {
+ if (client != null) {
+ apiKey = null;
+ client = null;
+ }
+ super.disconnect();
+ }
- /**
- * Close the connection. Call this before discarding the connection.
- */
- @Override
- public void disconnect() throws ManifoldCFException {
- if (client != null) {
- apiKey = null;
- restBuilder = null;
- client = null;
- }
- super.disconnect();
- }
+ /**
+ * Return the list of activities that this connector supports (i.e. writes
+ * into the log).
+ *
+ * @return the list.
+ */
+ @Override
+ public String[] getActivitiesList() {
+ return new String[] { INGEST_ACTIVITY, REMOVE_ACTIVITY,
+ CREATION_ACTIVITY };
+ }
- /**
- * Return the list of activities that this connector supports (i.e. writes
- * into the log).
- *
- * @return the list.
- */
- @Override
- public String[] getActivitiesList() {
- return new String[] { INGEST_ACTIVITY, REMOVE_ACTIVITY,
- CREATION_ACTIVITY };
- }
+ protected void getSession()
+ throws ManifoldCFException {
+ if (client == null) {
+ String connectionTimeoutString = params.getParameter(SEARCHBLOX_CONNECTION_TIMEOUT);
+ if (connectionTimeoutString == null)
+ connectionTimeoutString = BUILDER_DEFAULT_CONNECTION_TIMEOUT;
+ long connectionTimeout;
+ try {
+ connectionTimeout = Integer.parseInt(connectionTimeoutString);
+ } catch (NumberFormatException e) {
+ throw new ManifoldCFException("Bad connection timeout: "+e.getMessage(),e);
+ }
+ String socketTimeoutString = params.getParameter(SEARCHBLOX_SOCKET_TIMEOUT);
+ if (socketTimeoutString == null)
+ socketTimeoutString = BUILDER_DEFAULT_SOCKET_TIMEOUT;
+ long socketTimeout;
+ try {
+ socketTimeout = Integer.parseInt(socketTimeoutString);
+ } catch (NumberFormatException e) {
+ throw new ManifoldCFException("Bad socket timeout: "+e.getMessage(),e);
+ }
+ final String endpoint = params.getParameter(SEARCHBLOX_ENDPOINT);
+ this.apiKey = params.getParameter(SearchBloxDocument.API_KEY);
+ ResteasyClientBuilder builder = new ResteasyClientBuilder();
+ builder.connectionPoolSize(1);
+ builder.establishConnectionTimeout(connectionTimeout, TimeUnit.SECONDS);
+ builder.socketTimeout(socketTimeout, TimeUnit.SECONDS);
+ client = new SearchBloxClient(apiKey, builder, endpoint);
+ }
+
+ }
- protected synchronized void getSession(Map<String, List<String>> args, String version) {
- if (client == null) {
- String endpoint = params.getParameter(SEARCHBLOX_ENDPOINT);
- this.apiKey = params.getParameter(SearchBloxDocument.API_KEY);
- ResteasyClientBuilder builder = new ResteasyClientBuilder();
- builder.connectionPoolSize(BUILDER_DEFAULT_POOL_SIZE);
- builder.socketTimeout(BUILDER_DEFAULT_SOCKET_TIMEOUT,
- TimeUnit.SECONDS);
- builder.establishConnectionTimeout(
- BUILDER_DEFAULT_CONNECTION_TIMEOUT, TimeUnit.SECONDS);
- this.restBuilder = builder;
- client = new SearchBloxClient(apiKey, this.restBuilder, endpoint);
- }
-
- if(args != null && !version.equals(lastVersion)){
-
- lastVersion = version;
-
- try{
- int poolSize =
- Integer.parseInt(args.get(
- SearchBloxConfig.ATTRIBUTE_POOLSIZE).get(0));
- this.restBuilder.connectionPoolSize(poolSize);
- }catch(NumberFormatException e){
- Logging.connectors.error("Incorrect Argument Value for Client Pool Size", e);
- }
-
- try{
- int connectionTimeout =
- Integer.parseInt(args.get(
- SearchBloxConfig.ATTRIBUTE_TIMEOUT_CONNECTION).get(0));
- this.restBuilder.connectionPoolSize(connectionTimeout);
- }catch(NumberFormatException e){
- Logging.connectors.error("Incorrect Argument Value for Client Connection Timeout", e);
- }
-
- try{
- int socketTimeout =
- Integer.parseInt(args.get(
- SearchBloxConfig.ATTRIBUTE_TIMEOUT_SOCKET).get(0));
- this.restBuilder.connectionPoolSize(socketTimeout);
- }catch(NumberFormatException e){
- Logging.connectors.error("Incorrect Argument Value for Client Socket Timeout", e);
- }
- }
- }
+ @Override
+ public String check() throws ManifoldCFException {
+ getSession();
+ try {
+ String format = getConfiguration().getParameter(SEARCHBLOX_INDEXING_FORMAT);
+ if (client.ping(format)) {
+ return super.check();
+ } else {
+ return "Connection Not Working!. Check SearchBlox Server is up and the configuration is correct.";
+ }
+ } catch (SearchBloxException e) {
+ Logging.connectors.error("Connection Not Working", e);
+ return "Connection Not Working!" + e.getMessage();
+ }
+ }
- @Override
- public String check() throws ManifoldCFException {
- getSession(null, null);
- try {
- String format = getConfiguration().getParameter(SEARCHBLOX_INDEXING_FORMAT);
- if (client.ping(format)) {
- return super.check();
- } else {
- return "Connection Not Working!. Check SearchBlox Server is up and the configuration is correct.";
- }
- } catch (SearchBloxException e) {
- Logging.connectors.error("Connection Not Working", e);
- return "Connection Not Working!" + e.getMessage();
- }
- }
+ /**
+ * Get an output version string, given an output specification. The output
+ * version string is used to uniquely describe the pertinent details of the
+ * output specification and the configuration, to allow the Connector
+ * Framework to determine whether a document will need to be output again.
+ * Note that the contents of the document cannot be considered by this
+ * method, and that a different version string (defined in
+ * IRepositoryConnector) is used to describe the version of the actual
+ * document.
+ * <p/>
+ * This method presumes that the connector object has been configured, and
+ * it is thus able to communicate with the output data store should that be
+ * necessary.
+ *
+ * @param spec
+ * is the current output specification for the job that is doing
+ * the crawling.
+ * @return a string, of unlimited length, which uniquely describes output
+ * configuration and specification in such a way that if two such
+ * strings are equal, the document will not need to be sent again to
+ * the output data store.
+ */
+ @Override
+ public VersionContext getPipelineDescription(Specification spec)
+ throws ManifoldCFException, ServiceInterruption {
+ SpecPacker sp = new SpecPacker(spec);
+ return new VersionContext(sp.toPackedString(), params, spec);
+ }
- /**
- * Get an output version string, given an output specification. The output
- * version string is used to uniquely describe the pertinent details of the
- * output specification and the configuration, to allow the Connector
- * Framework to determine whether a document will need to be output again.
- * Note that the contents of the document cannot be considered by this
- * method, and that a different version string (defined in
- * IRepositoryConnector) is used to describe the version of the actual
- * document.
- * <p/>
- * This method presumes that the connector object has been configured, and
- * it is thus able to communicate with the output data store should that be
- * necessary.
- *
- * @param spec
- * is the current output specification for the job that is doing
- * the crawling.
- * @return a string, of unlimited length, which uniquely describes output
- * configuration and specification in such a way that if two such
- * strings are equal, the document will not need to be sent again to
- * the output data store.
- */
- @Override
- public VersionContext getPipelineDescription(Specification spec)
- throws ManifoldCFException, ServiceInterruption {
- SpecPacker sp = new SpecPacker(spec);
- return new VersionContext(sp.toPackedString(), params, spec);
- }
+ /**
+ * Detect if a mime type is indexable or not. This method is used by
+ * participating repository connectors to pre-filter the number of unusable
+ * documents that will be passed to this output connector.
+ *
+ * @param outputDescription
+ * is the document's output version.
+ * @param mimeType
+ * is the mime type of the document.
+ * @return true if the mime type is indexable by this connector.
+ */
+ @Override
+ public boolean checkMimeTypeIndexable(VersionContext outputDescription,
+ String mimeType, IOutputCheckActivity activities)
+ throws ManifoldCFException, ServiceInterruption {
+ // We work against the API, so we need to suppose the content reached by
+ // the connector is a String convertible stream
+ return true;
+ }
- /**
- * Detect if a mime type is indexable or not. This method is used by
- * participating repository connectors to pre-filter the number of unusable
- * documents that will be passed to this output connector.
- *
- * @param outputDescription
- * is the document's output version.
- * @param mimeType
- * is the mime type of the document.
- * @return true if the mime type is indexable by this connector.
- */
- @Override
- public boolean checkMimeTypeIndexable(VersionContext outputDescription,
- String mimeType, IOutputCheckActivity activities)
- throws ManifoldCFException, ServiceInterruption {
- // We work against the API, so we need to suppose the content reached by
- // the connector is a String convertible stream
- return true;
- }
+ /**
+ * Pre-determine whether a document's length is indexable by this connector.
+ * This method is used by participating repository connectors to help filter
+ * out documents that are too long to be indexable.
+ *
+ * @param outputDescription
+ * is the document's output version.
+ * @param length
+ * is the length of the document.
+ * @return true if the file is indexable.
+ */
+ @Override
+ public boolean checkLengthIndexable(VersionContext outputDescription,
+ long length, IOutputCheckActivity activities)
+ throws ManifoldCFException, ServiceInterruption {
+ // No Size Limit for SearchBlox
+ return true;
+ }
- /**
- * Pre-determine whether a document's length is indexable by this connector.
- * This method is used by participating repository connectors to help filter
- * out documents that are too long to be indexable.
- *
- * @param outputDescription
- * is the document's output version.
- * @param length
- * is the length of the document.
- * @return true if the file is indexable.
- */
- @Override
- public boolean checkLengthIndexable(VersionContext outputDescription,
- long length, IOutputCheckActivity activities)
- throws ManifoldCFException, ServiceInterruption {
- // No Size Limit for SearchBlox
- return true;
- }
+ /**
+ * Add (or replace) a document in the output data store using the connector.
+ * This method presumes that the connector object has been configured, and
+ * it is thus able to communicate with the output data store should that be
+ * necessary.
+ *
+ * @param documentURI
+ * is the URI of the document. The URI is presumed to be the
+ * unique identifier which the output data store will use to
+ * process and serve the document. This URI is constructed by the
+ * repository connector which fetches the document, and is thus
+ * universal across all output connectors.
+ * @param pipelineDescription
+ * includes the description string that was constructed for this
+ * document by the getOutputDescription() method.
+ * @param document
+ * is the document data to be processed (handed to the output
+ * data store).
+ * @param authorityNameString
+ * is the name of the authority responsible for authorizing any
+ * access tokens passed in with the repository document. May be
+ * null.
+ * @param activities
+ * is the handle to an object that the implementer of a pipeline
+ * connector may use to perform operations, such as logging
+ * processing activity, or sending a modified document to the
+ * next stage in the pipeline.
+ * @return the document status (accepted or permanently rejected).
+ * @throws IOException
+ * only if there's a stream error reading the document data.
+ */
+ @Override
+ public int addOrReplaceDocumentWithException(String documentURI,
+ VersionContext pipelineDescription, RepositoryDocument document,
+ String authorityNameString, IOutputAddActivity activities)
+ throws ManifoldCFException, ServiceInterruption, IOException {
- /**
- * Add (or replace) a document in the output data store using the connector.
- * This method presumes that the connector object has been configured, and
- * it is thus able to communicate with the output data store should that be
- * necessary.
- *
- * @param documentURI
- * is the URI of the document. The URI is presumed to be the
- * unique identifier which the output data store will use to
- * process and serve the document. This URI is constructed by the
- * repository connector which fetches the document, and is thus
- * universal across all output connectors.
- * @param pipelineDescription
- * includes the description string that was constructed for this
- * document by the getOutputDescription() method.
- * @param document
- * is the document data to be processed (handed to the output
- * data store).
- * @param authorityNameString
- * is the name of the authority responsible for authorizing any
- * access tokens passed in with the repository document. May be
- * null.
- * @param activities
- * is the handle to an object that the implementer of a pipeline
- * connector may use to perform operations, such as logging
- * processing activity, or sending a modified document to the
- * next stage in the pipeline.
- * @return the document status (accepted or permanently rejected).
- * @throws IOException
- * only if there's a stream error reading the document data.
- */
- @Override
- public int addOrReplaceDocumentWithException(String documentURI,
- VersionContext pipelineDescription, RepositoryDocument document,
- String authorityNameString, IOutputAddActivity activities)
- throws ManifoldCFException, ServiceInterruption, IOException {
+ Logging.connectors.info("Indexing Document " + documentURI);
+ long indexingTime = System.currentTimeMillis();
+ SpecPacker sp = new SpecPacker(pipelineDescription.getSpecification());
+ Map<String, List<String>> args = sp.getArgs();
+ // Establish a session
+ getSession();
- Logging.connectors.info("Indexing Document " + documentURI);
- long indexingTime = System.currentTimeMillis();
- SpecPacker sp = new SpecPacker(pipelineDescription.getSpecification());
- Map<String, List<String>> args = sp.getArgs();
- // Establish a session
- getSession(args, pipelineDescription.getVersionString());
-
- SearchBloxDocument sbDoc = new SearchBloxDocument(this.apiKey,
- documentURI, document, sp.getArgs());
- String format = this.getConfiguration().getParameter(SEARCHBLOX_INDEXING_FORMAT);
- long startTime = System.currentTimeMillis();
- try {
- ResponseCode code = client.addUpdateDocument(sbDoc, format);
- if (code == ResponseCode.DOCUMENT_INDEXED) {
+ SearchBloxDocument sbDoc = new SearchBloxDocument(this.apiKey,
+ documentURI, document, args);
+ String format = this.getConfiguration().getParameter(SEARCHBLOX_INDEXING_FORMAT);
+ long startTime = System.currentTimeMillis();
+ try {
+ ResponseCode code = client.addUpdateDocument(sbDoc, format);
+ if (code == ResponseCode.DOCUMENT_INDEXED) {
Logging.connectors.info("Document Indexed" + documentURI);
- activities.recordActivity(startTime, INGEST_ACTIVITY, null,
- documentURI, "OK", "Document Indexed");}
- else
- activities.recordActivity(startTime, INGEST_ACTIVITY, null,
- documentURI, "" + code.getCode(), code.name());
- } catch (SearchBloxException e) {
- Logging.connectors
- .error("[Indexing - Add] Exception indexing document :"
- + document, e);
- String activityCode = e.getClass().getSimpleName()
- .toUpperCase(Locale.ROOT);
- String activityDetails = e.getMessage()
- + ((e.getCause() != null) ? ": "
- + e.getCause().getMessage() : "");
- activities.recordActivity(startTime, INGEST_ACTIVITY, null,
- documentURI, activityCode, activityDetails);
- return DOCUMENTSTATUS_REJECTED;
- }
- indexingTime = System.currentTimeMillis() - indexingTime;
- Logging.connectors.info("Indexing Time for document " + documentURI + ": " + indexingTime);
- return DOCUMENTSTATUS_ACCEPTED;
- }
+ activities.recordActivity(startTime, INGEST_ACTIVITY, null,
+ documentURI, "OK", "Document Indexed");}
+ else
+ activities.recordActivity(startTime, INGEST_ACTIVITY, null,
+ documentURI, "" + code.getCode(), code.name());
+ } catch (SearchBloxException e) {
+ Logging.connectors
+ .error("[Indexing - Add] Exception indexing document :"
+ + document, e);
+ String activityCode = e.getClass().getSimpleName()
+ .toUpperCase(Locale.ROOT);
+ String activityDetails = e.getMessage()
+ + ((e.getCause() != null) ? ": "
+ + e.getCause().getMessage() : "");
+ activities.recordActivity(startTime, INGEST_ACTIVITY, null,
+ documentURI, activityCode, activityDetails);
+ return DOCUMENTSTATUS_REJECTED;
+ }
+ indexingTime = System.currentTimeMillis() - indexingTime;
+ Logging.connectors.info("Indexing Time for document " + documentURI + ": " + indexingTime);
+ return DOCUMENTSTATUS_ACCEPTED;
+ }
- /**
- * Remove a document using the connector. Note that the last
- * outputDescription is included, since it may be necessary for the
- * connector to use such information to know how to properly remove the
- * document.
- *
- * @param documentURI
- * is the URI of the document. The URI is presumed to be the
- * unique identifier which the output data store will use to
- * process and serve the document. This URI is constructed by the
- * repository connector which fetches the document, and is thus
- * universal across all output connectors.
- * @param outputDescription
- * is the last description string that was constructed for this
- * document by the getOutputDescription() method above.
- * @param activities
- * is the handle to an object that the implementer of an output
- * connector may use to perform operations, such as logging
- * processing activity.
- */
- @Override
- public void removeDocument(String documentURI, String outputDescription,
- IOutputRemoveActivity activities) throws ManifoldCFException,
- ServiceInterruption {
- Logging.ingest.debug("Deleting SearchBlox Document: '" + documentURI
- + "'");
+ /**
+ * Remove a document using the connector. Note that the last
+ * outputDescription is included, since it may be necessary for the
+ * connector to use such information to know how to properly remove the
+ * document.
+ *
+ * @param documentURI
+ * is the URI of the document. The URI is presumed to be the
+ * unique identifier which the output data store will use to
+ * process and serve the document. This URI is constructed by the
+ * repository connector which fetches the document, and is thus
+ * universal across all output connectors.
+ * @param outputDescription
+ * is the last description string that was constructed for this
+ * document by the getOutputDescription() method above.
+ * @param activities
+ * is the handle to an object that the implementer of an output
+ * connector may use to perform operations, such as logging
+ * processing activity.
+ */
+ @Override
+ public void removeDocument(String documentURI, String outputDescription,
+ IOutputRemoveActivity activities) throws ManifoldCFException,
+ ServiceInterruption {
+ Logging.ingest.debug("Deleting SearchBlox Document: '" + documentURI
+ + "'");
- SpecPacker packer = new SpecPacker(outputDescription);
- Map<String, List<String>> args = packer.getArgs();
- // Establish a session
- getSession(args, packer.toPackedString());
-
- SearchBloxDocument document = new SearchBloxDocument(this.apiKey);
- document.uid = documentURI;
- // document.apiKey = args.get(API_KEY).get(0);
- document.colName = args.get(SearchBloxDocument.SEARCHBLOX_COLLECTION).get(0);
- String format = args.get(SEARCHBLOX_INDEXING_FORMAT).get(0);
- long startTime = System.currentTimeMillis();
- try {
- ResponseCode code = client.deleteDocument(document, format);
- if (code == ResponseCode.DOCUMENT_DELETED)
- activities.recordActivity(startTime, REMOVE_ACTIVITY, null,
- documentURI, "OK", "Document Deleted");
- else
- activities.recordActivity(startTime, REMOVE_ACTIVITY, null,
- documentURI, "" + code.getCode(), code.name());
- } catch (SearchBloxException e) {
- Logging.connectors.error(
- "[Indexing - Remove] Exception indexing document :"
- + document, e);
- String activityCode = e.getClass().getSimpleName()
- .toUpperCase(Locale.ROOT);
- String activityDetails = e.getMessage()
- + ((e.getCause() != null) ? ": "
- + e.getCause().getMessage() : "");
- activities.recordActivity(startTime, REMOVE_ACTIVITY, null,
- documentURI, activityCode, activityDetails);
- }
- }
+ SpecPacker packer = new SpecPacker(outputDescription);
+ Map<String, List<String>> args = packer.getArgs();
+ // Establish a session
+ getSession();
+
+ SearchBloxDocument document = new SearchBloxDocument(this.apiKey);
+ document.uid = documentURI;
+ // document.apiKey = args.get(API_KEY).get(0);
+ document.colName = args.get(SearchBloxDocument.SEARCHBLOX_COLLECTION).get(0);
+ String format = args.get(SEARCHBLOX_INDEXING_FORMAT).get(0);
+ long startTime = System.currentTimeMillis();
+ try {
+ ResponseCode code = client.deleteDocument(document, format);
+ if (code == ResponseCode.DOCUMENT_DELETED)
+ activities.recordActivity(startTime, REMOVE_ACTIVITY, null,
+ documentURI, "OK", "Document Deleted");
+ else
+ activities.recordActivity(startTime, REMOVE_ACTIVITY, null,
+ documentURI, "" + code.getCode(), code.name());
+ } catch (SearchBloxException e) {
+ Logging.connectors.error(
+ "[Indexing - Remove] Exception indexing document :"
+ + document, e);
+ String activityCode = e.getClass().getSimpleName()
+ .toUpperCase(Locale.ROOT);
+ String activityDetails = e.getMessage()
+ + ((e.getCause() != null) ? ": "
+ + e.getCause().getMessage() : "");
+ activities.recordActivity(startTime, REMOVE_ACTIVITY, null,
+ documentURI, activityCode, activityDetails);
+ }
+ }
- /**
- * Read the content of a resource, replace the variable ${PARAMNAME} with
- * the value and copy it to the out.
- *
- * @param resName
- * @param out
- * @throws ManifoldCFException
- */
- private static void outputResource(String resName, IHTTPOutput out,
- Locale locale, Map<String, String> params, String tabName,
- Integer sequenceNumber, Integer currentSequenceNumber)
- throws ManifoldCFException {
- Map<String, String> paramMap = null;
- if (params != null) {
- paramMap = params;
- if (tabName != null) {
- paramMap.put("TabName", tabName);
- }
- if (currentSequenceNumber != null)
- paramMap.put("SelectedNum", currentSequenceNumber.toString());
- } else {
- paramMap = new HashMap<String, String>();
- }
- if (sequenceNumber != null)
- paramMap.put("SeqNum", sequenceNumber.toString());
+ /**
+ * Read the content of a resource, replace the variable ${PARAMNAME} with
+ * the value and copy it to the out.
+ *
+ * @param resName
+ * @param out
+ * @throws ManifoldCFException
+ */
+ private static void outputResource(String resName, IHTTPOutput out,
+ Locale locale, Map<String, String> params, String tabName,
+ Integer sequenceNumber, Integer currentSequenceNumber)
+ throws ManifoldCFException {
+ Map<String, String> paramMap = null;
+ if (params != null) {
+ paramMap = params;
+ if (tabName != null) {
+ paramMap.put("TabName", tabName);
+ }
+ if (currentSequenceNumber != null)
+ paramMap.put("SelectedNum", currentSequenceNumber.toString());
+ } else {
+ paramMap = new HashMap<String, String>();
+ }
+ if (sequenceNumber != null)
+ paramMap.put("SeqNum", sequenceNumber.toString());
- Messages.outputResourceWithVelocity(out, locale, resName, paramMap,
- true);
- }
+ Messages.outputResourceWithVelocity(out, locale, resName, paramMap,
+ true);
+ }
- @Override
- public void outputConfigurationHeader(IThreadContext threadContext,
- IHTTPOutput out, Locale locale, ConfigParams parameters,
- List<String> tabsArray) throws ManifoldCFException, IOException {
- super.outputConfigurationHeader(threadContext, out, locale, parameters,
- tabsArray);
- tabsArray.add(Messages.getString(locale, SEARCHBLOX_TAB_PARAMETERS));
- outputResource(EDIT_CONFIG_HEADER_FORWARD, out, locale, null, null,
- null, null);
- }
+ @Override
+ public void outputConfigurationHeader(IThreadContext threadContext,
+ IHTTPOutput out, Locale locale, ConfigParams parameters,
+ List<String> tabsArray) throws ManifoldCFException, IOException {
+ super.outputConfigurationHeader(threadContext, out, locale, parameters,
+ tabsArray);
+ tabsArray.add(Messages.getString(locale, SEARCHBLOX_TAB_PARAMETERS));
+ outputResource(EDIT_CONFIG_HEADER_FORWARD, out, locale, null, null,
+ null, null);
+ }
- @Override
- public void outputConfigurationBody(IThreadContext threadContext,
- IHTTPOutput out, Locale locale, ConfigParams parameters,
- String tabName) throws ManifoldCFException, IOException {
- super.outputConfigurationBody(threadContext, out, locale, parameters,
- tabName);
- Map<String, String> config = this.getConfigParameters(parameters);
- outputResource(EDIT_CONFIG_FORWARD_PARAMETERS, out, locale, config,
- tabName, null, null);
- }
+ @Override
+ public void outputConfigurationBody(IThreadContext threadContext,
+ IHTTPOutput out, Locale locale, ConfigParams parameters,
+ String tabName) throws ManifoldCFException, IOException {
+ super.outputConfigurationBody(threadContext, out, locale, parameters,
+ tabName);
+ Map<String, String> config = getConfigParameters(parameters);
+ outputResource(EDIT_CONFIG_FORWARD_PARAMETERS, out, locale, config,
+ tabName, null, null);
+ }
- /**
- * Build a Map of SearchBlox parameters. If configParams is null,
- * getConfiguration() is used.
- *
- * @param configParams
- */
- final private Map<String, String> getConfigParameters(
- ConfigParams configParams) {
- Map<String, String> map = new HashMap<String, String>();
- if (configParams == null)
- configParams = getConfiguration();
+ /**
+ * Build a Map of SearchBlox parameters. If configParams is null,
+ * getConfiguration() is used.
+ *
+ * @param configParams
+ */
+ final private Map<String, String> getConfigParameters(
+ ConfigParams configParams) {
+ Map<String, String> map = new HashMap<String, String>();
- String apiKey = configParams.getParameter(SearchBloxDocument.API_KEY);
- if(apiKey == null || apiKey.isEmpty())
- apiKey = DEFAULT_APIKEY;
- map.put(SearchBloxDocument.API_KEY, apiKey);
-
- String endpoint = configParams.getParameter(SEARCHBLOX_ENDPOINT);
- if(endpoint == null || endpoint.isEmpty()) {
- endpoint = SearchBloxClient.DEFAULT_ENDPOINT;
- }
- map.put(SEARCHBLOX_ENDPOINT,
- endpoint);
-
- String indexFormat = configParams.getParameter(SEARCHBLOX_INDEXING_FORMAT);
- indexFormat = indexFormat == null ? IndexingFormat.JSON.name() : indexFormat;
- map.put(SEARCHBLOX_INDEXING_FORMAT, indexFormat);
- return map;
- }
+ String apiKey = configParams.getParameter(SearchBloxDocument.API_KEY);
+ if(apiKey == null)
+ apiKey = DEFAULT_APIKEY;
+ map.put(SearchBloxDocument.API_KEY, apiKey);
+
+ String endpoint = configParams.getParameter(SEARCHBLOX_ENDPOINT);
+ if(endpoint == null) {
+ endpoint = SearchBloxClient.DEFAULT_ENDPOINT;
+ }
+ map.put(SEARCHBLOX_ENDPOINT,
+ endpoint);
+
+ String indexFormat = configParams.getParameter(SEARCHBLOX_INDEXING_FORMAT);
+ if (indexFormat == null) {
+ indexFormat = IndexingFormat.JSON.name();
+ }
+ map.put(SEARCHBLOX_INDEXING_FORMAT, indexFormat);
+
+ String connectionTimeout = configParams.getParameter(SEARCHBLOX_CONNECTION_TIMEOUT);
+ if (connectionTimeout == null) {
+ connectionTimeout = BUILDER_DEFAULT_CONNECTION_TIMEOUT;
+ }
+ map.put(SEARCHBLOX_CONNECTION_TIMEOUT, connectionTimeout);
+
+ String socketTimeout = configParams.getParameter(SEARCHBLOX_SOCKET_TIMEOUT);
+ if (socketTimeout == null) {
+ socketTimeout = BUILDER_DEFAULT_SOCKET_TIMEOUT;
+ }
+ map.put(SEARCHBLOX_SOCKET_TIMEOUT, socketTimeout);
+
+ return map;
+ }
- @Override
- public void viewConfiguration(IThreadContext threadContext,
- IHTTPOutput out, Locale locale, ConfigParams parameters)
- throws ManifoldCFException, IOException {
- outputResource(VIEW_CONFIG_FORWARD, out, locale,
- getConfigParameters(parameters), null, null, null);
- }
+ @Override
+ public void viewConfiguration(IThreadContext threadContext,
+ IHTTPOutput out, Locale locale, ConfigParams parameters)
+ throws ManifoldCFException, IOException {
+ outputResource(VIEW_CONFIG_FORWARD, out, locale,
+ getConfigParameters(parameters), null, null, null);
+ }
- /**
- * Process a configuration post. This method is called at the start of the
- * connector's configuration page, whenever there is a possibility that form
- * data for a connection has been posted. Its purpose is to gather form
- * information and modify the configuration parameters accordingly. The name
- * of the posted form is "editconnection".
- *
- * @param threadContext
- * is the local thread context.
- * @param variableContext
- * is the set of variables available from the post, including
- * binary file post information.
- * @param parameters
- * are the configuration parameters, as they currently exist, for
- * this connection being configured.
- * @return null if all is well, or a string error message if there is an
- * error that should prevent saving of the connection (and cause a
- * redirection to an error page).
- */
- @Override
- public String processConfigurationPost(IThreadContext threadContext,
- IPostParameters variableContext, Locale locale,
- ConfigParams parameters) throws ManifoldCFException {
- String apiKey = variableContext.getParameter(SearchBloxDocument.API_KEY);
- if (apiKey != null)
- parameters.setParameter(SearchBloxDocument.API_KEY, apiKey);
+ /**
+ * Process a configuration post. This method is called at the start of the
+ * connector's configuration page, whenever there is a possibility that form
+ * data for a connection has been posted. Its purpose is to gather form
+ * information and modify the configuration parameters accordingly. The name
+ * of the posted form is "editconnection".
+ *
+ * @param threadContext
+ * is the local thread context.
+ * @param variableContext
+ * is the set of variables available from the post, including
+ * binary file post information.
+ * @param parameters
+ * are the configuration parameters, as they currently exist, for
+ * this connection being configured.
+ * @return null if all is well, or a string error message if there is an
+ * error that should prevent saving of the connection (and cause a
+ * redirection to an error page).
+ */
+ @Override
+ public String processConfigurationPost(IThreadContext threadContext,
+ IPostParameters variableContext, Locale locale,
+ ConfigParams parameters) throws ManifoldCFException {
+ String apiKey = variableContext.getParameter(SearchBloxDocument.API_KEY);
+ if (apiKey != null)
+ parameters.setParameter(SearchBloxDocument.API_KEY, apiKey);
- String endpoint = variableContext.getParameter(SEARCHBLOX_ENDPOINT);
- if (endpoint != null)
- parameters.setParameter(SEARCHBLOX_ENDPOINT, endpoint);
-
- String indexformat = variableContext.getParameter(SEARCHBLOX_INDEXING_FORMAT);
- if (indexformat != null)
- parameters.setParameter(SEARCHBLOX_INDEXING_FORMAT, indexformat);
+ String endpoint = variableContext.getParameter(SEARCHBLOX_ENDPOINT);
+ if (endpoint != null)
+ parameters.setParameter(SEARCHBLOX_ENDPOINT, endpoint);
+
+ String indexformat = variableContext.getParameter(SEARCHBLOX_INDEXING_FORMAT);
+ if (indexformat != null)
+ parameters.setParameter(SEARCHBLOX_INDEXING_FORMAT, indexformat);
- return null;
- }
+ String connectionTimeout = variableContext.getParameter(SEARCHBLOX_CONNECTION_TIMEOUT);
+ if (connectionTimeout != null)
+ parameters.setParameter(SEARCHBLOX_CONNECTION_TIMEOUT, connectionTimeout);
+
+ String socketTimeout = variableContext.getParameter(SEARCHBLOX_SOCKET_TIMEOUT);
+ if (socketTimeout != null)
+ parameters.setParameter(SEARCHBLOX_SOCKET_TIMEOUT, socketTimeout);
+
+ return null;
+ }
- /**
- * Output the specification header section. This method is called in the
- * head section of a job page which has selected a pipeline connection of
- * the current type. Its purpose is to add the required tabs to the list,
- * and to output any javascript methods that might be needed by the job
- * editing HTML.
- *
- * @param out
- * is the output to which any HTML should be sent.
- * @param locale
- * is the preferred local of the output.
- * @param os
- * is the current pipeline specification for this connection.
- * @param connectionSequenceNumber
- * is the unique number of this connection within the job.
- * @param tabsArray
- * is an array of tab names. Add to this array any tab names that
- * are specific to the connector.
- */
- @Override
- public void outputSpecificationHeader(IHTTPOutput out, Locale locale,
- Specification os, int connectionSequenceNumber,
- List<String> tabsArray) throws ManifoldCFException, IOException {
- Map<String, Object> paramMap = new HashMap<String, Object>();
- paramMap.put("SEQNUM", Integer.toString(connectionSequenceNumber));
+ /**
+ * Output the specification header section. This method is called in the
+ * head section of a job page which has selected a pipeline connection of
+ * the current type. Its purpose is to add the required tabs to the list,
+ * and to output any javascript methods that might be needed by the job
+ * editing HTML.
+ *
+ * @param out
+ * is the output to which any HTML should be sent.
+ * @param locale
+ * is the preferred local of the output.
+ * @param os
+ * is the current pipeline specification for this connection.
+ * @param connectionSequenceNumber
+ * is the unique number of this connection within the job.
+ * @param tabsArray
+ * is an array of tab names. Add to this array any tab names that
+ * are specific to the connector.
+ */
+ @Override
+ public void outputSpecificationHeader(IHTTPOutput out, Locale locale,
+ Specification os, int connectionSequenceNumber,
+ List<String> tabsArray) throws ManifoldCFException, IOException {
+ Map<String, Object> paramMap = new HashMap<String, Object>();
+ paramMap.put("SEQNUM", Integer.toString(connectionSequenceNumber));
- tabsArray.add(Messages.getString(locale,
- "SearchBloxConnector.Configuration"));
+ tabsArray.add(Messages.getString(locale,
+ "SearchBloxConnector.Configuration"));
- // Fill in the specification header map, using data from all tabs.
- fillInConfigurationSpecificationMap(paramMap, os);
+ // Fill in the specification header map, using data from all tabs.
+ fillInSpecificationMap(paramMap, os);
- Messages.outputResourceWithVelocity(out, locale, EDIT_SPECIFICATION_JS,
- paramMap);
- }
+ Messages.outputResourceWithVelocity(out, locale, EDIT_SPECIFICATION_JS,
+ paramMap);
+ }
- private void fillInConfigurationSpecificationMap(
- Map<String, Object> paramMap, Specification os) {
+ private void fillInSpecificationMap(
+ Map<String, Object> paramMap, Specification os) {
- for (int i = 0, len = os.getChildCount(); i < len; i++) {
- SpecificationNode sn = os.getChild(i);
- if (sn.getType().equals(SearchBloxConfig.NODE_CONFIGURATION)) {
+ for (int i = 0, len = os.getChildCount(); i < len; i++) {
+ SpecificationNode sn = os.getChild(i);
+ if (sn.getType().equals(SearchBloxConfig.NODE_CONFIGURATION)) {
- String titleBoost = sn
- .getAttributeValue(SearchBloxConfig.ATTRIBUTE_TITLEBOOST);
- if (titleBoost == null || titleBoost.isEmpty())
- titleBoost = "0";
- String contentBoost = sn
- .getAttributeValue(SearchBloxConfig.ATTRIBUTE_CONTENTBOOST);
- if (contentBoost == null || contentBoost.isEmpty())
- contentBoost = "0";
- String keywordsBoost = sn
- .getAttributeValue(SearchBloxConfig.ATTRIBUTE_KEYWORDSBOOST);
- if (keywordsBoost == null || keywordsBoost.isEmpty())
- keywordsBoost = "0";
- String descriptionBoost = sn
- .getAttributeValue(SearchBloxConfig.ATTRIBUTE_DESCRIPTIONBOOST);
- if (descriptionBoost == null || descriptionBoost.isEmpty())
- descriptionBoost = "0";
- String poolSize = sn
- .getAttributeValue(SearchBloxConfig.ATTRIBUTE_POOLSIZE);
- if (poolSize == null || poolSize.isEmpty())
- poolSize = String.valueOf(BUILDER_DEFAULT_POOL_SIZE);
- String timeoutConnection = sn
- .getAttributeValue(SearchBloxConfig.ATTRIBUTE_TIMEOUT_CONNECTION);
- if (timeoutConnection == null || timeoutConnection.isEmpty())
- timeoutConnection = String
- .valueOf(BUILDER_DEFAULT_CONNECTION_TIMEOUT);
- String timeoutSocket = sn
- .getAttributeValue(SearchBloxConfig.ATTRIBUTE_TIMEOUT_SOCKET);
- if (timeoutSocket == null || timeoutSocket.isEmpty())
- timeoutSocket = String
- .valueOf(BUILDER_DEFAULT_SOCKET_TIMEOUT);
+ String titleBoost = sn
+ .getAttributeValue(SearchBloxConfig.ATTRIBUTE_TITLEBOOST);
+ if (titleBoost == null || titleBoost.isEmpty())
+ titleBoost = "0";
+ String contentBoost = sn
+ .getAttributeValue(SearchBloxConfig.ATTRIBUTE_CONTENTBOOST);
+ if (contentBoost == null || contentBoost.isEmpty())
+ contentBoost = "0";
+ String keywordsBoost = sn
+ .getAttributeValue(SearchBloxConfig.ATTRIBUTE_KEYWORDSBOOST);
+ if (keywordsBoost == null || keywordsBoost.isEmpty())
+ keywordsBoost = "0";
+ String descriptionBoost = sn
+ .getAttributeValue(SearchBloxConfig.ATTRIBUTE_DESCRIPTIONBOOST);
+ if (descriptionBoost == null || descriptionBoost.isEmpty())
+ descriptionBoost = "0";
- String collection = sn
- .getAttributeValue(SearchBloxConfig.ATTRIBUTE_COLLECTION_NAME);
- if (collection == null)
- collection = DEFAULT_COLLECTION;
-
- paramMap.put(SearchBloxConfig.ATTRIBUTE_TITLEBOOST.toUpperCase(),
- titleBoost);
- paramMap.put(SearchBloxConfig.ATTRIBUTE_CONTENTBOOST.toUpperCase(),
- contentBoost);
- paramMap.put(SearchBloxConfig.ATTRIBUTE_KEYWORDSBOOST.toUpperCase(),
- keywordsBoost);
- paramMap.put(SearchBloxConfig.ATTRIBUTE_DESCRIPTIONBOOST.toUpperCase(),
- descriptionBoost);
- paramMap.put(SearchBloxConfig.ATTRIBUTE_POOLSIZE.toUpperCase(),
- poolSize);
- paramMap.put(SearchBloxConfig.ATTRIBUTE_TIMEOUT_CONNECTION
- .toUpperCase(), timeoutConnection);
- paramMap.put(
- SearchBloxConfig.ATTRIBUTE_TIMEOUT_SOCKET.toUpperCase(),
- timeoutSocket);
- paramMap.put(SearchBloxConfig.ATTRIBUTE_COLLECTION_NAME
- .toUpperCase(), collection);
+ String collection = sn
+ .getAttributeValue(SearchBloxConfig.ATTRIBUTE_COLLECTION_NAME);
+ if (collection == null)
+ collection = DEFAULT_COLLECTION;
+
+ paramMap.put(SearchBloxConfig.ATTRIBUTE_TITLEBOOST.toUpperCase(),
+ titleBoost);
+ paramMap.put(SearchBloxConfig.ATTRIBUTE_CONTENTBOOST.toUpperCase(),
+ contentBoost);
+ paramMap.put(SearchBloxConfig.ATTRIBUTE_KEYWORDSBOOST.toUpperCase(),
+ keywordsBoost);
+ paramMap.put(SearchBloxConfig.ATTRIBUTE_DESCRIPTIONBOOST.toUpperCase(),
+ descriptionBoost);
+ paramMap.put(SearchBloxConfig.ATTRIBUTE_COLLECTION_NAME
+ .toUpperCase(), collection);
- return;
- }
+ return;
+ }
- }
+ }
- paramMap.put(SearchBloxConfig.ATTRIBUTE_TITLEBOOST.toUpperCase(), 0);
- paramMap.put(SearchBloxConfig.ATTRIBUTE_CONTENTBOOST.toUpperCase(), 0);
- paramMap.put(SearchBloxConfig.ATTRIBUTE_KEYWORDSBOOST.toUpperCase(), 0);
- paramMap.put(SearchBloxConfig.ATTRIBUTE_DESCRIPTIONBOOST.toUpperCase(), 0);
- paramMap.put(SearchBloxConfig.ATTRIBUTE_POOLSIZE.toUpperCase(),
- BUILDER_DEFAULT_POOL_SIZE);
- paramMap.put(
- SearchBloxConfig.ATTRIBUTE_TIMEOUT_CONNECTION.toUpperCase(),
- BUILDER_DEFAULT_CONNECTION_TIMEOUT);
- paramMap.put(SearchBloxConfig.ATTRIBUTE_TIMEOUT_SOCKET.toUpperCase(),
- BUILDER_DEFAULT_SOCKET_TIMEOUT);
- paramMap.put(SearchBloxConfig.ATTRIBUTE_INDEX_FORMAT.toUpperCase(),
- IndexingFormat.XML.name());
- paramMap.put(SearchBloxConfig.ATTRIBUTE_COLLECTION_NAME.toUpperCase(),
- "");
+ paramMap.put(SearchBloxConfig.ATTRIBUTE_TITLEBOOST.toUpperCase(), 0);
+ paramMap.put(SearchBloxConfig.ATTRIBUTE_CONTENTBOOST.toUpperCase(), 0);
+ paramMap.put(SearchBloxConfig.ATTRIBUTE_KEYWORDSBOOST.toUpperCase(), 0);
+ paramMap.put(SearchBloxConfig.ATTRIBUTE_DESCRIPTIONBOOST.toUpperCase(), 0);
+ paramMap.put(SearchBloxConfig.ATTRIBUTE_INDEX_FORMAT.toUpperCase(),
+ IndexingFormat.XML.name());
+ paramMap.put(SearchBloxConfig.ATTRIBUTE_COLLECTION_NAME.toUpperCase(),
+ "");
- }
+ }
- /**
- * View specification. This method is called in the body section of a job's
- * view page. Its purpose is to present the pipeline specification
- * information to the user. The coder can presume that the HTML that is
- * output from this configuration will be within appropriate <html> and
- * <body> tags.
- *
- * @param out
- * is the output to which any HTML should be sent.
- * @param locale
- * is the preferred local of the output.
- * @param connectionSequenceNumber
- * is the unique number of this connection within the job.
- * @param os
- * is the current pipeline specification for this job.
- */
- @Override
- public void viewSpecification(IHTTPOutput out, Locale locale,
- Specification os, int connectionSequenceNumber)
- throws ManifoldCFException, IOException {
- Map<String, Object> paramMap = new HashMap<String, Object>();
- paramMap.put("SEQNUM", Integer.toString(connectionSequenceNumber));
+ /**
+ * View specification. This method is called in the body section of a job's
+ * view page. Its purpose is to present the pipeline specification
+ * information to the user. The coder can presume that the HTML that is
+ * output from this configuration will be within appropriate <html> and
+ * <body> tags.
+ *
+ * @param out
+ * is the output to which any HTML should be sent.
+ * @param locale
+ * is the preferred local of the output.
+ * @param connectionSequenceNumber
+ * is the unique number of this connection within the job.
+ * @param os
+ * is the current pipeline specification for this job.
+ */
+ @Override
+ public void viewSpecification(IHTTPOutput out, Locale locale,
+ Specification os, int connectionSequenceNumber)
+ throws ManifoldCFException, IOException {
+ Map<String, Object> paramMap = new HashMap<String, Object>();
+ paramMap.put("SEQNUM", Integer.toString(connectionSequenceNumber));
- // Fill in the map with data from all tabs
- fillInConfigurationSpecificationMap(paramMap, os);
+ // Fill in the map with data from all tabs
+ fillInSpecificationMap(paramMap, os);
- Messages.outputResourceWithVelocity(out, locale,
- VIEW_SPECIFICATION_HTML, paramMap);
+ Messages.outputResourceWithVelocity(out, locale,
+ VIEW_SPECIFICATION_HTML, paramMap);
- }
+ }
- /**
- * Output the specification body section. This method is called in the body
- * section of a job page which has selected a pipeline connection of the
- * current type. Its purpose is to present the required form elements for
- * editing. The coder can presume that the HTML that is output from this
- * configuration will be within appropriate <html>, <body>, and <form> tags.
- * The name of the form is "editjob".
- *
- * @param out
- * is the output to which any HTML should be sent.
- * @param locale
- * is the preferred local of the output.
- * @param os
- * is the current pipeline specification for this job.
- * @param connectionSequenceNumber
- * is the unique number of this connection within the job.
- * @param actualSequenceNumber
- * is the connection within the job that has currently been
- * selected.
- * @param tabName
- * is the current tab name.
- */
- @Override
- public void outputSpecificationBody(IHTTPOutput out, Locale locale,
- Specification os, int connectionSequenceNumber,
- int actualSequenceNumber, String tabName)
- throws ManifoldCFException, IOException {
- Map<String, Object> paramMap = new HashMap<String, Object>();
+ /**
+ * Output the specification body section. This method is called in the body
+ * section of a job page which has selected a pipeline connection of the
+ * current type. Its purpose is to present the required form elements for
+ * editing. The coder can presume that the HTML that is output from this
+ * configuration will be within appropriate <html>, <body>, and <form> tags.
+ * The name of the form is "editjob".
+ *
+ * @param out
+ * is the output to which any HTML should be sent.
+ * @param locale
+ * is the preferred local of the output.
+ * @param os
+ * is the current pipeline specification for this job.
+ * @param connectionSequenceNumber
+ * is the unique number of this connection within the job.
+ * @param actualSequenceNumber
+ * is the connection within the job that has currently been
+ * selected.
+ * @param tabName
+ * is the current tab name.
+ */
+ @Override
+ public void outputSpecificationBody(IHTTPOutput out, Locale locale,
+ Specification os, int connectionSequenceNumber,
+ int actualSequenceNumber, String tabName)
+ throws ManifoldCFException, IOException {
+ Map<String, Object> paramMap = new HashMap<String, Object>();
- // Set the tab name
- paramMap.put("TABNAME", tabName);
- paramMap.put("SEQNUM", Integer.toString(connectionSequenceNumber));
- paramMap.put("SELECTEDNUM", Integer.toString(actualSequenceNumber));
+ // Set the tab name
+ paramMap.put("TABNAME", tabName);
+ paramMap.put("SEQNUM", Integer.toString(connectionSequenceNumber));
+ paramMap.put("SELECTEDNUM", Integer.toString(actualSequenceNumber));
- // Fill in the field mapping tab data
- fillInConfigurationSpecificationMap(paramMap, os);
+ // Fill in the field mapping tab data
+ fillInSpecificationMap(paramMap, os);
- Messages.outputResourceWithVelocity(out, locale,
- EDIT_SPECIFICATION_CONFIGURATION_HTML, paramMap);
+ Messages.outputResourceWithVelocity(out, locale,
+ EDIT_SPECIFICATION_CONFIGURATION_HTML, paramMap);
- }
+ }
- /**
- * Process a specification post. This method is called at the start of job's
- * edit or view page, whenever there is a possibility that form data for a
- * connection has been posted. Its purpose is to gather form information and
- * modify the transformation specification accordingly. The name of the
- * posted form is "editjob".
- *
- * @param variableContext
- * contains the post data, including binary file-upload
- * information.
- * @param locale
- * is the preferred local of the output.
- * @param os
- * is the current pipeline specification for this job.
- * @param connectionSequenceNumber
- * is the unique number of this connection within the job.
- * @return null if all is well, or a string error message if there is an
- * error that should prevent saving of the job (and cause a
- * redirection to an error page).
- */
- @Override
- public String processSpecificationPost(IPostParameters variableContext,
- Locale locale, Specification os, int connectionSequenceNumber)
- throws ManifoldCFException {
- String seqPrefix = "s" + connectionSequenceNumber + "_";
- String titleBoost = variableContext.getParameter(seqPrefix
- + SearchBloxConfig.ATTRIBUTE_TITLEBOOST);
- String contentBoost = variableContext.getParameter(seqPrefix
- + SearchBloxConfig.ATTRIBUTE_CONTENTBOOST);
- String keywordsBoost = variableContext.getParameter(seqPrefix
- + SearchBloxConfig.ATTRIBUTE_KEYWORDSBOOST);
- String descriptionBoost = variableContext.getParameter(seqPrefix
- + SearchBloxConfig.ATTRIBUTE_DESCRIPTIONBOOST);
- String poolSize = variableContext.getParameter(seqPrefix
- + SearchBloxConfig.ATTRIBUTE_POOLSIZE);
- String timeoutConnection = variableContext.getParameter(seqPrefix
- + SearchBloxConfig.ATTRIBUTE_TIMEOUT_CONNECTION);
- String timeoutSocket = variableContext.getParameter(seqPrefix
- + SearchBloxConfig.ATTRIBUTE_TIMEOUT_SOCKET);
- String collection = variableContext.getParameter(seqPrefix
- + SearchBloxConfig.ATTRIBUTE_COLLECTION_NAME);
- String indexFormat = variableContext.getParameter(seqPrefix
- + SearchBloxConfig.ATTRIBUTE_INDEX_FORMAT);
+ /**
+ * Process a specification post. This method is called at the start of job's
+ * edit or view page, whenever there is a possibility that form data for a
+ * connection has been posted. Its purpose is to gather form information and
+ * modify the transformation specification accordingly. The name of the
+ * posted form is "editjob".
+ *
+ * @param variableContext
+ * contains the post data, including binary file-upload
+ * information.
+ * @param locale
+ * is the preferred local of the output.
+ * @param os
+ * is the current pipeline specification for this job.
+ * @param connectionSequenceNumber
+ * is the unique number of this connection within the job.
+ * @return null if all is well, or a string error message if there is an
+ * error that should prevent saving of the job (and cause a
+ * redirection to an error page).
+ */
+ @Override
+ public String processSpecificationPost(IPostParameters variableContext,
+ Locale locale, Specification os, int connectionSequenceNumber)
+ throws ManifoldCFException {
+ String seqPrefix = "s" + connectionSequenceNumber + "_";
+ String titleBoost = variableContext.getParameter(seqPrefix
+ + SearchBloxConfig.ATTRIBUTE_TITLEBOOST);
+ String contentBoost = variableContext.getParameter(seqPrefix
+ + SearchBloxConfig.ATTRIBUTE_CONTENTBOOST);
+ String keywordsBoost = variableContext.getParameter(seqPrefix
+ + SearchBloxConfig.ATTRIBUTE_KEYWORDSBOOST);
+ String descriptionBoost = variableContext.getParameter(seqPrefix
+ + SearchBloxConfig.ATTRIBUTE_DESCRIPTIONBOOST);
+ String collection = variableContext.getParameter(seqPrefix
+ + SearchBloxConfig.ATTRIBUTE_COLLECTION_NAME);
+ String indexFormat = variableContext.getParameter(seqPrefix
+ + SearchBloxConfig.ATTRIBUTE_INDEX_FORMAT);
- // About to gather the configuration values, so get rid of the old one.
- int i = 0, len = os.getChildCount();
- while (i < len) {
- SpecificationNode node = os.getChild(i);
- if (node.getType().equals(SearchBloxConfig.NODE_CONFIGURATION))
- os.removeChild(i);
- else
- i++;
- }
+ // About to gather the configuration values, so get rid of the old one.
+ int i = 0, len = os.getChildCount();
+ while (i < len) {
+ SpecificationNode node = os.getChild(i);
+ if (node.getType().equals(SearchBloxConfig.NODE_CONFIGURATION))
+ os.removeChild(i);
+ else
+ i++;
+ }
- SpecificationNode node = new SpecificationNode(
- SearchBloxConfig.NODE_CONFIGURATION);
- node.setAttribute(SearchBloxConfig.ATTRIBUTE_TITLEBOOST, titleBoost);
- node.setAttribute(SearchBloxConfig.ATTRIBUTE_CONTENTBOOST, contentBoost);
- node.setAttribute(SearchBloxConfig.ATTRIBUTE_KEYWORDSBOOST, keywordsBoost);
- node.setAttribute(SearchBloxConfig.ATTRIBUTE_DESCRIPTIONBOOST, descriptionBoost);
- node.setAttribute(SearchBloxConfig.ATTRIBUTE_POOLSIZE, poolSize);
- node.setAttribute(SearchBloxConfig.ATTRIBUTE_TIMEOUT_CONNECTION,
- timeoutConnection);
- node.setAttribute(SearchBloxConfig.ATTRIBUTE_TIMEOUT_SOCKET,
- timeoutSocket);
- node.setAttribute(SearchBloxConfig.ATTRIBUTE_COLLECTION_NAME,
- collection);
- node.setAttribute(SearchBloxConfig.ATTRIBUTE_INDEX_FORMAT, indexFormat);
- os.addChild(os.getChildCount(), node);
+ SpecificationNode node = new SpecificationNode(
+ SearchBloxConfig.NODE_CONFIGURATION);
+ node.setAttribute(SearchBloxConfig.ATTRIBUTE_TITLEBOOST, titleBoost);
+ node.setAttribute(SearchBloxConfig.ATTRIBUTE_CONTENTBOOST, contentBoost);
+ node.setAttribute(SearchBloxConfig.ATTRIBUTE_KEYWORDSBOOST, keywordsBoost);
+ node.setAttribute(SearchBloxConfig.ATTRIBUTE_DESCRIPTIONBOOST, descriptionBoost);
+ node.setAttribute(SearchBloxConfig.ATTRIBUTE_COLLECTION_NAME,
+ collection);
+ node.setAttribute(SearchBloxConfig.ATTRIBUTE_INDEX_FORMAT, indexFormat);
+ os.addChild(os.getChildCount(), node);
- return null;
+ return null;
- }
-
+ }
+
- protected class SpecPacker {
- /** Arguments, from configuration */
- private final Multimap<String, String> args = HashMultimap.create();
-
- public SpecPacker(String outputDescription) {
- String[] parts = outputDescription.split(",");
- for(String part : parts) {
- String[] keyValue = part.split("=");
- if(keyValue.length != 2) {
- continue;
- }
-
- args.put(keyValue[0], keyValue[1]);
- }
- }
-
- public SpecPacker(Specification spec) {
- // Process arguments
- for (int i = 0; i < spec.getChildCount(); i++)
- {
- SpecificationNode node = spec.getChild(i);
- if (node.getType().equals(SearchBloxConfig.NODE_CONFIGURATION))
- {
- String titleBoost = node.getAttributeValue(SearchBloxConfig.ATTRIBUTE_TITLEBOOST);
- String contentBoost = node.getAttributeValue(SearchBloxConfig.ATTRIBUTE_CONTENTBOOST);
- String keywordsBoost = node.getAttributeValue(SearchBloxConfig.ATTRIBUTE_KEYWORDSBOOST);
- String descriptionBoost = node.getAttributeValue(SearchBloxConfig.ATTRIBUTE_DESCRIPTIONBOOST);
- String collection = node.getAttributeValue(SearchBloxConfig.ATTRIBUTE_COLLECTION_NAME);
- String poolSize = node.getAttributeValue(SearchBloxConfig.ATTRIBUTE_POOLSIZE);
- String connectTimeout = node.getAttributeValue(SearchBloxConfig.ATTRIBUTE_TIMEOUT_CONNECTION);
- String socketTimeout = node.getAttributeValue(SearchBloxConfig.ATTRIBUTE_TIMEOUT_SOCKET);
- args.put(SearchBloxConfig.ATTRIBUTE_TITLEBOOST, titleBoost);
- args.put(SearchBloxConfig.ATTRIBUTE_CONTENTBOOST, contentBoost);
- args.put(SearchBloxConfig.ATTRIBUTE_KEYWORDSBOOST, keywordsBoost);
- args.put(SearchBloxConfig.ATTRIBUTE_DESCRIPTIONBOOST, descriptionBoost);
- args.put(SearchBloxDocument.SEARCHBLOX_COLLECTION, collection);
- args.put(SearchBloxConfig.ATTRIBUTE_POOLSIZE, poolSize);
- args.put(SearchBloxConfig.ATTRIBUTE_TIMEOUT_CONNECTION, connectTimeout);
- args.put(SearchBloxConfig.ATTRIBUTE_TIMEOUT_SOCKET, socketTimeout);
-
- }
- }
-
- }
-
- public String toPackedString() {
- Map<String, List<String>> mapList = getArgs();
- StringBuilder sb = new StringBuilder();
- for(String s : mapList.keySet()) {
- sb.append(s).append("=").append(mapList.get(s).get(0));
- sb.append(",");
- }
- if(sb.toString().length()!=0)
- return sb.substring(0, sb.length()-1);
- else
- return "";
-
- }
-
- public Map<String,List<String>> getArgs() {
- Map<String,List<String>> result = Maps.newHashMap();
- for(String s : args.keySet()) {
- Collection<String> list = args.get(s);
- if(list instanceof List) {
- result.put(s, (List<String>) list);
- }
- else {
- List<String> l = Lists.newArrayList(list);
- result.put(s, l);
- }
- }
- return result;
- }
- }
+ protected static class SpecPacker {
+ /** Arguments, from configuration */
+ private final Multimap<String, String> args = HashMultimap.create();
+
+ public SpecPacker(String outputDescription) {
+ String[] parts = outputDescription.split(",");
+ for(String part : parts) {
+ String[] keyValue = part.split("=");
+ if(keyValue.length != 2) {
+ continue;
+ }
+
+ args.put(keyValue[0], keyValue[1]);
+ }
+ }
+
+ public SpecPacker(Specification spec) {
+ // Process arguments
+ for (int i = 0; i < spec.getChildCount(); i++)
+ {
+ SpecificationNode node = spec.getChild(i);
+ if (node.getType().equals(SearchBloxConfig.NODE_CONFIGURATION))
+ {
+ String titleBoost = node.getAttributeValue(SearchBloxConfig.ATTRIBUTE_TITLEBOOST);
+ String contentBoost = node.getAttributeValue(SearchBloxConfig.ATTRIBUTE_CONTENTBOOST);
+ String keywordsBoost = node.getAttributeValue(SearchBloxConfig.ATTRIBUTE_KEYWORDSBOOST);
+ String descriptionBoost = node.getAttributeValue(SearchBloxConfig.ATTRIBUTE_DESCRIPTIONBOOST);
+ String collection = node.getAttributeValue(SearchBloxConfig.ATTRIBUTE_COLLECTION_NAME);
+ args.put(SearchBloxConfig.ATTRIBUTE_TITLEBOOST, titleBoost);
+ args.put(SearchBloxConfig.ATTRIBUTE_CONTENTBOOST, contentBoost);
+ args.put(SearchBloxConfig.ATTRIBUTE_KEYWORDSBOOST, keywordsBoost);
+ args.put(SearchBloxConfig.ATTRIBUTE_DESCRIPTIONBOOST, descriptionBoost);
+ args.put(SearchBloxDocument.SEARCHBLOX_COLLECTION, collection);
+
+ }
+ }
+
+ }
+
+ public String toPackedString() {
+ Map<String, List<String>> mapList = getArgs();
+ StringBuilder sb = new StringBuilder();
+ for(String s : mapList.keySet()) {
+ sb.append(s).append("=").append(mapList.get(s).get(0));
+ sb.append(",");
+ }
+ if(sb.toString().length()!=0)
+ return sb.substring(0, sb.length()-1);
+ else
+ return "";
+
+ }
+
+ public Map<String,List<String>> getArgs() {
+ Map<String,List<String>> result = Maps.newHashMap();
+ for(String s : args.keySet()) {
+ Collection<String> list = args.get(s);
+ if(list instanceof List) {
+ result.put(s, (List<String>) list);
+ }
+ else {
+ List<String> l = Lists.newArrayList(list);
+ result.put(s, l);
+ }
+ }
+ return result;
+ }
+ }
}
diff --git a/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxDocument.java b/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxDocument.java
index a6de179..6a21252 100644
--- a/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxDocument.java
+++ b/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxDocument.java
@@ -51,316 +51,316 @@
*/
public class SearchBloxDocument {
- static final String API_KEY = "apikey";
- static final String SEARCHBLOX_COLLECTION = "collection";
- static final String DATE_FORMAT = "dd MMMM yyyy HH:mm:ss z";
+ static final String API_KEY = "apikey";
+ static final String SEARCHBLOX_COLLECTION = "collection";
+ static final String DATE_FORMAT = "dd MMMM yyyy HH:mm:ss z";
- public enum IndexingFormat {
- JSON, XML
- }
+ public enum IndexingFormat {
+ JSON, XML
+ }
- public enum DocumentAction {
- ADD_UPDATE, DELETE, STATUS, CREATE, CLEAR
- }
- static final List<String> xmlElements= Lists.newArrayList("searchblox","document","url","title","keywords","content","description","lastmodified","size",
- "alpha","contenttype","category","meta","uid");
+ public enum DocumentAction {
+ ADD_UPDATE, DELETE, STATUS, CREATE, CLEAR
+ }
+ static final List<String> xmlElements= Lists.newArrayList("searchblox","document","url","title","keywords","content","description","lastmodified","size",
+ "alpha","contenttype","category","meta","uid");
- static final String COLNAME_ATTRIBUTE = "colname";
- static final String APIKEY_ATTRIBUTE = "apikey";
- static final String NAME_ATTRIBUTE = "name";
- static final String UID_ATTRIBUTE = "uid";
- static final String BOOST_ATTRIBUTE = "boost";
+ static final String COLNAME_ATTRIBUTE = "colname";
+ static final String APIKEY_ATTRIBUTE = "apikey";
+ static final String NAME_ATTRIBUTE = "name";
+ static final String UID_ATTRIBUTE = "uid";
+ static final String BOOST_ATTRIBUTE = "boost";
- private Multimap<String, Object> data_fields = HashMultimap.create();
+ private Multimap<String, Object> data_fields = HashMultimap.create();
- /**
- * API key accessible in the SearchBlox Admin Console.
- */
- String apiKey;
+ /**
+ * API key accessible in the SearchBlox Admin Console.
+ */
+ String apiKey;
- /**
- * Name of the Custom collection
- */
- String colName;
+ /**
+ * Name of the Custom collection
+ */
+ String colName;
- /**
- * unique identifer for a document (default when unassigned is url location)
- */
- String uid;
+ /**
+ * unique identifer for a document (default when unassigned is url location)
+ */
+ String uid;
- public SearchBloxDocument(String apikey) {
- this.apiKey = apikey;
- }
+ public SearchBloxDocument(String apikey) {
+ this.apiKey = apikey;
+ }
- public SearchBloxDocument(String apikey, String documentURI,
- RepositoryDocument rd, Map<String, List<String>> args) {
- this(apikey);
- SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
+ public SearchBloxDocument(String apikey, String documentURI,
+ RepositoryDocument rd, Map<String, List<String>> args) {
+ this(apikey);
+ SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
- this.uid = documentURI;
- this.colName = args.get(SEARCHBLOX_COLLECTION).get(0);
+ this.uid = documentURI;
+ this.colName = args.get(SEARCHBLOX_COLLECTION).get(0);
- Date date = rd.getModifiedDate();
- if(date!=null){
- data_fields.put(xmlElements.get(7),
- dateFormat.format(rd.getModifiedDate()));
- }
+ Date date = rd.getModifiedDate();
+ if(date!=null){
+ data_fields.put(xmlElements.get(7),
+ dateFormat.format(rd.getModifiedDate()));
+ }
- // content
- String content = "";
- try {
- if (rd.getField(xmlElements.get(5)) != null)
- content = (String) rd.getField(xmlElements.get(5))[0];
- else
- content = this.buildString(rd.getBinaryStream());
- } catch (IOException e) {
- Logging.connectors
- .error("[Parsing Content]Content is not text plain, verify you are properly using Apache Tika Transformer",
- e);
- }
- data_fields.put(xmlElements.get(5), this.clean(content));
+ // content
+ String content = "";
+ try {
+ if (rd.getField(xmlElements.get(5)) != null)
+ content = (String) rd.getField(xmlElements.get(5))[0];
+ else
+ content = this.buildString(rd.getBinaryStream());
+ } catch (IOException e) {
+ Logging.connectors
+ .error("[Parsing Content]Content is not text plain, verify you are properly using Apache Tika Transformer",
+ e);
+ }
+ data_fields.put(xmlElements.get(5), this.clean(content));
- // Content Type
- data_fields.put(xmlElements.get(10), rd.getMimeType());
+ // Content Type
+ data_fields.put(xmlElements.get(10), rd.getMimeType());
- // Size
- data_fields.put(xmlElements.get(8), "" + rd.getBinaryLength());
+ // Size
+ data_fields.put(xmlElements.get(8), "" + rd.getBinaryLength());
- // Boosting
- for(String boostId:args.keySet()){
- if(boostId.endsWith("_boost")){
- List<String> argBoost = args.get(boostId);
- if(argBoost!=null && !argBoost.isEmpty())
- data_fields.put(boostId,argBoost.get(0));
- }
- }
+ // Boosting
+ for(String boostId:args.keySet()){
+ if(boostId.endsWith("_boost")){
+ List<String> argBoost = args.get(boostId);
+ if(argBoost!=null && !argBoost.isEmpty())
+ data_fields.put(boostId,argBoost.get(0));
+ }
+ }
- // Metadata
- Multimap<String, String> metadata = HashMultimap.create();
- Iterator<String> it = rd.getFields();
- while (it.hasNext()) {
- String name = it.next();
- try {
- String[] values = rd.getFieldAsStrings(name);
- for (String value : values) {
- String key = name.toLowerCase();
- int indexOf = xmlElements.indexOf(key);
- if(indexOf != 5)
- if (indexOf != -1 &&
- indexOf != 0 &&
- indexOf != 7 &&
- indexOf != 8) {
- data_fields.put(key, value);
- } else
- metadata.put(name, value);
- }
- } catch (IOException e) {
- Logging.connectors.error(
- "[Getting Field Values]Impossible to read value for metadata "
- + name, e);
- }
- }
+ // Metadata
+ Multimap<String, String> metadata = HashMultimap.create();
+ Iterator<String> it = rd.getFields();
+ while (it.hasNext()) {
+ String name = it.next();
+ try {
+ String[] values = rd.getFieldAsStrings(name);
+ for (String value : values) {
+ String key = name.toLowerCase();
+ int indexOf = xmlElements.indexOf(key);
+ if(indexOf != 5)
+ if (indexOf != -1 &&
+ indexOf != 0 &&
+ indexOf != 7 &&
+ indexOf != 8) {
+ data_fields.put(key, value);
+ } else
+ metadata.put(name, value);
+ }
+ } catch (IOException e) {
+ Logging.connectors.error(
+ "[Getting Field Values]Impossible to read value for metadata "
+ + name, e);
+ }
+ }
- // ACLS must be stored as metadata, as Searchblox use that construct to index custom named fields
- //the approach has been implemented and tested live
- Iterator<String> aclTypes = rd.securityTypesIterator();
- while (aclTypes.hasNext()) {
- String aclType = aclTypes.next();
- String[] allow_tokens = rd.getSecurityACL(aclType);
- for (String token : allow_tokens)
- metadata.put(aclType+"_allow", token);
- String[] deny_tokens = rd.getSecurityDenyACL(aclType);
- for (String token : deny_tokens)
- metadata.put(aclType+"_deny", token);
- }
- data_fields.put(xmlElements.get(12), metadata);
- }
+ // ACLS must be stored as metadata, as Searchblox use that construct to index custom named fields
+ //the approach has been implemented and tested live
+ Iterator<String> aclTypes = rd.securityTypesIterator();
+ while (aclTypes.hasNext()) {
+ String aclType = aclTypes.next();
+ String[] allow_tokens = rd.getSecurityACL(aclType);
+ for (String token : allow_tokens)
+ metadata.put(aclType+"_allow", token);
+ String[] deny_tokens = rd.getSecurityDenyACL(aclType);
+ for (String token : deny_tokens)
+ metadata.put(aclType+"_deny", token);
+ }
+ data_fields.put(xmlElements.get(12), metadata);
+ }
- /**
- * Clean a String from html tags or break lines
- * @param content
- * @return
- */
- private String clean(String content) {
- content = content.replaceAll("(\r\n|\n)", " ");
- String cleanContent= Jsoup.parseBodyFragment(content).text();
- return cleanContent;
- }
+ /**
+ * Clean a String from html tags or break lines
+ * @param content
+ * @return
+ */
+ private String clean(String content) {
+ content = content.replaceAll("(\r\n|\n)", " ");
+ String cleanContent= Jsoup.parseBodyFragment(content).text();
+ return cleanContent;
+ }
- private String buildString(InputStream binaryStream) throws IOException {
- StringWriter writer = new StringWriter();
- IOUtils.copy(binaryStream, writer, "UTF-8");
- return writer.toString();
- }
+ private String buildString(InputStream binaryStream) throws IOException {
+ StringWriter writer = new StringWriter();
+ IOUtils.copy(binaryStream, writer, "UTF-8");
+ return writer.toString();
+ }
- public String toString(IndexingFormat format, DocumentAction action)
- throws SearchBloxException {
- if(format == IndexingFormat.XML)
- return toStringXML(action);
- else
- return toStringJSON(action);
- }
+ public String toString(IndexingFormat format, DocumentAction action)
+ throws SearchBloxException {
+ if(format == IndexingFormat.XML)
+ return toStringXML(action);
+ else
+ return toStringJSON(action);
+ }
- private String toStringJSON(DocumentAction action) throws SearchBloxException {
- JSONObject result = new JSONObject();
- if (apiKey == null)
- throw new SearchBloxException(
- "The API Key for accessing SearchBlox Server CAN'T be NULL");
- try {
- result.put(APIKEY_ATTRIBUTE, apiKey);
+ private String toStringJSON(DocumentAction action) throws SearchBloxException {
+ JSONObject result = new JSONObject();
+ if (apiKey == null)
+ throw new SearchBloxException(
+ "The API Key for accessing SearchBlox Server CAN'T be NULL");
+ try {
+ result.put(APIKEY_ATTRIBUTE, apiKey);
- JSONObject document = new JSONObject();
- if (colName == null)
- throw new SearchBloxException(
- "The Collection Name of the SearchBlox Server CAN'T be NULL");
- document.put(COLNAME_ATTRIBUTE, colName);
- document.put(UID_ATTRIBUTE, uid);
+ JSONObject document = new JSONObject();
+ if (colName == null)
+ throw new SearchBloxException(
+ "The Collection Name of the SearchBlox Server CAN'T be NULL");
+ document.put(COLNAME_ATTRIBUTE, colName);
+ document.put(UID_ATTRIBUTE, uid);
- if(action == DocumentAction.ADD_UPDATE){
- for(String element:xmlElements){
- if (!element.equals(xmlElements.get(12))) {
- Collection<Object> values = data_fields.get(element);
- if (values!=null && values.size()>0) {
- Object next = values.iterator()
- .next();
- String value =(String) next;
- if (value != null && !value.isEmpty()) {
- if(element.equals("keywords"))
- document.put(element, StringUtils.join(values, ','));
- else
- document.put(element, value);
-// }
-
- }
- }
- }
- }
+ if(action == DocumentAction.ADD_UPDATE){
+ for(String element:xmlElements){
+ if (!element.equals(xmlElements.get(12))) {
+ Collection<Object> values = data_fields.get(element);
+ if (values!=null && values.size()>0) {
+ Object next = values.iterator()
+ .next();
+ String value =(String) next;
+ if (value != null && !value.isEmpty()) {
+ if(element.equals("keywords"))
+ document.put(element, StringUtils.join(values, ','));
+ else
+ document.put(element, value);
+
+ }
+ }
+ }
+ }
- // Metadata
- Collection<Object> metadataSet = data_fields
- .get(xmlElements.get(12));
- JSONObject metaObject = new JSONObject();
- if(metadataSet!=null && metadataSet.size()>0){
- Multimap<String, String> metadata = (Multimap<String, String>) metadataSet.iterator().next();
- if (metadata != null && !metadata.isEmpty()) {
- for (String name : metadata.keySet()){
- JSONArray nextMetadata = new JSONArray();
- for (String value : metadata.get(name)) {
- nextMetadata.put(value);
- }
- metaObject.put(name, nextMetadata);
- }
- }
- }
- document.put(xmlElements.get(12), metaObject);
- }
+ // Metadata
+ Collection<Object> metadataSet = data_fields
+ .get(xmlElements.get(12));
+ JSONObject metaObject = new JSONObject();
+ if(metadataSet!=null && metadataSet.size()>0){
+ Multimap<String, String> metadata = (Multimap<String, String>) metadataSet.iterator().next();
+ if (metadata != null && !metadata.isEmpty()) {
+ for (String name : metadata.keySet()){
+ JSONArray nextMetadata = new JSONArray();
+ for (String value : metadata.get(name)) {
+ nextMetadata.put(value);
+ }
+ metaObject.put(name, nextMetadata);
+ }
+ }
+ }
+ document.put(xmlElements.get(12), metaObject);
+ }
- result.put(xmlElements.get(1), document);
+ result.put(xmlElements.get(1), document);
- } catch (JSONException e) {
- throw new SearchBloxException("Error while building Document JSON object", e);
- }
- return result.toString();
- }
+ } catch (JSONException e) {
+ throw new SearchBloxException("Error while building Document JSON object", e);
+ }
+ return result.toString();
+ }
- private String toStringXML(DocumentAction action) throws SearchBloxException{
- Document doc = null;
- try {
- doc = DocumentBuilderFactory.newInstance().newDocumentBuilder()
- .newDocument();
+ private String toStringXML(DocumentAction action) throws SearchBloxException{
+ Document doc = null;
+ try {
+ doc = DocumentBuilderFactory.newInstance().newDocumentBuilder()
+ .newDocument();
- } catch (ParserConfigurationException e) {
- throw new SearchBloxException(e);
- }
+ } catch (ParserConfigurationException e) {
+ throw new SearchBloxException(e);
+ }
- // Document Base Data
- Element root = doc.createElement(xmlElements.get(0));
- if (apiKey == null)
- throw new SearchBloxException(
- "The API Key for accessing SearchBlox Server CAN'T be NULL");
- root.setAttribute(APIKEY_ATTRIBUTE, apiKey);
- doc.appendChild(root);
- Element document = doc.createElement(xmlElements.get(1));
- if (colName == null)
- throw new SearchBloxException(
- "The Collection Name of the SearchBlox Server CAN'T be NULL");
- document.setAttribute(COLNAME_ATTRIBUTE, colName);
- if(action == DocumentAction.DELETE)
- document.setAttribute(UID_ATTRIBUTE,uid);
- root.appendChild(document);
+ // Document Base Data
+ Element root = doc.createElement(xmlElements.get(0));
+ if (apiKey == null)
+ throw new SearchBloxException(
+ "The API Key for accessing SearchBlox Server CAN'T be NULL");
+ root.setAttribute(APIKEY_ATTRIBUTE, apiKey);
+ doc.appendChild(root);
+ Element document = doc.createElement(xmlElements.get(1));
+ if (colName == null)
+ throw new SearchBloxException(
+ "The Collection Name of the SearchBlox Server CAN'T be NULL");
+ document.setAttribute(COLNAME_ATTRIBUTE, colName);
+ if(action == DocumentAction.DELETE)
+ document.setAttribute(UID_ATTRIBUTE,uid);
+ root.appendChild(document);
- if (action == DocumentAction.ADD_UPDATE) {
- // Uid
- if (uid != null && !uid.isEmpty()) {
- Element uidElement = doc.createElement(xmlElements.get(13));
- uidElement.setTextContent(uid);
- document.appendChild(uidElement);
- }
+ if (action == DocumentAction.ADD_UPDATE) {
+ // Uid
+ if (uid != null && !uid.isEmpty()) {
+ Element uidElement = doc.createElement(xmlElements.get(13));
+ uidElement.setTextContent(uid);
+ document.appendChild(uidElement);
+ }
- for(String element:xmlElements){
- if (!element.equals(xmlElements.get(12))) {
- Collection<Object> values = data_fields.get(element);
- if (values!=null && values.size()>0) {
- Object next = values.iterator()
- .next();
- String value =(String) next;
- if (value != null && !value.isEmpty()) {
- Element eValue = doc.createElement(element);
- if(element.equals("keywords"))
- eValue.setTextContent(StringUtils.join(values, ','));
- else
- eValue.setTextContent(value);
- Collection<Object> boostElement = data_fields
- .get(element + "_boost");
- if(boostElement!=null && boostElement.size()>0){
- String value_boost = (String) boostElement.iterator()
- .next();
- eValue.setAttribute(BOOST_ATTRIBUTE, "" + value_boost);
- }
- document.appendChild(eValue);
- }
- }
- }
- }
+ for(String element:xmlElements){
+ if (!element.equals(xmlElements.get(12))) {
+ Collection<Object> values = data_fields.get(element);
+ if (values!=null && values.size()>0) {
+ Object next = values.iterator()
+ .next();
+ String value =(String) next;
+ if (value != null && !value.isEmpty()) {
+ Element eValue = doc.createElement(element);
+ if(element.equals("keywords"))
+ eValue.setTextContent(StringUtils.join(values, ','));
+ else
+ eValue.setTextContent(value);
+ Collection<Object> boostElement = data_fields
+ .get(element + "_boost");
+ if(boostElement!=null && boostElement.size()>0){
+ String value_boost = (String) boostElement.iterator()
+ .next();
+ eValue.setAttribute(BOOST_ATTRIBUTE, "" + value_boost);
+ }
+ document.appendChild(eValue);
+ }
+ }
+ }
+ }
- // Metadata
- Collection<Object> metadataSet = data_fields
- .get(xmlElements.get(12));
- if(metadataSet!=null && metadataSet.size()>0){
- Multimap<String, String> metadata = (Multimap<String, String>) metadataSet.iterator().next();
- if (metadata != null && !metadata.isEmpty()) {
- for (String name : metadata.keySet())
- for (String value : metadata.get(name)) {
- Element metaElement = doc.createElement(xmlElements.get(12));
- metaElement.setAttribute(NAME_ATTRIBUTE, name);
- metaElement.setTextContent(value);
- document.appendChild(metaElement);
- }
- } }
- }
+ // Metadata
+ Collection<Object> metadataSet = data_fields
+ .get(xmlElements.get(12));
+ if(metadataSet!=null && metadataSet.size()>0){
+ Multimap<String, String> metadata = (Multimap<String, String>) metadataSet.iterator().next();
+ if (metadata != null && !metadata.isEmpty()) {
+ for (String name : metadata.keySet())
+ for (String value : metadata.get(name)) {
+ Element metaElement = doc.createElement(xmlElements.get(12));
+ metaElement.setAttribute(NAME_ATTRIBUTE, name);
+ metaElement.setTextContent(value);
+ document.appendChild(metaElement);
+ }
+ }
+ }
+ }
- return getStringFromDocument(doc);
- }
+ return getStringFromDocument(doc);
+ }
- /**
- * <p>Transform a {@code Document} to its XML string representation</p>
- * @param doc the document to transform
- * @return the document in the XML-String format
- */
- private String getStringFromDocument(Document doc) {
- try {
- DOMSource domSource = new DOMSource(doc);
- StringWriter writer = new StringWriter();
- StreamResult result = new StreamResult(writer);
- TransformerFactory tf = TransformerFactory.newInstance();
- Transformer transformer = tf.newTransformer();
- // transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "no");
- transformer.transform(domSource, result);
- return writer.toString();
- } catch (TransformerException ex) {
- ex.printStackTrace();
- return null;
- }
+ /**
+ * <p>Transform a {@code Document} to its XML string representation</p>
+ * @param doc the document to transform
+ * @return the document in the XML-String format
+ */
+ private String getStringFromDocument(Document doc) {
+ try {
+ DOMSource domSource = new DOMSource(doc);
+ StringWriter writer = new StringWriter();
+ StreamResult result = new StreamResult(writer);
+ TransformerFactory tf = TransformerFactory.newInstance();
+ Transformer transformer = tf.newTransformer();
+ // transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "no");
+ transformer.transform(domSource, result);
+ return writer.toString();
+ } catch (TransformerException ex) {
+ ex.printStackTrace();
+ return null;
+ }
- }
+ }
}
diff --git a/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxException.java b/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxException.java
index d3d6c51..2888d8d 100644
--- a/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxException.java
+++ b/connectors/searchblox/connector/src/main/java/org/apache/manifoldcf/agents/output/searchblox/SearchBloxException.java
@@ -21,19 +21,19 @@
* @author Rafa Haro <rharo@apache.org>
*/
public class SearchBloxException
- extends Exception {
+ extends Exception {
- private static final long serialVersionUID = -6792055510634993398L;
+ private static final long serialVersionUID = -6792055510634993398L;
- public SearchBloxException(String reason, Throwable cause) {
- super(reason, cause);
- }
+ public SearchBloxException(String reason, Throwable cause) {
+ super(reason, cause);
+ }
- public SearchBloxException(String reason) {
- super(reason);
- }
+ public SearchBloxException(String reason) {
+ super(reason);
+ }
- public SearchBloxException(Throwable cause) {
- super(cause);
- }
+ public SearchBloxException(Throwable cause) {
+ super(cause);
+ }
}
diff --git a/connectors/searchblox/connector/src/main/resources/org/apache/manifoldcf/agents/output/searchblox/editConfiguration_Parameters.html b/connectors/searchblox/connector/src/main/resources/org/apache/manifoldcf/agents/output/searchblox/editConfiguration_Parameters.html
index ab2935b..dde5487 100644
--- a/connectors/searchblox/connector/src/main/resources/org/apache/manifoldcf/agents/output/searchblox/editConfiguration_Parameters.html
+++ b/connectors/searchblox/connector/src/main/resources/org/apache/manifoldcf/agents/output/searchblox/editConfiguration_Parameters.html
@@ -42,6 +42,19 @@
</select>
</td>
</tr>
+ <tr>
+ <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('SearchBloxConnector.TimeoutConnection'))</nobr></td>
+ <td class="value">
+ <input type="text" name="connectiontimeout" size="16" value="$Encoder.attributeEscape($CONNECTIONTIMEOUT)" />
+ </td>
+ </tr>
+ <tr>
+ <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('SearchBloxConnector.TimeoutSocket'))</nobr></td>
+ <td class="value">
+ <input type="text" name="sockettimeout" size="16" value="$Encoder.attributeEscape($SOCKETTIMEOUT)" />
+ </td>
+ </tr>
+
</table>
#else
@@ -49,6 +62,8 @@
<input type="hidden" name="apikey" value="$Encoder.attributeEscape($APIKEY)" />
<input type="hidden" name="endpoint" value="$Encoder.attributeEscape($ENDPOINT)" />
<input type="hidden" name="indexformat" value="$Encoder.attributeEscape($INDEXFORMAT)"/>
+<input type="hidden" name="connectiontimeout" value="$Encoder.attributeEscape($CONNECTIONTIMEOUT)"/>
+<input type="hidden" name="sockettimeout" value="$Encoder.attributeEscape($SOCKETTIMEOUT)"/>
diff --git a/connectors/searchblox/connector/src/main/resources/org/apache/manifoldcf/agents/output/searchblox/editSpecification_Configuration.html b/connectors/searchblox/connector/src/main/resources/org/apache/manifoldcf/agents/output/searchblox/editSpecification_Configuration.html
index 6d7e4d0..2feffb8 100644
--- a/connectors/searchblox/connector/src/main/resources/org/apache/manifoldcf/agents/output/searchblox/editSpecification_Configuration.html
+++ b/connectors/searchblox/connector/src/main/resources/org/apache/manifoldcf/agents/output/searchblox/editSpecification_Configuration.html
@@ -44,24 +44,6 @@
</td>
</tr>
<tr>
- <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('SearchBloxConnector.PoolSize'))</nobr></td>
- <td class="value">
- <input type="text" name="s${SEQNUM}_poolsize" value="$Encoder.attributeEscape($POOLSIZE)" />
- </td>
- </tr>
- <tr>
- <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('SearchBloxConnector.TimeoutConnection'))</nobr></td>
- <td class="value">
- <input type="text" name="s${SEQNUM}_timeoutconnection" value="$Encoder.attributeEscape($TIMEOUTCONNECTION)" />
- </td>
- </tr>
- <tr>
- <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('SearchBloxConnector.TimeoutSocket'))</nobr></td>
- <td class="value">
- <input type="text" name="s${SEQNUM}_timeoutsocket" value="$Encoder.attributeEscape($TIMEOUTSOCKET)" />
- </td>
- </tr>
- <tr>
<td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('SearchBloxConnector.Collection'))</nobr></td>
<td class="value">
<input type="text" name="s${SEQNUM}_collection" value="$Encoder.attributeEscape($COLLECTION)" />
@@ -76,9 +58,6 @@
<input type="hidden" name="s${SEQNUM}_content_boost" value="${CONTENT_BOOST}"/>
<input type="hidden" name="s${SEQNUM}_description_boost" value="${DESCRIPTION_BOOST}"/>
<input type="hidden" name="s${SEQNUM}_keywords_boost" value="${KEYWORDS_BOOST}"/>
-<input type="hidden" name="s${SEQNUM}_poolsize" value="${POOLSIZE}"/>
-<input type="hidden" name="s${SEQNUM}_timeoutconnection" value="${TIMEOUTCONNECTION}"/>
-<input type="hidden" name="s${SEQNUM}_timeoutsocket" value="${TIMEOUTSOCKET}"/>
<input type="hidden" name="s${SEQNUM}_collection" value="${COLLECTION}"/>
#end
\ No newline at end of file
diff --git a/connectors/searchblox/connector/src/main/resources/org/apache/manifoldcf/agents/output/searchblox/viewConfiguration.html b/connectors/searchblox/connector/src/main/resources/org/apache/manifoldcf/agents/output/searchblox/viewConfiguration.html
index 3bac075..5cd9a90 100644
--- a/connectors/searchblox/connector/src/main/resources/org/apache/manifoldcf/agents/output/searchblox/viewConfiguration.html
+++ b/connectors/searchblox/connector/src/main/resources/org/apache/manifoldcf/agents/output/searchblox/viewConfiguration.html
@@ -28,4 +28,13 @@
<td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('SearchBloxConnector.IndexFormat'))</nobr></td>
<td class="value">$Encoder.bodyEscape($INDEXFORMAT)</td>
</tr>
+ <tr><td class="separator" colspan="2"><hr/></td></tr>
+ <tr>
+ <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('SearchBloxConnector.TimeoutConnection'))</nobr></td>
+ <td class="value"><nobr>$Encoder.bodyEscape($CONNECTIONTIMEOUT)</nobr></td>
+ </tr>
+ <tr>
+ <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('SearchBloxConnector.TimeoutSocket'))</nobr></td>
+ <td class="value"><nobr>$Encoder.bodyEscape($SOCKETTIMEOUT)</nobr></td>
+ </tr>
</table>
\ No newline at end of file
diff --git a/connectors/searchblox/connector/src/main/resources/org/apache/manifoldcf/agents/output/searchblox/viewSpecification.html b/connectors/searchblox/connector/src/main/resources/org/apache/manifoldcf/agents/output/searchblox/viewSpecification.html
index 79ddd7d..35764dc 100644
--- a/connectors/searchblox/connector/src/main/resources/org/apache/manifoldcf/agents/output/searchblox/viewSpecification.html
+++ b/connectors/searchblox/connector/src/main/resources/org/apache/manifoldcf/agents/output/searchblox/viewSpecification.html
@@ -38,21 +38,6 @@
</tr>
<tr><td class="separator" colspan="2"><hr/></td></tr>
<tr>
- <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('SearchBloxConnector.PoolSize'))</nobr></td>
- <td class="value"><nobr>$Encoder.bodyEscape($POOLSIZE)</nobr></td>
- </tr>
- <tr><td class="separator" colspan="2"><hr/></td></tr>
- <tr>
- <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('SearchBloxConnector.TimeoutConnection'))</nobr></td>
- <td class="value"><nobr>$Encoder.bodyEscape($TIMEOUTCONNECTION)</nobr></td>
- </tr>
- <tr><td class="separator" colspan="2"><hr/></td></tr>
- <tr>
- <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('SearchBloxConnector.TimeoutSocket'))</nobr></td>
- <td class="value"><nobr>$Encoder.bodyEscape($TIMEOUTSOCKET)</nobr></td>
- </tr>
- <tr><td class="separator" colspan="2"><hr/></td></tr>
- <tr>
<td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('SearchBloxConnector.Collection'))</nobr></td>
<td class="value"><nobr>$Encoder.bodyEscape($COLLECTION)</nobr></td>
</tr>
diff --git a/connectors/searchblox/connector/src/test/java/org/apache/manifoldcf/agents/output/searchblox/tests/SearchBloxDocumentTest.java b/connectors/searchblox/connector/src/test/java/org/apache/manifoldcf/agents/output/searchblox/tests/SearchBloxDocumentTest.java
index c27314b..6966fbe 100644
--- a/connectors/searchblox/connector/src/test/java/org/apache/manifoldcf/agents/output/searchblox/tests/SearchBloxDocumentTest.java
+++ b/connectors/searchblox/connector/src/test/java/org/apache/manifoldcf/agents/output/searchblox/tests/SearchBloxDocumentTest.java
@@ -1,8 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.manifoldcf.agents.output.searchblox.tests;
import com.google.common.collect.Lists;
-import junit.framework.TestCase;
+//import junit.framework.TestCase;
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
@@ -11,6 +27,8 @@
import org.apache.manifoldcf.agents.output.searchblox.SearchBloxDocument.IndexingFormat;
import org.apache.manifoldcf.agents.output.searchblox.SearchBloxException;
import org.junit.Test;
+import org.junit.Ignore;
+import static org.junit.Assert.*;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
@@ -25,7 +43,7 @@
* 07/03/2015
* mcf-searchblox-connector
*/
-public class SearchBloxDocumentTest extends TestCase{
+public class SearchBloxDocumentTest /*extends TestCase */ {
SearchBloxDocument toTest;
RepositoryDocument rd;
@@ -38,7 +56,8 @@
}
@Test
- public void testUpdateXmlString() throws SearchBloxException {
+ @Ignore("fails on jdk 8 due to hash order")
+ public void updateXmlString() throws SearchBloxException {
String xmlGenerated=toTest.toString(IndexingFormat.XML, DocumentAction.ADD_UPDATE);
System.out.println(xmlGenerated);
String xmlExpected="<?xml version=\"1.0\" encoding=\"UTF-8\"?>" +
@@ -55,14 +74,16 @@
}
@Test
- public void testUpdateJsonString() throws SearchBloxException {
+ @Ignore("fails on jdk 8 due to hash order")
+ public void updateJsonString() throws SearchBloxException {
String jsonGenerated=toTest.toString(IndexingFormat.JSON, DocumentAction.ADD_UPDATE);
String expectedJson="{\"document\":{\"content\":\"I am a nice content in english!\",\"uid\":\"URI\",\"title\":\"I am a nice title\",\"description\":\"I am a little tiny description\",\"contenttype\":\"html\",\"colname\":\"collection1\",\"meta\":{\"meta2\":[\"I am META2!\"],\"meta1\":[\"I am META1!\"],\"share_allow\":[\"user3\",\"user2\",\"user1\"],\"share_deny\":[\"user4\",\"user5\"],\"document_deny\":[\"user52\",\"user42\"],\"document_allow\":[\"user22\",\"user12\",\"user33\"]},\"size\":\"100\"},\"apikey\":\"apikey\"}";
assertEquals(expectedJson,jsonGenerated);
}
@Test
- public void testDeleteJsonString() throws SearchBloxException {
+ @Ignore("fails on jdk 8 due to hash order")
+ public void deleteJsonString() throws SearchBloxException {
String jsonGenerated=toTest.toString(IndexingFormat.JSON, DocumentAction.DELETE);
System.out.println(jsonGenerated);
String xmlExpected="{\"document\":{\"uid\":\"URI\",\"colname\":\"collection1\"},\"apikey\":\"apikey\"}";
@@ -70,7 +91,8 @@
}
@Test
- public void testDeleteXmlString() throws SearchBloxException {
+ @Ignore("fails on jdk 8 due to hash order")
+ public void deleteXmlString() throws SearchBloxException {
String xmlGenerated=toTest.toString(IndexingFormat.XML, DocumentAction.DELETE);
System.out.println(xmlGenerated);
String xmlExpected="<?xml version=\"1.0\" encoding=\"UTF-8\"?><searchblox apikey=\"apikey\"><document colname=\"collection1\" uid=\"URI\"/></searchblox>";
diff --git a/connectors/searchblox/pom.xml b/connectors/searchblox/pom.xml
index fc0ab3e..56444de 100644
--- a/connectors/searchblox/pom.xml
+++ b/connectors/searchblox/pom.xml
@@ -6,7 +6,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<artifactId>mcf-searchblox-connector</artifactId>
diff --git a/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java b/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java
index 274f857..51f176b 100644
--- a/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java
+++ b/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java
@@ -45,8 +45,11 @@
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.config.SocketConfig;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.DefaultHostnameVerifier;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
@@ -792,16 +795,30 @@
// Set up ssl if indicated
- connectionManager = new PoolingHttpClientConnectionManager();
-
- CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-
SSLConnectionSocketFactory myFactory = null;
if (keystoreData != null)
{
keystoreManager = KeystoreManagerFactory.make("",keystoreData);
- myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), new BrowserCompatHostnameVerifier());
+ myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), new DefaultHostnameVerifier());
}
+ else
+ {
+ myFactory = SSLConnectionSocketFactory.getSocketFactory();
+ }
+
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory>create()
+ .register("http", PlainConnectionSocketFactory.getSocketFactory())
+ .register("https", myFactory)
+ .build());
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ connectionManager = poolingConnectionManager;
+
+ CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
if (strippedUserName != null)
{
@@ -813,7 +830,6 @@
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(false)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
@@ -842,16 +858,9 @@
HttpClientBuilder builder = HttpClients.custom()
.setConnectionManager(connectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(credentialsProvider);
- if (myFactory != null)
- builder.setSSLSocketFactory(myFactory);
builder.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
.setRedirectStrategy(new DefaultRedirectStrategy());
httpClient = builder.build();
diff --git a/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java b/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
index 9859a78..87db622 100644
--- a/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
+++ b/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
@@ -55,8 +55,11 @@
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.config.SocketConfig;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.DefaultHostnameVerifier;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
@@ -255,16 +258,30 @@
int connectionTimeout = 60000;
int socketTimeout = 900000;
- connectionManager = new PoolingHttpClientConnectionManager();
-
- CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-
SSLConnectionSocketFactory myFactory = null;
if (keystoreData != null)
{
keystoreManager = KeystoreManagerFactory.make("",keystoreData);
- myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), new BrowserCompatHostnameVerifier());
+ myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), new DefaultHostnameVerifier());
}
+ else
+ {
+ myFactory = SSLConnectionSocketFactory.getSocketFactory();
+ }
+
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory>create()
+ .register("http", PlainConnectionSocketFactory.getSocketFactory())
+ .register("https", myFactory)
+ .build());
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ connectionManager = poolingConnectionManager;
+
+ CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
if (strippedUserName != null)
{
@@ -276,7 +293,6 @@
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(false)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
@@ -305,16 +321,9 @@
HttpClientBuilder builder = HttpClients.custom()
.setConnectionManager(connectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(credentialsProvider);
- if (myFactory != null)
- builder.setSSLSocketFactory(myFactory);
builder.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
.setRedirectStrategy(new DefaultRedirectStrategy());
httpClient = builder.build();
diff --git a/connectors/sharepoint/pom.xml b/connectors/sharepoint/pom.xml
index 0037b32..d2f731e 100644
--- a/connectors/sharepoint/pom.xml
+++ b/connectors/sharepoint/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java b/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
index 4494a0e..d7c806e 100644
--- a/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
+++ b/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
@@ -22,7 +22,11 @@
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.config.SocketConfig;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.impl.client.*;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.protocol.HttpRequestExecutor;
@@ -50,9 +54,11 @@
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.impl.XMLResponseParser;
import org.apache.solr.common.util.ContentStreamBase;
+import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.response.SolrPingResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.SolrException;
@@ -210,42 +216,43 @@
}
// Initialize standard solr-j.
- // First, we need an HttpClient where basic auth is properly set up.
- connectionManager = new PoolingHttpClientConnectionManager();
- connectionManager.setMaxTotal(1);
-
+
SSLConnectionSocketFactory myFactory;
if (keystoreManager != null)
{
- myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), NoopHostnameVerifier.INSTANCE);
}
else
{
// Use the "trust everything" one
- myFactory = new SSLConnectionSocketFactory(KeystoreManagerFactory.getTrustingSecureSocketFactory(),SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ myFactory = new SSLConnectionSocketFactory(KeystoreManagerFactory.getTrustingSecureSocketFactory(),NoopHostnameVerifier.INSTANCE);
}
+ // First, we need an HttpClient where basic auth is properly set up.
+ connectionManager = new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory>create()
+ .register("http", PlainConnectionSocketFactory.getSocketFactory())
+ .register("https", myFactory)
+ .build());
+ connectionManager.setDefaultMaxPerRoute(1);
+ connectionManager.setValidateAfterInactivity(60000);
+ connectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
- HttpClientBuilder clientBuilder = HttpClients.custom()
- .setConnectionManager(connectionManager)
- .setMaxConnTotal(1)
- .disableAutomaticRetries()
- .setDefaultRequestConfig(requestBuilder.build())
- .setRedirectStrategy(new DefaultRedirectStrategy())
- .setSSLSocketFactory(myFactory)
- .setRequestExecutor(new HttpRequestExecutor(socketTimeout))
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build()
- );
+ HttpClientBuilder clientBuilder = HttpClients.custom()
+ .setConnectionManager(connectionManager)
+ .disableAutomaticRetries()
+ .setDefaultRequestConfig(requestBuilder.build())
+ .setRedirectStrategy(new DefaultRedirectStrategy())
+ .setRequestExecutor(new HttpRequestExecutor(socketTimeout));
if (userID != null && userID.length() > 0 && password != null)
@@ -1436,7 +1443,7 @@
// Do the operation!
try
{
- SolrPingResponse response = new SolrPing(postStatusAction).process(solrServer);
+ SolrResponse response = new SolrPing(postStatusAction).process(solrServer);
}
catch (InterruptedIOException ioe)
{
@@ -1531,14 +1538,17 @@
*/
protected static class SolrPing extends SolrRequest
{
+ /** Request parameters. */
private ModifiableSolrParams params;
- public SolrPing()
- {
- super( METHOD.GET, "/admin/ping" );
+ /**
+ * Create a new SolrPing object.
+ */
+ public SolrPing() {
+ super(METHOD.GET, "/admin/ping");
params = new ModifiableSolrParams();
}
-
+
public SolrPing(String url)
{
super( METHOD.GET, url );
@@ -1551,19 +1561,60 @@
}
@Override
- public ModifiableSolrParams getParams() {
- return params;
+ protected SolrPingResponse createResponse(SolrClient client) {
+ return new SolrPingResponse();
}
@Override
- public SolrPingResponse process( SolrClient server ) throws SolrServerException, IOException
- {
- long startTime = System.currentTimeMillis();
- SolrPingResponse res = new SolrPingResponse();
- res.setResponse( server.request( this ) );
- res.setElapsedTime( System.currentTimeMillis()-startTime );
- return res;
+ public ModifiableSolrParams getParams() {
+ return params;
}
+
+ /**
+ * Remove the action parameter from this request. This will result in the same
+ * behavior as {@code SolrPing#setActionPing()}. For Solr server version 4.0
+ * and later.
+ *
+ * @return this
+ */
+ public SolrPing removeAction() {
+ params.remove(CommonParams.ACTION);
+ return this;
+ }
+
+ /**
+ * Set the action parameter on this request to enable. This will delete the
+ * health-check file for the Solr core. For Solr server version 4.0 and later.
+ *
+ * @return this
+ */
+ public SolrPing setActionDisable() {
+ params.set(CommonParams.ACTION, CommonParams.DISABLE);
+ return this;
+ }
+
+ /**
+ * Set the action parameter on this request to enable. This will create the
+ * health-check file for the Solr core. For Solr server version 4.0 and later.
+ *
+ * @return this
+ */
+ public SolrPing setActionEnable() {
+ params.set(CommonParams.ACTION, CommonParams.ENABLE);
+ return this;
+ }
+
+ /**
+ * Set the action parameter on this request to ping. This is the same as not
+ * including the action at all. For Solr server version 4.0 and later.
+ *
+ * @return this
+ */
+ public SolrPing setActionPing() {
+ params.set(CommonParams.ACTION, CommonParams.PING);
+ return this;
+ }
+
}
/** See CONNECTORS-956. Make a safe lucene field name from a possibly
diff --git a/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java b/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java
index 1086ea3..abdd7b0 100644
--- a/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java
+++ b/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java
@@ -95,7 +95,6 @@
private final HttpClient httpClient;
- private int maxRetries = 0;
private boolean useMultiPartPost = true;
public ModifiedHttpSolrClient(String baseURL, HttpClient client, ResponseParser parser) {
@@ -104,9 +103,8 @@
}
@Override
- protected HttpRequestBase createMethod(final SolrRequest request) throws IOException, SolrServerException {
- HttpRequestBase method = null;
- InputStream is = null;
+ protected HttpRequestBase createMethod(final SolrRequest request, String collection) throws IOException, SolrServerException {
+
SolrParams params = request.getParams();
Collection<ContentStream> streams = requestWriter.getContentStreams(request);
String path = requestWriter.getPath(request);
@@ -129,165 +127,139 @@
if (invariantParams != null) {
wparams.add(invariantParams);
}
-
- int tries = maxRetries + 1;
- try {
- while( tries-- > 0 ) {
- // Note: since we aren't do intermittent time keeping
- // ourselves, the potential non-timeout latency could be as
- // much as tries-times (plus scheduling effects) the given
- // timeAllowed.
- try {
- if( SolrRequest.METHOD.GET == request.getMethod() ) {
- if( streams != null ) {
- throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!" );
- }
- method = new HttpGet( baseUrl + path + toQueryString( wparams, false ) );
- }
- else if( SolrRequest.METHOD.POST == request.getMethod() || SolrRequest.METHOD.PUT == request.getMethod() ) {
- String url = baseUrl + path;
- boolean hasNullStreamName = false;
- if (streams != null) {
- for (ContentStream cs : streams) {
- if (cs.getName() == null) {
- hasNullStreamName = true;
- break;
- }
- }
- }
- boolean isMultipart = ((this.useMultiPartPost && SolrRequest.METHOD.POST == request.getMethod())
- || ( streams != null && streams.size() > 1 )) && !hasNullStreamName;
-
- LinkedList<NameValuePair> postOrPutParams = new LinkedList<>();
- if (streams == null || isMultipart) {
- // send server list and request list as query string params
- ModifiableSolrParams queryParams = calculateQueryParams(getQueryParams(), wparams);
- queryParams.add(calculateQueryParams(request.getQueryParams(), wparams));
- String fullQueryUrl = url + toQueryString( queryParams, false );
- HttpEntityEnclosingRequestBase postOrPut = SolrRequest.METHOD.POST == request.getMethod() ?
- new HttpPost(fullQueryUrl) : new HttpPut(fullQueryUrl);
- if (!isMultipart) {
- postOrPut.addHeader("Content-Type",
- "application/x-www-form-urlencoded; charset=UTF-8");
- }
+ String basePath = baseUrl;
+ if (collection != null)
+ basePath += "/" + collection;
- List<FormBodyPart> parts = new LinkedList<>();
- Iterator<String> iter = wparams.getParameterNamesIterator();
- while (iter.hasNext()) {
- String p = iter.next();
- String[] vals = wparams.getParams(p);
- if (vals != null) {
- for (String v : vals) {
- if (isMultipart) {
- parts.add(new FormBodyPart(p, new StringBody(v, StandardCharsets.UTF_8)));
- } else {
- postOrPutParams.add(new BasicNameValuePair(p, v));
- }
- }
- }
- }
+ if (SolrRequest.METHOD.GET == request.getMethod()) {
+ if (streams != null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!");
+ }
+ return new HttpGet(basePath + path + toQueryString(wparams, false));
+ }
- if (isMultipart && streams != null) {
- for (ContentStream content : streams) {
- String contentType = content.getContentType();
- if(contentType==null) {
- contentType = BinaryResponseParser.BINARY_CONTENT_TYPE; // default
- }
- String name = content.getName();
- if(name==null) {
- name = "";
- }
- parts.add(new FormBodyPart(name,
- new InputStreamBody(
- content.getStream(),
- contentType,
- content.getName())));
- }
- }
-
- if (parts.size() > 0) {
- ModifiedMultipartEntity entity = new ModifiedMultipartEntity(HttpMultipartMode.STRICT, null, StandardCharsets.UTF_8);
- //MultipartEntity entity = new MultipartEntity(HttpMultipartMode.STRICT);
- for(FormBodyPart p: parts) {
- entity.addPart(p);
- }
- postOrPut.setEntity(entity);
- } else {
- //not using multipart
- postOrPut.setEntity(new UrlEncodedFormEntity(postOrPutParams, StandardCharsets.UTF_8));
- }
+ if (SolrRequest.METHOD.POST == request.getMethod() || SolrRequest.METHOD.PUT == request.getMethod()) {
- method = postOrPut;
- }
- // It is has one stream, it is the post body, put the params in the URL
- else {
- String pstr = toQueryString(wparams, false);
- HttpEntityEnclosingRequestBase postOrPut = SolrRequest.METHOD.POST == request.getMethod() ?
- new HttpPost(url + pstr) : new HttpPut(url + pstr);
-
- // Single stream as body
- // Using a loop just to get the first one
- final ContentStream[] contentStream = new ContentStream[1];
- for (ContentStream content : streams) {
- contentStream[0] = content;
- break;
- }
- if (contentStream[0] instanceof RequestWriter.LazyContentStream) {
- postOrPut.setEntity(new InputStreamEntity(contentStream[0].getStream(), -1) {
- @Override
- public Header getContentType() {
- return new BasicHeader("Content-Type", contentStream[0].getContentType());
- }
-
- @Override
- public boolean isRepeatable() {
- return false;
- }
-
- });
- } else {
- postOrPut.setEntity(new InputStreamEntity(contentStream[0].getStream(), -1) {
- @Override
- public Header getContentType() {
- return new BasicHeader("Content-Type", contentStream[0].getContentType());
- }
-
- @Override
- public boolean isRepeatable() {
- return false;
- }
- });
- }
- method = postOrPut;
- }
- }
- else {
- throw new SolrServerException("Unsupported method: "+request.getMethod() );
- }
- }
- catch( NoHttpResponseException r ) {
- method = null;
- if(is != null) {
- is.close();
- }
- // If out of tries then just rethrow (as normal error).
- if (tries < 1) {
- throw r;
+ String url = basePath + path;
+ boolean hasNullStreamName = false;
+ if (streams != null) {
+ for (ContentStream cs : streams) {
+ if (cs.getName() == null) {
+ hasNullStreamName = true;
+ break;
}
}
}
- } catch (IOException ex) {
- throw new SolrServerException("error reading streams", ex);
- }
-
- return method;
- }
+ boolean isMultipart = ((this.useMultiPartPost && SolrRequest.METHOD.POST == request.getMethod())
+ || (streams != null && streams.size() > 1)) && !hasNullStreamName;
- @Override
- public void setMaxRetries(int maxRetries) {
- super.setMaxRetries(maxRetries);
- this.maxRetries = maxRetries;
+ LinkedList<NameValuePair> postOrPutParams = new LinkedList<>();
+ if (streams == null || isMultipart) {
+ // send server list and request list as query string params
+ ModifiableSolrParams queryParams = calculateQueryParams(getQueryParams(), wparams);
+ queryParams.add(calculateQueryParams(request.getQueryParams(), wparams));
+ String fullQueryUrl = url + toQueryString(queryParams, false);
+ HttpEntityEnclosingRequestBase postOrPut = SolrRequest.METHOD.POST == request.getMethod() ?
+ new HttpPost(fullQueryUrl) : new HttpPut(fullQueryUrl);
+ if (!isMultipart) {
+ postOrPut.addHeader("Content-Type",
+ "application/x-www-form-urlencoded; charset=UTF-8");
+ }
+
+ List<FormBodyPart> parts = new LinkedList<>();
+ Iterator<String> iter = wparams.getParameterNamesIterator();
+ while (iter.hasNext()) {
+ String p = iter.next();
+ String[] vals = wparams.getParams(p);
+ if (vals != null) {
+ for (String v : vals) {
+ if (isMultipart) {
+ parts.add(new FormBodyPart(p, new StringBody(v, StandardCharsets.UTF_8)));
+ } else {
+ postOrPutParams.add(new BasicNameValuePair(p, v));
+ }
+ }
+ }
+ }
+
+ if (isMultipart && streams != null) {
+ for (ContentStream content : streams) {
+ String contentType = content.getContentType();
+ if (contentType == null) {
+ contentType = BinaryResponseParser.BINARY_CONTENT_TYPE; // default
+ }
+ String name = content.getName();
+ if (name == null) {
+ name = "";
+ }
+ parts.add(new FormBodyPart(name,
+ new InputStreamBody(
+ content.getStream(),
+ contentType,
+ content.getName())));
+ }
+ }
+
+ if (parts.size() > 0) {
+ ModifiedMultipartEntity entity = new ModifiedMultipartEntity(HttpMultipartMode.STRICT, null, StandardCharsets.UTF_8);
+ //MultipartEntity entity = new MultipartEntity(HttpMultipartMode.STRICT);
+ for (FormBodyPart p : parts) {
+ entity.addPart(p);
+ }
+ postOrPut.setEntity(entity);
+ } else {
+ //not using multipart
+ postOrPut.setEntity(new UrlEncodedFormEntity(postOrPutParams, StandardCharsets.UTF_8));
+ }
+
+ return postOrPut;
+ }
+ // It is has one stream, it is the post body, put the params in the URL
+ else {
+ String pstr = toQueryString(wparams, false);
+ HttpEntityEnclosingRequestBase postOrPut = SolrRequest.METHOD.POST == request.getMethod() ?
+ new HttpPost(url + pstr) : new HttpPut(url + pstr);
+
+ // Single stream as body
+ // Using a loop just to get the first one
+ final ContentStream[] contentStream = new ContentStream[1];
+ for (ContentStream content : streams) {
+ contentStream[0] = content;
+ break;
+ }
+ if (contentStream[0] instanceof RequestWriter.LazyContentStream) {
+ postOrPut.setEntity(new InputStreamEntity(contentStream[0].getStream(), -1) {
+ @Override
+ public Header getContentType() {
+ return new BasicHeader("Content-Type", contentStream[0].getContentType());
+ }
+
+ @Override
+ public boolean isRepeatable() {
+ return false;
+ }
+
+ });
+ } else {
+ postOrPut.setEntity(new InputStreamEntity(contentStream[0].getStream(), -1) {
+ @Override
+ public Header getContentType() {
+ return new BasicHeader("Content-Type", contentStream[0].getContentType());
+ }
+
+ @Override
+ public boolean isRepeatable() {
+ return false;
+ }
+ });
+ }
+ return postOrPut;
+ }
+ }
+
+ throw new SolrServerException("Unsupported method: " + request.getMethod());
+
}
public static String toQueryString( SolrParams params, boolean xml ) {
diff --git a/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java b/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
index 2ab8059..9b55da9 100644
--- a/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
+++ b/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
@@ -96,6 +96,9 @@
/** Whether or not to commit */
protected boolean doCommits = false;
+
+ /** Collection name (non-empty only if SolrCloud) */
+ protected String collectionName = null;
/** Idle connection expiration interval */
protected final static long EXPIRATION_INTERVAL = 300000L;
@@ -185,6 +188,7 @@
mimeTypeAttributeName = null;
contentAttributeName = null;
useExtractUpdateHandler = true;
+ collectionName = null;
super.disconnect();
}
@@ -340,6 +344,8 @@
if (connectTimeoutString == null)
connectTimeoutString = "60";
+ collectionName = null;
+
try
{
int socketTimeout = Integer.parseInt(socketTimeoutString) * 1000;
@@ -385,6 +391,7 @@
String collection = params.getParameter(SolrConfig.PARAM_COLLECTION);
if (collection == null)
collection = "collection1";
+ collectionName = collection;
// Pick up timeouts
String zkClientTimeoutString = params.getParameter(SolrConfig.PARAM_ZOOKEEPER_CLIENT_TIMEOUT);
@@ -2326,6 +2333,14 @@
else
sb.append('-');
+ if (collectionName != null)
+ {
+ sb.append('+');
+ pack(sb,collectionName,'+');
+ }
+ else
+ sb.append('-');
+
return sb.toString();
}
diff --git a/connectors/solr/pom.xml b/connectors/solr/pom.xml
index 5ddac11..f6c6d90 100644
--- a/connectors/solr/pom.xml
+++ b/connectors/solr/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -205,6 +205,21 @@
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
<version>${solr.version}</version>
+ <exclusions>
+ <!-- Solrj pulls in an older httpclient -->
+ <exclusion>
+ <artifactId>httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>httpcore</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>httpmime</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.noggit</groupId>
@@ -222,6 +237,16 @@
<version>${httpcomponent.httpclient.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>${httpcomponent.httpcore.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpmime</artifactId>
+ <version>${httpcomponent.httpmime.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
diff --git a/connectors/tika/pom.xml b/connectors/tika/pom.xml
index dc98b88..55ed918 100644
--- a/connectors/tika/pom.xml
+++ b/connectors/tika/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/FindContentHandler.java b/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/FindContentHandler.java
index a115d66..9dfc763 100644
--- a/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/FindContentHandler.java
+++ b/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/FindContentHandler.java
@@ -29,6 +29,9 @@
protected final Pattern contentPattern;
protected final StringBuilder contentBuffer = new StringBuilder();
+ protected final static int MAX_LENGTH = 65536;
+ protected final static int OVERLAP_AMOUNT = 16384;
+
public FindContentHandler(String parentURI, Pattern contentPattern)
{
super(parentURI);
@@ -57,13 +60,26 @@
return;
// Build characters up into lines, and apply the regexp against them
if (textCharacter == '\t' || textCharacter >= ' ')
+ {
contentBuffer.append(textCharacter);
+ // If too big, do the search and clear out the buffer, retaining some of it for overlap purposes
+ if (contentBuffer.length() >= MAX_LENGTH)
+ {
+ // Process what we have, and keep around what we need for
+ // continuity
+ String bufferContents = contentBuffer.toString();
+ contentBuffer.setLength(0);
+ if (contentPattern.matcher(bufferContents).find())
+ targetURI = "";
+ else
+ {
+ contentBuffer.append(bufferContents.substring(bufferContents.length() - OVERLAP_AMOUNT));
+ }
+ }
+ }
else
{
- String bufferContents = contentBuffer.toString();
- contentBuffer.setLength(0);
- if (contentPattern.matcher(bufferContents).find())
- targetURI = "";
+ processBuffer();
}
}
@@ -123,5 +139,22 @@
{
}
+ /** Finish up all processing. Called ONLY if we haven't already aborted.
+ */
+ @Override
+ public void finishUp()
+ throws ManifoldCFException
+ {
+ if (targetURI == null)
+ processBuffer();
+ }
+
+ protected void processBuffer()
+ {
+ String bufferContents = contentBuffer.toString();
+ contentBuffer.setLength(0);
+ if (contentPattern.matcher(bufferContents).find())
+ targetURI = "";
+ }
}
diff --git a/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/FindHTMLFormHandler.java b/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/FindHTMLFormHandler.java
index 5ab8be2..b7c816f 100644
--- a/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/FindHTMLFormHandler.java
+++ b/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/FindHTMLFormHandler.java
@@ -188,4 +188,10 @@
{
}
+ @Override
+ public void finishUp()
+ throws ManifoldCFException
+ {
+ }
+
}
diff --git a/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/FindHTMLHrefHandler.java b/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/FindHTMLHrefHandler.java
index 4b10267..b0c47d7 100644
--- a/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/FindHTMLHrefHandler.java
+++ b/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/FindHTMLHrefHandler.java
@@ -144,4 +144,10 @@
noteDiscoveredLink(rawURL);
}
+ @Override
+ public void finishUp()
+ throws ManifoldCFException
+ {
+ }
+
}
diff --git a/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IHTMLHandler.java b/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IHTMLHandler.java
index bd19925..273383d 100644
--- a/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IHTMLHandler.java
+++ b/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IHTMLHandler.java
@@ -59,4 +59,9 @@
public void noteTextCharacter(char textCharacter)
throws ManifoldCFException;
+ /** Done with the document.
+ */
+ public void finishUp()
+ throws ManifoldCFException;
+
}
diff --git a/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java b/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java
index 2e834c6..c24b11a 100644
--- a/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java
+++ b/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java
@@ -66,14 +66,13 @@
* @param urlPath is the path part of the url, e.g. "/robots.txt"
* @param userAgent is the value of the userAgent header to use.
* @param from is the value of the from header to use.
- * @param connectionTimeoutMilliseconds is the maximum number of milliseconds to wait on socket connect.
* @param redirectOK should be set to true if you want redirects to be automatically followed.
* @param host is the value to use as the "Host" header, or null to use the default.
* @param formData describes additional form arguments and how to fetch the page.
* @param loginCookies describes the cookies that should be in effect for this page fetch.
*/
- public void executeFetch(String urlPath, String userAgent, String from, int connectionTimeoutMilliseconds,
- int socketTimeoutMilliseconds, boolean redirectOK, String host, FormData formData,
+ public void executeFetch(String urlPath, String userAgent, String from,
+ boolean redirectOK, String host, FormData formData,
LoginCookies loginCookies)
throws ManifoldCFException, ServiceInterruption;
diff --git a/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/LinkParseState.java b/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/LinkParseState.java
index 8005d60..1b87561 100644
--- a/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/LinkParseState.java
+++ b/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/LinkParseState.java
@@ -67,4 +67,12 @@
return false;
}
+ @Override
+ public void finishUp()
+ throws ManifoldCFException
+ {
+ handler.finishUp();
+ super.finishUp();
+ }
+
}
diff --git a/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java b/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
index d788c37..167e522 100644
--- a/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
+++ b/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
@@ -48,8 +48,11 @@
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.NameValuePair;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
@@ -69,9 +72,16 @@
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.cookie.CookieOrigin;
import org.apache.http.cookie.ClientCookie;
+import org.apache.http.cookie.CookieSpecProvider;
import org.apache.http.cookie.Cookie;
+import org.apache.http.impl.cookie.DefaultCookieSpec;
+import org.apache.http.impl.cookie.LaxBrowserCompatSpec;
+import org.apache.http.impl.cookie.RFC6265CookieSpecProvider;
import org.apache.http.impl.cookie.BasicPathHandler;
-import org.apache.http.impl.cookie.BrowserCompatSpec;
+import org.apache.http.impl.cookie.LaxMaxAgeHandler;
+import org.apache.http.impl.cookie.BasicDomainHandler;
+import org.apache.http.impl.cookie.BasicSecureHandler;
+import org.apache.http.impl.cookie.LaxExpiresHandler;
import org.apache.http.cookie.CookieSpec;
import org.apache.http.client.CookieStore;
import org.apache.http.protocol.HttpContext;
@@ -79,15 +89,9 @@
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.cookie.CookieIdentityComparator;
import org.apache.http.client.HttpRequestRetryHandler;
-import org.apache.http.cookie.CookieSpecProvider;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.config.Registry;
import org.apache.http.client.config.CookieSpecs;
-import org.apache.http.impl.cookie.BestMatchSpecFactory;
-import org.apache.http.impl.cookie.BrowserCompatSpecFactory;
-import org.apache.http.impl.cookie.RFC2965SpecFactory;
-import org.apache.http.impl.cookie.NetscapeDraftSpecFactory;
-import org.apache.http.impl.cookie.IgnoreSpecFactory;
import org.apache.http.cookie.MalformedCookieException;
import org.apache.http.conn.ConnectTimeoutException;
@@ -147,11 +151,7 @@
private static final Registry<CookieSpecProvider> cookieSpecRegistry =
RegistryBuilder.<CookieSpecProvider>create()
- .register(CookieSpecs.BEST_MATCH, new BestMatchSpecFactory())
- .register(CookieSpecs.STANDARD, new RFC2965SpecFactory())
- .register(CookieSpecs.BROWSER_COMPATIBILITY, new LaxBrowserCompatSpecFactory())
- .register(CookieSpecs.NETSCAPE, new NetscapeDraftSpecFactory())
- .register(CookieSpecs.IGNORE_COOKIES, new IgnoreSpecFactory())
+ .register(CookieSpecs.STANDARD, new LaxBrowserCompatSpecProvider())
.build();
/** Constructor. Private since we never instantiate.
@@ -183,6 +183,7 @@
IThrottleSpec throttleDescription, String[] binNames,
int connectionLimit,
String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword,
+ int socketTimeoutMilliseconds, int connectionTimeoutMilliseconds,
IAbortActivity activities)
throws ManifoldCFException, ServiceInterruption
{
@@ -208,7 +209,8 @@
// Construct a connection pool key
ConnectionPoolKey poolKey = new ConnectionPoolKey(protocol,server,port,authentication,
- trustStoreString,proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);
+ trustStoreString,proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword,
+ socketTimeoutMilliseconds,connectionTimeoutMilliseconds);
ConnectionPool p;
synchronized (connectionPools)
@@ -220,7 +222,8 @@
IConnectionThrottler connectionThrottler =
throttleGroups.obtainConnectionThrottler(webThrottleGroupType,throttleGroupName,binNames);
p = new ConnectionPool(connectionThrottler,protocol,server,port,authentication,baseFactory,
- proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);
+ proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword,
+ socketTimeoutMilliseconds,connectionTimeoutMilliseconds);
connectionPools.put(poolKey,p);
}
}
@@ -294,6 +297,10 @@
protected final String proxyAuthPassword;
/** Https protocol */
protected final javax.net.ssl.SSLSocketFactory httpsSocketFactory;
+ /** Socket timeout milliseconds */
+ protected final int socketTimeoutMilliseconds;
+ /** Connection timeout milliseconds */
+ protected final int connectionTimeoutMilliseconds;
/** The thread that is actually doing the work */
protected ExecuteMethodThread methodThread = null;
@@ -308,7 +315,8 @@
public ThrottledConnection(ConnectionPool myPool, IFetchThrottler fetchThrottler,
String protocol, String server, int port, PageCredentials authentication,
javax.net.ssl.SSLSocketFactory httpsSocketFactory,
- String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)
+ String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword,
+ int socketTimeoutMilliseconds, int connectionTimeoutMilliseconds)
{
this.myPool = myPool;
this.fetchThrottler = fetchThrottler;
@@ -322,6 +330,8 @@
this.port = port;
this.authentication = authentication;
this.httpsSocketFactory = httpsSocketFactory;
+ this.socketTimeoutMilliseconds = socketTimeoutMilliseconds;
+ this.connectionTimeoutMilliseconds = connectionTimeoutMilliseconds;
}
/** Set the abort checker. This must be done before the connection is actually used.
@@ -402,21 +412,20 @@
* @param urlPath is the path part of the url, e.g. "/robots.txt"
* @param userAgent is the value of the userAgent header to use.
* @param from is the value of the from header to use.
- * @param connectionTimeoutMilliseconds is the maximum number of milliseconds to wait on socket connect.
* @param redirectOK should be set to true if you want redirects to be automatically followed.
* @param host is the value to use as the "Host" header, or null to use the default.
* @param formData describes additional form arguments and how to fetch the page.
* @param loginCookies describes the cookies that should be in effect for this page fetch.
*/
@Override
- public void executeFetch(String urlPath, String userAgent, String from, int connectionTimeoutMilliseconds,
- int socketTimeoutMilliseconds, boolean redirectOK, String host, FormData formData,
+ public void executeFetch(String urlPath, String userAgent, String from,
+ boolean redirectOK, String host, FormData formData,
LoginCookies loginCookies)
throws ManifoldCFException, ServiceInterruption
{
// Set up scheme
SSLConnectionSocketFactory myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(httpsSocketFactory,connectionTimeoutMilliseconds),
- SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ NoopHostnameVerifier.INSTANCE);
int hostPort;
String displayedPort;
@@ -462,7 +471,17 @@
if (connManager == null)
{
- connManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingConnManager = new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory>create()
+ .register("http", PlainConnectionSocketFactory.getSocketFactory())
+ .register("https", myFactory)
+ .build());
+ poolingConnManager.setDefaultMaxPerRoute(1);
+ poolingConnManager.setValidateAfterInactivity(60000);
+ poolingConnManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeoutMilliseconds)
+ .build());
+ connManager = poolingConnManager;
}
long startTime = 0L;
@@ -486,11 +505,10 @@
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeoutMilliseconds)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeoutMilliseconds)
.setConnectionRequestTimeout(socketTimeoutMilliseconds)
- .setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY)
+ .setCookieSpec(CookieSpecs.STANDARD)
.setRedirectsEnabled(redirectOK);
// If there's a proxy, set that too.
@@ -512,20 +530,11 @@
httpClient = HttpClients.custom()
.setConnectionManager(connManager)
- .setMaxConnTotal(1)
- .setMaxConnPerRoute(1)
.disableAutomaticRetries()
- .setDefaultCookieSpecRegistry(cookieSpecRegistry)
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeoutMilliseconds)
- .build())
.setDefaultCredentialsProvider(credentialsProvider)
- .setSSLSocketFactory(myFactory)
.setRequestExecutor(new HttpRequestExecutor(socketTimeoutMilliseconds))
.setRedirectStrategy(new DefaultRedirectStrategy())
- // ??? need to add equivalent of setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY)
.build();
/*
@@ -1312,35 +1321,14 @@
/** Class to create a cookie spec.
*/
- protected static class LaxBrowserCompatSpecFactory extends BrowserCompatSpecFactory
+ protected static class LaxBrowserCompatSpecProvider extends RFC6265CookieSpecProvider
{
+ @Override
public CookieSpec create(HttpContext context)
{
return new LaxBrowserCompatSpec();
}
}
-
- /** Class to override browser compatibility to make it not check cookie paths. See CONNECTORS-97.
- */
- protected static class LaxBrowserCompatSpec extends BrowserCompatSpec
- {
-
- public LaxBrowserCompatSpec()
- {
- super();
- registerAttribHandler(ClientCookie.PATH_ATTR, new BasicPathHandler()
- {
- @Override
- public void validate(Cookie cookie, CookieOrigin origin) throws MalformedCookieException
- {
- // No validation
- }
-
- }
- );
- }
-
- }
/** This thread does the actual socket communication with the server.
* It's set up so that it can be abandoned at shutdown time.
@@ -1859,11 +1847,14 @@
protected final String proxyAuthDomain;
protected final String proxyAuthUsername;
protected final String proxyAuthPassword;
+ protected final int socketTimeoutMilliseconds;
+ protected final int connectionTimeoutMilliseconds;
public ConnectionPoolKey(String protocol,
String server, int port, PageCredentials authentication,
String trustStoreString, String proxyHost, int proxyPort,
- String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)
+ String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword,
+ int socketTimeoutMilliseconds, int connectionTimeoutMilliseconds)
{
this.protocol = protocol;
this.server = server;
@@ -1875,6 +1866,8 @@
this.proxyAuthDomain = proxyAuthDomain;
this.proxyAuthUsername = proxyAuthUsername;
this.proxyAuthPassword = proxyAuthPassword;
+ this.socketTimeoutMilliseconds = socketTimeoutMilliseconds;
+ this.connectionTimeoutMilliseconds = connectionTimeoutMilliseconds;
}
public int hashCode()
@@ -1888,7 +1881,9 @@
(proxyPort * 29) +
((proxyAuthDomain==null)?0:proxyAuthDomain.hashCode()) +
((proxyAuthUsername==null)?0:proxyAuthUsername.hashCode()) +
- ((proxyAuthPassword==null)?0:proxyAuthPassword.hashCode());
+ ((proxyAuthPassword==null)?0:proxyAuthPassword.hashCode()) +
+ new Integer(socketTimeoutMilliseconds).hashCode() +
+ new Integer(connectionTimeoutMilliseconds).hashCode();
}
public boolean equals(Object o)
@@ -1961,7 +1956,8 @@
if (!proxyAuthPassword.equals(other.proxyAuthPassword))
return false;
}
- return true;
+ return socketTimeoutMilliseconds == other.socketTimeoutMilliseconds &&
+ connectionTimeoutMilliseconds == other.connectionTimeoutMilliseconds;
}
}
@@ -1984,6 +1980,8 @@
protected final String proxyAuthDomain;
protected final String proxyAuthUsername;
protected final String proxyAuthPassword;
+ protected final int socketTimeoutMilliseconds;
+ protected final int connectionTimeoutMilliseconds;
/** The actual pool of connections */
protected final List<IThrottledConnection> connections = new ArrayList<IThrottledConnection>();
@@ -1993,7 +1991,8 @@
String server, int port, PageCredentials authentication,
javax.net.ssl.SSLSocketFactory baseFactory,
String proxyHost, int proxyPort,
- String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)
+ String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword,
+ int socketTimeoutMilliseconds, int connectionTimeoutMilliseconds)
{
this.connectionThrottler = connectionThrottler;
@@ -2007,6 +2006,8 @@
this.proxyAuthDomain = proxyAuthDomain;
this.proxyAuthUsername = proxyAuthUsername;
this.proxyAuthPassword = proxyAuthPassword;
+ this.socketTimeoutMilliseconds = socketTimeoutMilliseconds;
+ this.connectionTimeoutMilliseconds = connectionTimeoutMilliseconds;
}
public IThrottledConnection grab(IAbortActivity activities)
@@ -2031,7 +2032,8 @@
connection = new ThrottledConnection(this,connectionThrottler.getNewConnectionFetchThrottler(),
protocol,server,port,authentication,baseFactory,
proxyHost,proxyPort,
- proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);
+ proxyAuthDomain,proxyAuthUsername,proxyAuthPassword,
+ socketTimeoutMilliseconds,connectionTimeoutMilliseconds);
}
else
throw new IllegalStateException("Unexpected return value from waitConnectionAvailable(): "+result);
diff --git a/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java b/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java
index f0adedd..e8d0d5b 100644
--- a/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java
+++ b/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java
@@ -856,6 +856,7 @@
protocol,ipAddress,port,
credential,trustStore,throttleDescription,binNames,connectionLimit,
proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword,
+ socketTimeoutMilliseconds,connectionTimeoutMilliseconds,
activities);
try
{
@@ -863,8 +864,8 @@
try
{
// Execute the fetch!
- connection.executeFetch(url.getFile(),userAgent,from,connectionTimeoutMilliseconds,
- socketTimeoutMilliseconds,false,hostName,formData,lc);
+ connection.executeFetch(url.getFile(),userAgent,from,
+ false,hostName,formData,lc);
int response = connection.getResponseCode();
if (response == 200 || response == 302 || response == 301)
@@ -5345,13 +5346,14 @@
protocol,hostIPAddress,port,credential,
trustStore,throttleDescription,binNames,connectionLimit,
proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword,
+ socketTimeoutMilliseconds,connectionTimeoutMilliseconds,
versionActivities);
try
{
connection.beginFetch(FETCH_ROBOTS);
try
{
- connection.executeFetch("/robots.txt",userAgent,from,connectionTimeoutMilliseconds,socketTimeoutMilliseconds,true,hostName,null,null);
+ connection.executeFetch("/robots.txt",userAgent,from,true,hostName,null,null);
long expirationTime = currentTime+1000*60*60*24;
int code = connection.getResponseCode();
if (code == 200)
@@ -6155,6 +6157,12 @@
noteDiscoveredLink(rawURL);
}
+ @Override
+ public void finishUp()
+ throws ManifoldCFException
+ {
+ }
+
}
/** Class that describes XML handling */
diff --git a/connectors/webcrawler/pom.xml b/connectors/webcrawler/pom.xml
index d82416c..319d641 100644
--- a/connectors/webcrawler/pom.xml
+++ b/connectors/webcrawler/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java b/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
index 81b73c6..d003971 100644
--- a/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
+++ b/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
@@ -64,7 +64,11 @@
import org.apache.http.message.BasicHeader;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.protocol.HttpContext;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.entity.ContentType;
import org.apache.http.conn.ConnectTimeoutException;
@@ -227,10 +231,20 @@
javax.net.ssl.SSLSocketFactory httpsSocketFactory = KeystoreManagerFactory.getTrustingSecureSocketFactory();
SSLConnectionSocketFactory myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(httpsSocketFactory,connectionTimeout),
- SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ NoopHostnameVerifier.INSTANCE);
// Set up connection manager
- connectionManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory>create()
+ .register("http", PlainConnectionSocketFactory.getSocketFactory())
+ .register("https", myFactory)
+ .build());
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ connectionManager = poolingConnectionManager;
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
@@ -246,7 +260,6 @@
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(true)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
@@ -289,13 +302,8 @@
httpClient = HttpClients.custom()
.setConnectionManager(connectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(credentialsProvider)
.setSSLSocketFactory(myFactory)
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
diff --git a/connectors/wiki/pom.xml b/connectors/wiki/pom.xml
index a319ece..840ab8e 100644
--- a/connectors/wiki/pom.xml
+++ b/connectors/wiki/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-connectors</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/agents/pom.xml b/framework/agents/pom.xml
index 787948f..1abb618 100644
--- a/framework/agents/pom.xml
+++ b/framework/agents/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/api-service/pom.xml b/framework/api-service/pom.xml
index 0022508..9c939ce 100644
--- a/framework/api-service/pom.xml
+++ b/framework/api-service/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/api-servlet/pom.xml b/framework/api-servlet/pom.xml
index 77e932b..d8eaa46 100644
--- a/framework/api-servlet/pom.xml
+++ b/framework/api-servlet/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/authority-service/pom.xml b/framework/authority-service/pom.xml
index 96a38fc..e8e6f45 100644
--- a/framework/authority-service/pom.xml
+++ b/framework/authority-service/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/authority-servlet/pom.xml b/framework/authority-servlet/pom.xml
index cde172e..1f6e1de 100644
--- a/framework/authority-servlet/pom.xml
+++ b/framework/authority-servlet/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/combined-service/pom.xml b/framework/combined-service/pom.xml
index 9796caf..4b2eb0d 100644
--- a/framework/combined-service/pom.xml
+++ b/framework/combined-service/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/connector-common/pom.xml b/framework/connector-common/pom.xml
index 39d0596..99fd359 100644
--- a/framework/connector-common/pom.xml
+++ b/framework/connector-common/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/connector-common/src/main/java/org/apache/http/impl/cookie/LaxBrowserCompatSpec.java b/framework/connector-common/src/main/java/org/apache/http/impl/cookie/LaxBrowserCompatSpec.java
new file mode 100644
index 0000000..4fad9d8
--- /dev/null
+++ b/framework/connector-common/src/main/java/org/apache/http/impl/cookie/LaxBrowserCompatSpec.java
@@ -0,0 +1,51 @@
+/* $Id$ */
+
+/**`
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.http.impl.cookie;
+
+import org.apache.http.cookie.CookieOrigin;
+import org.apache.http.cookie.ClientCookie;
+import org.apache.http.cookie.CookieSpecProvider;
+import org.apache.http.cookie.Cookie;
+import org.apache.http.cookie.MalformedCookieException;
+
+/** Class to override browser compatibility to make it not check cookie paths. See CONNECTORS-97.
+* The class must be in the package described because it requires a protected constructor from the
+* class it extends.
+*/
+public class LaxBrowserCompatSpec extends RFC6265LaxSpec
+{
+
+ public LaxBrowserCompatSpec()
+ {
+ super(new BasicPathHandler()
+ {
+ @Override
+ public void validate(Cookie cookie, CookieOrigin origin) throws MalformedCookieException
+ {
+ // No validation
+ }
+ },
+ new BasicDomainHandler(),
+ new LaxMaxAgeHandler(),
+ new BasicSecureHandler(),
+ new LaxExpiresHandler());
+ }
+
+}
diff --git a/framework/core/pom.xml b/framework/core/pom.xml
index 0728585..f52f2ff 100644
--- a/framework/core/pom.xml
+++ b/framework/core/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/crawler-ui/pom.xml b/framework/crawler-ui/pom.xml
index 76af32f..0e2d5ab 100644
--- a/framework/crawler-ui/pom.xml
+++ b/framework/crawler-ui/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/crawler-ui/src/main/webapp/maxbandwidthreport.jsp b/framework/crawler-ui/src/main/webapp/maxbandwidthreport.jsp
index 9a80b40..f23a772 100644
--- a/framework/crawler-ui/src/main/webapp/maxbandwidthreport.jsp
+++ b/framework/crawler-ui/src/main/webapp/maxbandwidthreport.jsp
@@ -751,8 +751,8 @@
idBucketString = idBucketObject.toString();
String startTimeString = org.apache.manifoldcf.ui.util.Formatter.formatTime(Converter.asLong(row.getValue("starttime")));
String endTimeString = org.apache.manifoldcf.ui.util.Formatter.formatTime(Converter.asLong(row.getValue("endtime").toString()));
- long byteCount = Converter.asLong(row.getValue("bytecount"));
- double bandwidth = ((double)byteCount) * 1000.0 / intervalMilliseconds;
+ double byteCount = Converter.asDouble(row.getValue("bytecount"));
+ double bandwidth = byteCount * 1000.0 / intervalMilliseconds;
%>
<tr <%="class=\""+((zz%2==0)?"evendatarow":"odddatarow")+"\""%>>
diff --git a/framework/jetty-runner/pom.xml b/framework/jetty-runner/pom.xml
index cc9703d..5dfdfcf 100644
--- a/framework/jetty-runner/pom.xml
+++ b/framework/jetty-runner/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyShutdown.java b/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyShutdown.java
index c2039c3..6dc946f 100644
--- a/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyShutdown.java
+++ b/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyShutdown.java
@@ -71,25 +71,26 @@
int socketTimeout = 900000;
int connectionTimeout = 300000;
- HttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ HttpClientConnectionManager connectionManager = poolingConnectionManager;
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(false)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
HttpClient httpClient = HttpClients.custom()
.setConnectionManager(connectionManager)
- .setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
.setRedirectStrategy(new DefaultRedirectStrategy())
.build();
diff --git a/framework/pom.xml b/framework/pom.xml
index 040c0a5..596ac5d 100644
--- a/framework/pom.xml
+++ b/framework/pom.xml
@@ -20,13 +20,13 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-parent</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
<name>ManifoldCF - Framework</name>
<packaging>pom</packaging>
diff --git a/framework/pull-agent/pom.xml b/framework/pull-agent/pom.xml
index e0443e8..4bd1949 100644
--- a/framework/pull-agent/pom.xml
+++ b/framework/pull-agent/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
index 8c372fa..5103648 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
@@ -39,7 +39,6 @@
protected static final String cleanStufferLock = "_CLEANSTUFFER_";
protected static final String jobStopLock = "_JOBSTOP_";
protected static final String jobResumeLock = "_JOBRESUME_";
- protected static final String jobResetLock = "_JOBRESET_";
protected static final String hopLock = "_HOPLOCK_";
// Member variables
@@ -8339,7 +8338,9 @@
{
// Alternative to using a write lock here: Put this in a transaction, with a "FOR UPDATE" on the first query.
// I think that still causes way too much locking, though, on some databases.
- lockManager.enterWriteLock(jobResetLock);
+ // Note well: This MUST be the same lock as for finishStopJobs, since ABORTINGSHUTTINGDOWN is handled
+ // by one and SHUTTINGDOWN is handled by the other. CONNECTORS-1191.
+ lockManager.enterWriteLock(jobStopLock);
try
{
// Query for all jobs that fulfill the criteria
@@ -8417,7 +8418,7 @@
}
finally
{
- lockManager.leaveWriteLock(jobResetLock);
+ lockManager.leaveWriteLock(jobStopLock);
}
}
diff --git a/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeTester.java b/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeTester.java
index 0e55182..650a12d 100644
--- a/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeTester.java
+++ b/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeTester.java
@@ -97,7 +97,7 @@
// Now, delete the job.
jobManager.deleteJob(job.getID());
- instance.waitJobDeletedNative(jobManager,job.getID(),30000L);
+ instance.waitJobDeletedNative(jobManager,job.getID(),60000L);
// Shut down instance2
instance.stop();
diff --git a/framework/script-engine/pom.xml b/framework/script-engine/pom.xml
index 825ae0e..bda0227 100644
--- a/framework/script-engine/pom.xml
+++ b/framework/script-engine/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/framework/script-engine/src/main/java/org/apache/manifoldcf/scriptengine/ScriptParser.java b/framework/script-engine/src/main/java/org/apache/manifoldcf/scriptengine/ScriptParser.java
index ff21404..b0dcf2a 100644
--- a/framework/script-engine/src/main/java/org/apache/manifoldcf/scriptengine/ScriptParser.java
+++ b/framework/script-engine/src/main/java/org/apache/manifoldcf/scriptengine/ScriptParser.java
@@ -1221,14 +1221,20 @@
int socketTimeout = 900000;
int connectionTimeout = 300000;
- connectionManager = new PoolingHttpClientConnectionManager();
+ PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+ poolingConnectionManager.setDefaultMaxPerRoute(1);
+ poolingConnectionManager.setValidateAfterInactivity(60000);
+ poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+ .setTcpNoDelay(true)
+ .setSoTimeout(socketTimeout)
+ .build());
+ connectionManager = poolingConnectionManager;
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setSocketTimeout(socketTimeout)
- .setStaleConnectionCheckEnabled(false)
.setExpectContinueEnabled(true)
.setConnectTimeout(connectionTimeout)
.setConnectionRequestTimeout(socketTimeout);
@@ -1238,10 +1244,6 @@
.setMaxConnTotal(1)
.disableAutomaticRetries()
.setDefaultRequestConfig(requestBuilder.build())
- .setDefaultSocketConfig(SocketConfig.custom()
- .setTcpNoDelay(true)
- .setSoTimeout(socketTimeout)
- .build())
.setDefaultCredentialsProvider(credentialsProvider)
//.setSSLSocketFactory(myFactory)
.setRequestExecutor(new HttpRequestExecutor(socketTimeout))
diff --git a/framework/ui-core/pom.xml b/framework/ui-core/pom.xml
index c4c493c..a69b503 100644
--- a/framework/ui-core/pom.xml
+++ b/framework/ui-core/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-framework</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pom.xml b/pom.xml
index 1e9de51..73498d7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -28,7 +28,7 @@
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-parent</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
<name>ManifoldCF</name>
<packaging>pom</packaging>
@@ -56,14 +56,14 @@
<castor.version>1.0.5</castor.version>
<commons-codec.version>1.9</commons-codec.version>
<commons-io.version>2.4</commons-io.version>
- <commons-logging.version>1.1.3</commons-logging.version>
+ <commons-logging.version>1.2</commons-logging.version>
<commons-collections.version>3.2.1</commons-collections.version>
<commons-fileupload.version>1.2.2</commons-fileupload.version>
<commons-discovery.version>0.5</commons-discovery.version>
- <httpcomponent.httpclient.version>4.3.6</httpcomponent.httpclient.version>
- <httpcomponent.httpcore.version>4.3.3</httpcomponent.httpcore.version>
- <httpcomponent.httpmime.version>4.3.6</httpcomponent.httpmime.version>
- <solr.version>5.0.0</solr.version>
+ <httpcomponent.httpclient.version>4.4.1</httpcomponent.httpclient.version>
+ <httpcomponent.httpcore.version>4.4.1</httpcomponent.httpcore.version>
+ <httpcomponent.httpmime.version>4.4</httpcomponent.httpmime.version>
+ <solr.version>5.1.0</solr.version>
<noggit.version>0.6</noggit.version>
<commons-el.version>1.0</commons-el.version>
<commons-lang.version>2.6</commons-lang.version>
diff --git a/site/src/documentation/content/xdocs/ja_JP/end-user-documentation.xml b/site/src/documentation/content/xdocs/ja_JP/end-user-documentation.xml
index 9d0baa3..aa35bfd 100644
--- a/site/src/documentation/content/xdocs/ja_JP/end-user-documentation.xml
+++ b/site/src/documentation/content/xdocs/ja_JP/end-user-documentation.xml
@@ -579,9 +579,9 @@
<p>見ての通り、Active Directory権限では、異なってはいるがおそらく関連するドメインコントローラに対して複数のコネクションを設定できます。どのドメインコントローラにアクセスするかは、設定されたドメインコントローラの一覧を上から下まで見て、指定されたドメインサフィックスフィールドが最初に一致するものを選びます。ドメインサフィックスを空にすると<strong>すべての</strong>ユーザに一致すること注意してください。</p>
<p>一覧の最後にドメインコントローラを追加するには、必要な項目を入力してください。普通は「管理者ユーザ名」にはドメインを入力する必要はありませんが、ドメインコントローラの構成によっては「ユーザ名@ドメイン」形式で記入する必要があります。入力し終わったら「最後に追加する」ボタンを押して、一覧の最後にドメインコントローラを追加してください。後で、他のドメインコントローラが一覧にあれば、適切な所にある別のボタンを押してドメインコントローラを好きな所に挿入できます。</p>
<p>Active Directory権限コネクションタイプには個々のユーザの応答のキャッシュを制御するための「キャッシュ」タブもあります。</p>
- <br/><br/>
+ <!--br/><br/>
<figure src="images/ja_JP/ad-configure-cache_ja_JP.PNG" alt="AD設定、キャッシュタブ" width="80%"/>
- <br/><br/>
+ <br/><br/ -->
<p>ここではどれだけ多くの個々のユーザをどのくらい長くキャッシュするかを制御できます</p>
<p>入力した後に「保存」ボタンを押下すると、次のような設定概要と状態ページが表示されます：</p>
<br/><br/>
@@ -1089,17 +1089,6 @@
</section>
- <section id="wikirepository">
- <title>Wikiリポジトリコネクション</title>
- <p>Wikiリポジトリコネクションタイプは、Wiki及びMediaWikiサイトのコンテンツから索引を作成します。WikiリポジトリコネクションタイプはWiki APIを介してコンテンツを取得します。一般公開されている内容のみから索引を作成するため、認証設定はありません。</p>
- <p>リポジトリ編集ページで、Wikiコネクションを選択すると、「サーバ」タブが表示されます。「サーバ」タブを選択すると次のようなページが表示されます：</p>
- <br/><br/>
- <figure src="images/ja_JP/wiki-configure-server_ja_JP.PNG" alt="Wiki Connection, Server tab" width="80%"/>
- <br/><br/>
- <p>プロトコルは「プロトコル」ドロップダウンリストから選択してください。現バージョンは「http」プロトコルのみに対応しています。サーバ名を項目「サーバ名」に入力して、ポート番号を項目「ポート」にください。最後に、WikiのURIを項目「パス名」に入力してください。URIの先頭は文字「/」にしてください。</p>
- <p>ジョブのリポジトリコネクションにはWikiタイプ固有のタブは現バージョンにはありません。</p>
- </section>
-
<section id="jdbcrepository">
<title>汎用データベースリポジトリコネクション</title>
<p>汎用データベースコネクションタイプは次のデータベースのテーブルの内容から索引を作成します：</p>
@@ -1501,6 +1490,19 @@
<figure src="images/ja_JP/cmis-repository-connection-job-save_ja_JP.PNG" alt="CMIS Repository Connection, saving job" width="80%"/>
<br/><br/>
</section>
+
+ <section id="wikirepository">
+ <title>Wikiリポジトリコネクション</title>
+ <p>Wikiリポジトリコネクションタイプは、Wiki及びMediaWikiサイトのコンテンツから索引を作成します。WikiリポジトリコネクションタイプはWiki APIを介してコンテンツを取得します。一般公開されている内容のみから索引を作成するため、認証設定はありません。</p>
+ <p>このコネクションタイプでは、ジョブごとに手入力されたアクセストークン以外のいかなる種類の文書セキュリティもサポートされていません。</p>
+ <p>リポジトリ編集ページで、Wikiコネクションを選択すると、「サーバ」タブが表示されます。「サーバ」タブを選択すると次のようなページが表示されます：</p>
+ <br/><br/>
+ <figure src="images/ja_JP/wiki-configure-server_ja_JP.PNG" alt="Wikiコネクション、サーバタブ" width="80%"/>
+ <br/><br/>
+ <p>プロトコルは「プロトコル」ドロップダウンリストから選択してください。現バージョンは「http」プロトコルのみに対応しています。サーバ名を項目「サーバ名」に、ポート番号を項目「ポート」に入力してください。最後に、WikiのURLのパス部分を項目「パス名」に入力してください。URLのパス部分の先頭は文字「/」にしてください。</p>
+ <p>ジョブのリポジトリコネクションにはWikiタイプ固有のタブは現バージョンにはありません。</p>
+ </section>
+
</section>
diff --git a/test-materials/alfresco-4-war/pom.xml b/test-materials/alfresco-4-war/pom.xml
index c184880..c0055fd 100644
--- a/test-materials/alfresco-4-war/pom.xml
+++ b/test-materials/alfresco-4-war/pom.xml
@@ -14,7 +14,7 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>alfresco-4-war</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
<name>ManifoldCF - Test Materials - Alfresco WAR</name>
<description>Alfresco WAR builder</description>
<packaging>war</packaging>
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-test-materials</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<properties>
diff --git a/test-materials/pom.xml b/test-materials/pom.xml
index cb7775b..b7781bd 100644
--- a/test-materials/pom.xml
+++ b/test-materials/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.manifoldcf</groupId>
<artifactId>mcf-parent</artifactId>
- <version>2.1-SNAPSHOT</version>
+ <version>2.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>mcf-test-materials</artifactId>
