Implement feature and documentation.
git-svn-id: https://svn.apache.org/repos/asf/manifoldcf/branches/CONNECTORS-663@1457304 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java b/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java
index 6fc2b6d..e8ea8b8 100644
--- a/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java
+++ b/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java
@@ -54,6 +54,16 @@
{
}
+ /** Tell the world what model this connector uses for getDocumentIdentifiers().
+ * This must return a model value as specified above.
+ *@return the model type value.
+ */
+ @Override
+ public int getConnectorModel()
+ {
+ return MODEL_CHAINED_ADD_CHANGE;
+ }
+
/** Return the list of relationship types that this connector recognizes.
*@return the list.
*/
diff --git a/framework/crawler-ui/src/main/webapp/editjob.jsp b/framework/crawler-ui/src/main/webapp/editjob.jsp
index 2bf2594..334efab 100644
--- a/framework/crawler-ui/src/main/webapp/editjob.jsp
+++ b/framework/crawler-ui/src/main/webapp/editjob.jsp
@@ -75,6 +75,8 @@
EnumeratedValues minutesOfHour = null;
// Duration in minutes
Long duration = null;
+ // RequestMinimum flag
+ boolean requestMinimum = false;
// Priority
int priority = 5;
@@ -830,7 +832,7 @@
%>
<table class="displaytable">
<%
- if (model != -1 && model != IRepositoryConnector.MODEL_ADD_CHANGE_DELETE)
+ if (model != -1 && model != IRepositoryConnector.MODEL_ADD_CHANGE_DELETE && model != IRepositoryConnector.MODEL_CHAINED_ADD_CHANGE_DELETE)
{
%>
<tr>
@@ -902,6 +904,7 @@
EnumeratedValues srYear = sr.getYear();
EnumeratedValues srHourOfDay = sr.getHourOfDay();
EnumeratedValues srMinutesOfHour = sr.getMinutesOfHour();
+ boolean srRequestMinimum = sr.getRequestMinimum();
String postFix = Integer.toString(l);
int k;
@@ -936,10 +939,10 @@
int q = k;
String ampm;
if (k < 12)
- ampm = "am";
+ ampm = Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.am");
else
{
- ampm = "pm";
+ ampm = Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.pm");
q -= 12;
}
String hour;
@@ -989,15 +992,15 @@
int value = (k+1) % 10;
String suffix;
if (value == 1 && k != 10)
- suffix = "st";
+ suffix = Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.st");
else if (value == 2 && k != 11)
- suffix = "nd";
+ suffix = Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.nd");
else if (value == 3 && k != 12)
- suffix = "rd";
+ suffix = Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.rd");
else
- suffix = "th";
+ suffix = Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.th");
%>
- <option value='<%=Integer.toString(k)%>' <%=(srDayOfMonth!=null&&srDayOfMonth.checkValue(k))?"selected=\"selected\"":""%>><%=Integer.toString(k+1)+suffix+" day of month"%></option>
+ <option value='<%=Integer.toString(k)%>' <%=(srDayOfMonth!=null&&srDayOfMonth.checkValue(k))?"selected=\"selected\"":""%>><%=Integer.toString(k+1)+suffix+" "+Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.dayofmonth")%></option>
<%
k++;
}
@@ -1006,9 +1009,17 @@
</td>
</tr>
<tr>
- <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.MaximumRunTimeColon")%></nobr></td><td colspan="3" class="value">
+ <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.MaximumRunTimeColon")%></nobr></td>
+ <td class="value">
<input type="text" size="5" name='<%="duration"+postFix%>' value='<%=((srDuration==null)?"":new Long(srDuration.longValue()/60000L).toString())%>'/> <%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.minutes")%>
</td>
+ <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.JobInvocationColon")%></nobr></td>
+ <td class="value">
+ <select class="schedulepulldown" multiple="false" name='<%="invocation"+postFix%>' size="2">
+ <option value="complete" <%=(srRequestMinimum==false)?"selected=\"selected\"":""%>><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.Complete")%></option>
+ <option value="minimal" <%=srRequestMinimum?"selected=\"selected\"":""%>><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.Minimal")%></option>
+ </select>
+ </td>
</tr>
<tr>
<td class="message" colspan="4">
@@ -1047,10 +1058,10 @@
int q = k;
String ampm;
if (k < 12)
- ampm = "am";
+ ampm = Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.am");
else
{
- ampm = "pm";
+ ampm = Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.pm");
q -= 12;
}
String hour;
@@ -1100,15 +1111,15 @@
int value = (k+1) % 10;
String suffix;
if (value == 1 && k != 10)
- suffix = "st";
+ suffix = Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.st");
else if (value == 2 && k != 11)
- suffix = "nd";
+ suffix = Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.nd");
else if (value == 3 && k != 12)
- suffix = "rd";
+ suffix = Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.rd");
else
- suffix = "th";
+ suffix = Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.th");
%>
- <option value='<%=Integer.toString(k)%>' <%=(dayOfMonth!=null&&dayOfMonth.checkValue(k))?"selected=\"selected\"":""%>><%=Integer.toString(k+1)+suffix+" day of month"%></option>
+ <option value='<%=Integer.toString(k)%>' <%=(dayOfMonth!=null&&dayOfMonth.checkValue(k))?"selected=\"selected\"":""%>><%=Integer.toString(k+1)+suffix+" "+Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.dayofmonth")%></option>
<%
k++;
}
@@ -1117,9 +1128,17 @@
</td>
</tr>
<tr>
- <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.MaximumRunTimeColon")%></nobr></td><td colspan="3" class="value">
+ <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.MaximumRunTimeColon")%></nobr></td>
+ <td class="value">
<input type="text" size="5" name="duration" value='<%=((duration==null)?"":duration.toString())%>'/> <%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.minutes")%>
</td>
+ <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.JobInvocationColon")%></nobr></td>
+ <td class="value">
+ <select class="schedulepulldown" multiple="false" name="invocation" size="2">
+ <option value="complete" <%=(requestMinimum==false)?"selected=\"selected\"":""%>><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.Complete")%></option>
+ <option value="minimal" <%=requestMinimum?"selected=\"selected\"":""%>><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.Minimal")%></option>
+ </select>
+ </td>
</tr>
<tr>
<td class="message" colspan="4">
@@ -1150,6 +1169,7 @@
EnumeratedValues srYear = sr.getYear();
EnumeratedValues srHourOfDay = sr.getHourOfDay();
EnumeratedValues srMinutesOfHour = sr.getMinutesOfHour();
+ boolean srRequestMinimum = sr.getRequestMinimum();
String postFix = Integer.toString(l);
if (srDayOfWeek == null)
@@ -1243,6 +1263,7 @@
}
%>
<input type="hidden" name='<%="duration"+postFix%>' value='<%=((srDuration==null)?"":new Long(srDuration.longValue()/60000L).toString())%>'/>
+ <input type="hidden" name='<%="invocation"+postFix%>' value='<%=srRequestMinimum?"minimal":"complete"%>'/>
<input type="hidden" name='<%="year"+postFix%>' value="none"/>
<%
l++;
diff --git a/framework/crawler-ui/src/main/webapp/execute.jsp b/framework/crawler-ui/src/main/webapp/execute.jsp
index 09d7904..45e72a1 100644
--- a/framework/crawler-ui/src/main/webapp/execute.jsp
+++ b/framework/crawler-ui/src/main/webapp/execute.jsp
@@ -524,6 +524,7 @@
EnumeratedValues srHourOfDay = null;
EnumeratedValues srMinutesOfHour = null;
Long srDuration = null;
+ boolean srRequestMinimum = false;
y = variableContext.getParameterValues("dayofweek"+indexValue);
if (y != null)
@@ -581,12 +582,17 @@
else
srDuration = new Long(new Long(x).longValue()*60000L);
}
+ x = variableContext.getParameter("invocation"+indexValue);
+ if (x != null)
+ {
+ srRequestMinimum = x.equals("minimal");
+ }
x = variableContext.getParameter("recordop"+j);
if (x == null || !x.equals("Remove Schedule"))
{
ScheduleRecord sr = new ScheduleRecord(srDayOfWeek,srMonthOfYear,srDayOfMonth,srYear,srHourOfDay,srMinutesOfHour,
- null,srDuration);
+ null,srDuration,srRequestMinimum);
job.addScheduleRecord(sr);
}
j++;
@@ -604,6 +610,7 @@
EnumeratedValues srHourOfDay = null;
EnumeratedValues srMinutesOfHour = null;
Long srDuration = null;
+ boolean srRequestMinimum = false;
y = variableContext.getParameterValues("dayofweek");
if (y != null)
@@ -661,8 +668,14 @@
else
srDuration = new Long(new Long(x).longValue() * 60000L);
}
+ x = variableContext.getParameter("invocation");
+ if (x != null)
+ {
+ srRequestMinimum = x.equals("minimal");
+ }
+
ScheduleRecord sr = new ScheduleRecord(srDayOfWeek,srMonthOfYear,srDayOfMonth,srYear,srHourOfDay,srMinutesOfHour,
- null,srDuration);
+ null,srDuration,srRequestMinimum);
job.addScheduleRecord(sr);
}
@@ -894,11 +907,11 @@
}
else if (op != null && type != null && type.equals("jobstatus"))
{
- if (op.equals("Start"))
+ if (op.equals("Start") || op.equals("StartMinimal"))
{
// -- Start a job --
String jobID = variableContext.getParameter("jobid");
- manager.manualStart(new Long(jobID));
+ manager.manualStart(new Long(jobID),op.equals("StartMinimal"));
// Forward to showjobstatus
%>
<jsp:forward page="showjobstatus.jsp"/>
@@ -924,11 +937,11 @@
<jsp:forward page="showjobstatus.jsp"/>
<%
}
- else if (op.equals("Restart"))
+ else if (op.equals("Restart") || op.equals("RestartMinimal"))
{
// -- Restart a job --
String jobID = variableContext.getParameter("jobid");
- manager.manualAbortRestart(new Long(jobID));
+ manager.manualAbortRestart(new Long(jobID),op.equals("RestartMinimal"));
// Forward to showjobstatus
%>
<jsp:forward page="showjobstatus.jsp"/>
diff --git a/framework/crawler-ui/src/main/webapp/showjobstatus.jsp b/framework/crawler-ui/src/main/webapp/showjobstatus.jsp
index f7c0cf5..3875a10 100644
--- a/framework/crawler-ui/src/main/webapp/showjobstatus.jsp
+++ b/framework/crawler-ui/src/main/webapp/showjobstatus.jsp
@@ -44,6 +44,13 @@
document.liststatuses.submit();
}
+ function StartMinimal(jobID)
+ {
+ document.liststatuses.op.value="StartMinimal";
+ document.liststatuses.jobid.value=jobID;
+ document.liststatuses.submit();
+ }
+
function Abort(jobID)
{
document.liststatuses.op.value="Abort";
@@ -58,6 +65,13 @@
document.liststatuses.submit();
}
+ function RestartMinimal(jobID)
+ {
+ document.liststatuses.op.value="RestartMinimal";
+ document.liststatuses.jobid.value=jobID;
+ document.liststatuses.submit();
+ }
+
function Pause(jobID)
{
document.liststatuses.op.value="Pause";
@@ -117,66 +131,66 @@
switch (status)
{
case JobStatus.JOBSTATUS_NOTYETRUN:
- statusName = "Not yet run";
+ statusName = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Notyetrun");
break;
case JobStatus.JOBSTATUS_RUNNING:
- statusName = "Running";
+ statusName = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Running");
break;
case JobStatus.JOBSTATUS_RUNNING_UNINSTALLED:
- statusName = "Running, no connector";
+ statusName = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Runningnoconnector");
break;
case JobStatus.JOBSTATUS_ABORTING:
- statusName = "Aborting";
+ statusName = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Aborting");
break;
case JobStatus.JOBSTATUS_RESTARTING:
- statusName = "Restarting";
+ statusName = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Restarting");
break;
case JobStatus.JOBSTATUS_STOPPING:
- statusName = "Stopping";
+ statusName = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Stopping");
break;
case JobStatus.JOBSTATUS_RESUMING:
- statusName = "Resuming";
+ statusName = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Resuming");
break;
case JobStatus.JOBSTATUS_PAUSED:
- statusName = "Paused";
+ statusName = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Paused");
break;
case JobStatus.JOBSTATUS_COMPLETED:
- statusName = "Done";
+ statusName = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Done");
break;
case JobStatus.JOBSTATUS_WINDOWWAIT:
- statusName = "Waiting";
+ statusName = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus,Waiting");
break;
case JobStatus.JOBSTATUS_STARTING:
- statusName = "Starting up";
+ statusName = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Startingup");
break;
case JobStatus.JOBSTATUS_DESTRUCTING:
- statusName = "Cleaning up";
+ statusName = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Cleaningup");
break;
case JobStatus.JOBSTATUS_JOBENDCLEANUP:
- statusName = "Terminating";
+ statusName = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Terminating");
break;
case JobStatus.JOBSTATUS_JOBENDNOTIFICATION:
- statusName = "End notification";
+ statusName = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Endnotification");
break;
case JobStatus.JOBSTATUS_ERROR:
- statusName = "Error: "+js.getErrorText();
+ statusName = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.ErrorColon")+" "+js.getErrorText();
break;
default:
- statusName = "Unknown";
+ statusName = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Unknown");
break;
}
- String startTime = "Not started";
+ String startTime = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Notstarted");
if (js.getStartTime() != -1L)
startTime = new Date(js.getStartTime()).toString();
- String endTime = "Aborted";
+ String endTime = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Aborted");
if (js.getStartTime() == -1L)
- endTime = "Never run";
+ endTime = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Neverrun");
else
{
if (js.getEndTime() == -1L)
{
if (status == JobStatus.JOBSTATUS_COMPLETED)
- endTime = "Aborted";
+ endTime = Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Aborted");
else
endTime = "";
}
@@ -192,7 +206,8 @@
status == JobStatus.JOBSTATUS_ERROR)
{
%>
- <a href='<%="javascript:Start(\""+js.getJobID()+"\")"%>' alt='<%="Start job "+js.getJobID()%>'>Start</a>
+ <a href='<%="javascript:Start(\""+js.getJobID()+"\")"%>' alt='<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"showjobstatus.Startjob")+" "+js.getJobID()%>'><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Start")%></nobr></a>
+ <a href='<%="javascript:StartMinimal(\""+js.getJobID()+"\")"%>' alt='<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"showjobstatus.Startjob")+" "+js.getJobID()+" "+Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.minimally")%>'><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Startminimal")%></nobr></a>
<%
}
if (status == JobStatus.JOBSTATUS_RUNNING ||
@@ -202,7 +217,8 @@
status == JobStatus.JOBSTATUS_STARTING)
{
%>
- <a href='<%="javascript:Restart(\""+js.getJobID()+"\")"%>' alt='<%="Restart job "+js.getJobID()%>'>Restart</a>
+ <a href='<%="javascript:Restart(\""+js.getJobID()+"\")"%>' alt='<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"showjobstatus.Restartjob")+" "+js.getJobID()%>'><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Restart")%></nobr></a>
+ <a href='<%="javascript:RestartMinimal(\""+js.getJobID()+"\")"%>' alt='<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"showjobstatus.Restartjob")+" "+js.getJobID()+" "+Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.minimally")%>'><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Restartminimal")%></nobr></a>
<%
}
if (status == JobStatus.JOBSTATUS_RUNNING ||
@@ -210,7 +226,7 @@
status == JobStatus.JOBSTATUS_WINDOWWAIT)
{
%>
- <a href='<%="javascript:Pause(\""+js.getJobID()+"\")"%>' alt='<%="Pause job "+js.getJobID()%>'>Pause</a>
+ <a href='<%="javascript:Pause(\""+js.getJobID()+"\")"%>' alt='<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"showjobstatus.Pausejob")+" "+js.getJobID()%>'><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Pause")%></nobr></a>
<%
}
if (status == JobStatus.JOBSTATUS_RUNNING ||
@@ -223,13 +239,13 @@
status == JobStatus.JOBSTATUS_RESTARTING)
{
%>
- <a href='<%="javascript:Abort(\""+js.getJobID()+"\")"%>' alt='<%="Abort job "+js.getJobID()%>'>Abort</a>
+ <a href='<%="javascript:Abort(\""+js.getJobID()+"\")"%>' alt='<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"showjobstatus.Abortjob")+" "+js.getJobID()%>'><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Abort")%></nobr></a>
<%
}
if (status == JobStatus.JOBSTATUS_PAUSED)
{
%>
- <a href='<%="javascript:Resume(\""+js.getJobID()+"\")"%>' alt='<%="Resume job "+js.getJobID()%>'>Resume</a>
+ <a href='<%="javascript:Resume(\""+js.getJobID()+"\")"%>' alt='<%=Messages.getAttributeString(pageContext.getRequest().getLocale(),"showjobstatus.Resumejob")+" "+js.getJobID()%>'><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"showjobstatus.Resume")%></nobr></a>
<%
}
%>
diff --git a/framework/crawler-ui/src/main/webapp/viewjob.jsp b/framework/crawler-ui/src/main/webapp/viewjob.jsp
index 269312d..48d383f 100644
--- a/framework/crawler-ui/src/main/webapp/viewjob.jsp
+++ b/framework/crawler-ui/src/main/webapp/viewjob.jsp
@@ -139,7 +139,8 @@
<td class="separator" colspan="4"><hr/></td>
</tr>
<tr>
- <td class="description" colspan="1"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.NameColon")%></nobr></td><td class="value" colspan="3" ><%="<!--jobid="+jobID+"-->"%><%=org.apache.manifoldcf.ui.util.Encoder.bodyEscape(job.getDescription())%></td>
+ <td class="description" colspan="1"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.NameColon")%></nobr></td>
+ <td class="value" colspan="3" ><%="<!--jobid="+jobID+"-->"%><%=org.apache.manifoldcf.ui.util.Encoder.bodyEscape(job.getDescription())%></td>
</tr>
<tr>
<td class="separator" colspan="4"><hr/></td>
@@ -154,8 +155,10 @@
<td class="separator" colspan="4"><hr/></td>
</tr>
<tr>
- <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.PriorityColon")%></nobr></td><td class="value"><%=priority%></td>
- <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.StartMethodColon")%></nobr></td><td class="value"><%=startMethod%></td>
+ <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.PriorityColon")%></nobr></td>
+ <td class="value"><%=priority%></td>
+ <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.StartMethodColon")%></nobr></td>
+ <td class="value"><%=startMethod%></td>
</tr>
<%
if (model != -1 && model != IRepositoryConnector.MODEL_ADD_CHANGE_DELETE)
@@ -165,13 +168,16 @@
<td class="separator" colspan="4"><hr/></td>
</tr>
<tr>
- <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.ScheduleTypeColon")%></nobr></td><td class="value"><nobr><%=jobType%></nobr></td>
- <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.MinimumRecrawlIntervalColon")%></nobr></td><td class="value"><nobr><%=intervalString%></nobr>
- </td>
+ <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.ScheduleTypeColon")%></nobr></td>
+ <td class="value"><nobr><%=jobType%></nobr></td>
+ <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.MinimumRecrawlIntervalColon")%></nobr></td>
+ <td class="value"><nobr><%=intervalString%></nobr></td>
</tr>
<tr>
- <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.ExpirationIntervalColon")%></nobr></td><td class="value"><nobr><%=expirationIntervalString%></nobr></td>
- <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.ReseedIntervalColon")%></nobr></td><td class="value"><nobr><%=reseedIntervalString%></nobr></td>
+ <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.ExpirationIntervalColon")%></nobr></td>
+ <td class="value"><nobr><%=expirationIntervalString%></nobr></td>
+ <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.ReseedIntervalColon")%></nobr></td>
+ <td class="value"><nobr><%=reseedIntervalString%></nobr></td>
</tr>
<%
}
@@ -195,6 +201,7 @@
{
ScheduleRecord sr = job.getScheduleRecord(j);
Long srDuration = sr.getDuration();
+ boolean srRequestMinimum = sr.getRequestMinimum();
EnumeratedValues srDayOfWeek = sr.getDayOfWeek();
EnumeratedValues srMonthOfYear = sr.getMonthOfYear();
EnumeratedValues srDayOfMonth = sr.getDayOfMonth();
@@ -330,7 +337,7 @@
if (srMonthOfYear == null)
{
if (srDayOfMonth == null && srDayOfWeek == null && srHourOfDay == null && srMinutesOfHour == null)
- out.println(" "+Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.inJanuary"));
+ out.println(" "+Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.ineverymonthofyear"));
}
else
{
@@ -489,15 +496,13 @@
out.println(sb.toString());
}
%>
-
-
</td>
</tr>
<tr>
<td class="description">
<%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.MaximumRunTimeColon")%>
</td>
- <td class="value" colspan="3">
+ <td class="value">
<%
if (srDuration == null)
out.println(Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.Nolimit"));
@@ -505,8 +510,18 @@
out.println(new Long(srDuration.longValue()/60000L).toString() + " "+Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.minutes"));
%>
</td>
+ <td class="description">
+ <%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.JobInvocationColon")%>
+ </td>
+ <td class="value">
+<%
+ if (srRequestMinimum)
+ out.println(Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.Minimal"));
+ else
+ out.println(Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.Complete"));
+%>
+ </td>
</tr>
-
<%
j++;
}
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/AddScheduledTime.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/AddScheduledTime.java
index 60fc904..c9d1f13 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/AddScheduledTime.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/AddScheduledTime.java
@@ -28,265 +28,272 @@
*/
public class AddScheduledTime
{
- public static final String _rcsid = "@(#)$Id: AddScheduledTime.java 988245 2010-08-23 18:39:35Z kwright $";
+ public static final String _rcsid = "@(#)$Id: AddScheduledTime.java 988245 2010-08-23 18:39:35Z kwright $";
- private AddScheduledTime()
- {
- }
+ private AddScheduledTime()
+ {
+ }
- public static void main(String[] args)
- {
- if (args.length != 8)
- {
- System.err.println("Usage: AddScheduledTime <jobid> <interval_minutes> <day_of_week_list>");
- System.err.println(" <day_of_month_list> <month_list> <year_list> <hour_list> <minute_list>");
- System.err.println("");
- System.err.println("If <interval_minutes> is an empty string, no interval limit");
- System.err.println("All lists are comma-separated");
- System.err.println("All list values can be empty to indicate no constraint");
- System.err.println("Days of week are lower case, full day names, e.g. tuesday");
- System.err.println("Months are full month names in lower case, e.g. january");
- System.err.println("Years are full year values, e.g. 1999");
- System.err.println("Hours include am or pm in lower case, e.g. 12am or 2pm");
+ public static void main(String[] args)
+ {
+ if (args.length != 8 && args.length != 9)
+ {
+ System.err.println("Usage: AddScheduledTime <jobid> <interval_minutes> <day_of_week_list>");
+ System.err.println(" <day_of_month_list> <month_list> <year_list> <hour_list> <minute_list> [<request_minimum>]");
+ System.err.println("");
+ System.err.println("If <interval_minutes> is an empty string, no interval limit");
+ System.err.println("All lists are comma-separated");
+ System.err.println("All list values can be empty to indicate no constraint");
+ System.err.println("Days of week are lower case, full day names, e.g. tuesday");
+ System.err.println("Months are full month names in lower case, e.g. january");
+ System.err.println("Years are full year values, e.g. 1999");
+ System.err.println("Hours include am or pm in lower case, e.g. 12am or 2pm");
+ System.err.println("<request_minimum> is 'true' or 'false'");
- System.exit(1);
- }
+ System.exit(1);
+ }
- String jobID = args[0];
- String interval = args[1];
- String dayOfWeekList = args[2];
- String dayOfMonthList = args[3];
- String monthList = args[4];
- String yearList = args[5];
- String hourList = args[6];
- String minuteList = args[7];
+ String jobID = args[0];
+ String interval = args[1];
+ String dayOfWeekList = args[2];
+ String dayOfMonthList = args[3];
+ String monthList = args[4];
+ String yearList = args[5];
+ String hourList = args[6];
+ String minuteList = args[7];
+ String requestMinimum;
+ if (args.length == 9)
+ requestMinimum = args[8];
+ else
+ requestMinimum = "false";
- try
- {
- ManifoldCF.initializeEnvironment();
- IThreadContext tc = ThreadContextFactory.make();
- IJobManager jobManager = JobManagerFactory.make(tc);
+ try
+ {
+ ManifoldCF.initializeEnvironment();
+ IThreadContext tc = ThreadContextFactory.make();
+ IJobManager jobManager = JobManagerFactory.make(tc);
- IJobDescription desc = jobManager.load(new Long(jobID));
- if (desc == null)
- throw new ManifoldCFException("No such job: '"+jobID+"'");
+ IJobDescription desc = jobManager.load(new Long(jobID));
+ if (desc == null)
+ throw new ManifoldCFException("No such job: '"+jobID+"'");
- ScheduleRecord sr = new ScheduleRecord(
- parseDayOfWeek(dayOfWeekList),
- parseMonthOfYear(monthList),
- parseDayOfMonth(dayOfMonthList),
- parseYear(yearList),
- parseHourOfDay(hourList),
- parseMinutes(minuteList),
- null,
- (interval.length()>0)?new Long(interval):null);
+ ScheduleRecord sr = new ScheduleRecord(
+ parseDayOfWeek(dayOfWeekList),
+ parseMonthOfYear(monthList),
+ parseDayOfMonth(dayOfMonthList),
+ parseYear(yearList),
+ parseHourOfDay(hourList),
+ parseMinutes(minuteList),
+ null,
+ (interval.length()>0)?new Long(interval):null,
+ requestMinimum.equals("true"));
- desc.addScheduleRecord(sr);
- jobManager.save(desc);
+ desc.addScheduleRecord(sr);
+ jobManager.save(desc);
- System.out.println("Job updated");
- }
- catch (Exception e)
- {
- e.printStackTrace();
- System.exit(2);
- }
- }
+ System.out.println("Job updated");
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ System.exit(2);
+ }
+ }
- protected static EnumeratedValues parseDayOfWeek(String list)
- throws Exception
- {
- if (list.length() == 0)
- return null;
+ protected static EnumeratedValues parseDayOfWeek(String list)
+ throws Exception
+ {
+ if (list.length() == 0)
+ return null;
- StringBuilder sb = new StringBuilder();
- int index = 0;
- while (true)
- {
- if (index != 0)
- sb.append(",");
- int newIndex = list.indexOf(",",index);
- if (newIndex == -1)
- {
- sb.append(new Integer(mapToWeekday(list.substring(index))).toString());
- break;
- }
- sb.append(new Integer(mapToWeekday(list.substring(index,newIndex))).toString());
- index = newIndex+1;
- }
- return new EnumeratedValues(sb.toString());
- }
+ StringBuilder sb = new StringBuilder();
+ int index = 0;
+ while (true)
+ {
+ if (index != 0)
+ sb.append(",");
+ int newIndex = list.indexOf(",",index);
+ if (newIndex == -1)
+ {
+ sb.append(new Integer(mapToWeekday(list.substring(index))).toString());
+ break;
+ }
+ sb.append(new Integer(mapToWeekday(list.substring(index,newIndex))).toString());
+ index = newIndex+1;
+ }
+ return new EnumeratedValues(sb.toString());
+ }
- protected static int mapToWeekday(String day)
- throws Exception
- {
- if (day.equals("sunday"))
- return 0;
- if (day.equals("monday"))
- return 1;
- if (day.equals("tuesday"))
- return 2;
- if (day.equals("wednesday"))
- return 3;
- if (day.equals("thursday"))
- return 4;
- if (day.equals("friday"))
- return 5;
- if (day.equals("saturday"))
- return 6;
- throw new ManifoldCFException("Bad day of week: '"+day+"'");
- }
+ protected static int mapToWeekday(String day)
+ throws Exception
+ {
+ if (day.equals("sunday"))
+ return 0;
+ if (day.equals("monday"))
+ return 1;
+ if (day.equals("tuesday"))
+ return 2;
+ if (day.equals("wednesday"))
+ return 3;
+ if (day.equals("thursday"))
+ return 4;
+ if (day.equals("friday"))
+ return 5;
+ if (day.equals("saturday"))
+ return 6;
+ throw new ManifoldCFException("Bad day of week: '"+day+"'");
+ }
- protected static EnumeratedValues parseMonthOfYear(String list)
- throws Exception
- {
- if (list.length() == 0)
- return null;
+ protected static EnumeratedValues parseMonthOfYear(String list)
+ throws Exception
+ {
+ if (list.length() == 0)
+ return null;
- StringBuilder sb = new StringBuilder();
- int index = 0;
- while (true)
- {
- if (index != 0)
- sb.append(",");
- int newIndex = list.indexOf(",",index);
- if (newIndex == -1)
- {
- sb.append(new Integer(mapToMonth(list.substring(index))).toString());
- break;
- }
- sb.append(new Integer(mapToMonth(list.substring(index,newIndex))).toString());
- index = newIndex+1;
- }
- return new EnumeratedValues(sb.toString());
+ StringBuilder sb = new StringBuilder();
+ int index = 0;
+ while (true)
+ {
+ if (index != 0)
+ sb.append(",");
+ int newIndex = list.indexOf(",",index);
+ if (newIndex == -1)
+ {
+ sb.append(new Integer(mapToMonth(list.substring(index))).toString());
+ break;
+ }
+ sb.append(new Integer(mapToMonth(list.substring(index,newIndex))).toString());
+ index = newIndex+1;
+ }
+ return new EnumeratedValues(sb.toString());
- }
+ }
- protected static int mapToMonth(String day)
- throws Exception
- {
- if (day.equals("january"))
- return 0;
- if (day.equals("february"))
- return 1;
- if (day.equals("march"))
- return 2;
- if (day.equals("april"))
- return 3;
- if (day.equals("may"))
- return 4;
- if (day.equals("june"))
- return 5;
- if (day.equals("july"))
- return 6;
- if (day.equals("august"))
- return 7;
- if (day.equals("september"))
- return 8;
- if (day.equals("october"))
- return 9;
- if (day.equals("november"))
- return 10;
- if (day.equals("december"))
- return 11;
+ protected static int mapToMonth(String day)
+ throws Exception
+ {
+ if (day.equals("january"))
+ return 0;
+ if (day.equals("february"))
+ return 1;
+ if (day.equals("march"))
+ return 2;
+ if (day.equals("april"))
+ return 3;
+ if (day.equals("may"))
+ return 4;
+ if (day.equals("june"))
+ return 5;
+ if (day.equals("july"))
+ return 6;
+ if (day.equals("august"))
+ return 7;
+ if (day.equals("september"))
+ return 8;
+ if (day.equals("october"))
+ return 9;
+ if (day.equals("november"))
+ return 10;
+ if (day.equals("december"))
+ return 11;
- throw new ManifoldCFException("Bad month: '"+day+"'");
- }
+ throw new ManifoldCFException("Bad month: '"+day+"'");
+ }
- protected static EnumeratedValues parseDayOfMonth(String list)
- throws Exception
- {
- // Zero based internally
- if (list.length() == 0)
- return null;
+ protected static EnumeratedValues parseDayOfMonth(String list)
+ throws Exception
+ {
+ // Zero based internally
+ if (list.length() == 0)
+ return null;
- StringBuilder sb = new StringBuilder();
- int index = 0;
- while (true)
- {
- if (index != 0)
- sb.append(",");
- int newIndex = list.indexOf(",",index);
- if (newIndex == -1)
- {
- sb.append(new Integer(new Integer(list.substring(index)).intValue() - 1).toString());
- break;
- }
- sb.append(new Integer(new Integer(list.substring(index,newIndex)).intValue() - 1).toString());
- index = newIndex+1;
- }
- return new EnumeratedValues(sb.toString());
- }
+ StringBuilder sb = new StringBuilder();
+ int index = 0;
+ while (true)
+ {
+ if (index != 0)
+ sb.append(",");
+ int newIndex = list.indexOf(",",index);
+ if (newIndex == -1)
+ {
+ sb.append(new Integer(new Integer(list.substring(index)).intValue() - 1).toString());
+ break;
+ }
+ sb.append(new Integer(new Integer(list.substring(index,newIndex)).intValue() - 1).toString());
+ index = newIndex+1;
+ }
+ return new EnumeratedValues(sb.toString());
+ }
- protected static EnumeratedValues parseYear(String list)
- throws Exception
- {
- if (list.length() == 0)
- return null;
- return new EnumeratedValues(list);
- }
+ protected static EnumeratedValues parseYear(String list)
+ throws Exception
+ {
+ if (list.length() == 0)
+ return null;
+ return new EnumeratedValues(list);
+ }
- protected static EnumeratedValues parseHourOfDay(String list)
- throws Exception
- {
- if (list.length() == 0)
- return null;
+ protected static EnumeratedValues parseHourOfDay(String list)
+ throws Exception
+ {
+ if (list.length() == 0)
+ return null;
- StringBuilder sb = new StringBuilder();
- int index = 0;
- while (true)
- {
- if (index != 0)
- sb.append(",");
- int newIndex = list.indexOf(",",index);
- if (newIndex == -1)
- {
- sb.append(new Integer(mapToHour(list.substring(index))).toString());
- break;
- }
- sb.append(new Integer(mapToHour(list.substring(index,newIndex))).toString());
- index = newIndex+1;
- }
- return new EnumeratedValues(sb.toString());
+ StringBuilder sb = new StringBuilder();
+ int index = 0;
+ while (true)
+ {
+ if (index != 0)
+ sb.append(",");
+ int newIndex = list.indexOf(",",index);
+ if (newIndex == -1)
+ {
+ sb.append(new Integer(mapToHour(list.substring(index))).toString());
+ break;
+ }
+ sb.append(new Integer(mapToHour(list.substring(index,newIndex))).toString());
+ index = newIndex+1;
+ }
+ return new EnumeratedValues(sb.toString());
- }
+ }
- protected static int mapToHour(String hour)
- throws Exception
- {
- boolean isPM;
- String value;
- if (hour.endsWith("am"))
- {
- isPM = false;
- value = hour.substring(0,hour.length()-2);
- }
- else if (hour.endsWith("pm"))
- {
- isPM = true;
- value = hour.substring(0,hour.length()-2);
- }
- else
- {
- isPM = false;
- value = hour;
- }
+ protected static int mapToHour(String hour)
+ throws Exception
+ {
+ boolean isPM;
+ String value;
+ if (hour.endsWith("am"))
+ {
+ isPM = false;
+ value = hour.substring(0,hour.length()-2);
+ }
+ else if (hour.endsWith("pm"))
+ {
+ isPM = true;
+ value = hour.substring(0,hour.length()-2);
+ }
+ else
+ {
+ isPM = false;
+ value = hour;
+ }
- int rval = new Integer(value).intValue();
- if (rval == 12)
- rval = 0;
- if (isPM)
- rval += 12;
- return rval;
- }
+ int rval = new Integer(value).intValue();
+ if (rval == 12)
+ rval = 0;
+ if (isPM)
+ rval += 12;
+ return rval;
+ }
- protected static EnumeratedValues parseMinutes(String list)
- throws Exception
- {
- if (list.length() == 0)
- return null;
+ protected static EnumeratedValues parseMinutes(String list)
+ throws Exception
+ {
+ if (list.length() == 0)
+ return null;
- return new EnumeratedValues(list);
- }
+ return new EnumeratedValues(list);
+ }
}
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/GetJobSchedule.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/GetJobSchedule.java
index b59e89d..51dbecf 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/GetJobSchedule.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/GetJobSchedule.java
@@ -41,7 +41,7 @@
System.err.println("Usage: GetJobSchedule <job_id>");
System.err.println("");
System.err.println("The result will be UTF-8 encoded and will consist of the following columns:");
- System.err.println(" daysofweek,years,months,days,hours,minutes,timezone,duration");
+ System.err.println(" daysofweek,years,months,days,hours,minutes,timezone,duration,requestminimum");
System.exit(1);
}
@@ -70,7 +70,8 @@
enumerate(sr.getHourOfDay())+","+
enumerate(sr.getMinutesOfHour())+","+
((sr.getTimezone()==null)?"":sr.getTimezone())+","+
- presentInterval(sr.getDuration()));
+ presentInterval(sr.getDuration())+","+
+ (sr.getRequestMinimum()?"true":"false"));
}
System.err.println("Schedule list done");
}
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java
index c1245be..ef595a8 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java
@@ -671,6 +671,15 @@
* will not cease until complete. If the job is already running, this operation will assure that
* the job does not pause when its window ends. The job can be manually paused, or manually aborted.
*@param jobID is the ID of the job to start.
+ *@param requestMinimum is true if a minimal job run is requested.
+ */
+ public void manualStart(Long jobID, boolean requestMinimum)
+ throws ManifoldCFException;
+
+ /** Manually start a job. The specified job will be run REGARDLESS of the timed windows, and
+ * will not cease until complete. If the job is already running, this operation will assure that
+ * the job does not pause when its window ends. The job can be manually paused, or manually aborted.
+ *@param jobID is the ID of the job to start.
*/
public void manualStart(Long jobID)
throws ManifoldCFException;
@@ -685,6 +694,14 @@
/** Manually restart a running job. The job will be stopped and restarted. Any schedule affinity will be lost,
* until the job finishes on its own.
*@param jobID is the job to abort.
+ *@param requestMinimum is true if a minimal job run is requested.
+ */
+ public void manualAbortRestart(Long jobID, boolean requestMinimum)
+ throws ManifoldCFException;
+
+ /** Manually restart a running job. The job will be stopped and restarted. Any schedule affinity will be lost,
+ * until the job finishes on its own.
+ *@param jobID is the job to abort.
*/
public void manualAbortRestart(Long jobID)
throws ManifoldCFException;
@@ -732,7 +749,7 @@
*@return jobs that are active and are running in adaptive mode. These will be seeded
* based on what the connector says should be added to the queue.
*/
- public JobStartRecord[] getJobsReadyForSeeding(long currentTime)
+ public JobSeedingRecord[] getJobsReadyForSeeding(long currentTime)
throws ManifoldCFException;
/** Reset a seeding job back to "active" state.
@@ -744,7 +761,7 @@
/** Get the list of jobs that are ready for deletion.
*@return jobs that were in the "readyfordelete" state.
*/
- public JobStartRecord[] getJobsReadyForDelete()
+ public JobDeleteRecord[] getJobsReadyForDelete()
throws ManifoldCFException;
/** Get the list of jobs that are ready for startup.
@@ -756,7 +773,7 @@
/** Find the list of jobs that need to have their connectors notified of job completion.
*@return the ID's of jobs that need their output connectors notified in order to become inactive.
*/
- public JobStartRecord[] getJobsReadyForInactivity()
+ public JobNotifyRecord[] getJobsReadyForInactivity()
throws ManifoldCFException;
/** Inactivate a job, from the notification state.
@@ -791,22 +808,23 @@
public void prepareDeleteScan(Long jobID)
throws ManifoldCFException;
- /** Prepare for a full scan.
+ /** Prepare a job to be run.
+ * This method is called regardless of the details of the job; what differs is only the flags that are passed in.
+ * The code inside will determine the appropriate procedures.
+ * (This method replaces prepareFullScan() and prepareIncrementalScan(). )
*@param jobID is the job id.
*@param legalLinkTypes are the link types allowed for the job.
*@param hopcountMethod describes how to handle deletions for hopcount purposes.
+ *@param connectorModel is the model used by the connector for the job.
+ *@param continuousJob is true if the job is a continuous one.
+ *@param fromBeginningOfTime is true if the job is running starting from time 0.
+ *@param requestMinimum is true if the minimal amount of work is requested for the job run.
*/
- public void prepareFullScan(Long jobID, String[] legalLinkTypes, int hopcountMethod)
+ public void prepareJobScan(Long jobID, String[] legalLinkTypes, int hopcountMethod,
+ int connectorModel, boolean continuousJob, boolean fromBeginningOfTime,
+ boolean requestMinimum)
throws ManifoldCFException;
-
- /** Prepare for an incremental scan.
- *@param jobID is the job id.
- *@param legalLinkTypes are the link types allowed for the job.
- *@param hopcountMethod describes how to handle deletions for hopcount purposes.
- */
- public void prepareIncrementalScan(Long jobID, String[] legalLinkTypes, int hopcountMethod)
- throws ManifoldCFException;
-
+
/** Note job delete started.
*@param jobID is the job id.
*@param startTime is the job start time.
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IRepositoryConnector.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IRepositoryConnector.java
index af0a950..26741a4 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IRepositoryConnector.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IRepositoryConnector.java
@@ -69,9 +69,19 @@
// is the most restrictive that is still accurate. For example, if MODEL_ADD_CHANGE_DELETE applies, you would
// return that value rather than MODEL_ADD.
+ // For the CHAINED models, what the connector is describing are the documents that will be processed IF the seeded
+ // documents are followed to their leaves. For instance, imagine a hierarchy where the root document is the only one ever
+ // seeded, but if that document is processed, and its discovered changed children are processed as well, then all documents
+ // that have been added, changed, or deleted will eventually be discovered. In that case, model
+ // MODEL_CHAINED_ADD_CHANGE_DELETE would be appropriate. But, if a changed node can only discover child
+ // additions and changes, then MODEL_CHAINED_ADD_CHANGE would be the right choice.
+
/** Supply all seeds every time. The connector does not pay any attention to the start time or end time
* of the request, and simply returns a complete list of seeds. */
public static final int MODEL_ALL = 0;
+ /** This indicates that the seeds are never complete; the previous seeds are lost and cannot be retrieved. */
+ public static final int MODEL_PARTIAL = 4;
+
/** Supply at least the documents that have been added since the specified start time. Connector is
* aware of the start time and end time of the request, and supplies at least the documents that have been
* added within the specified time range. */
@@ -80,9 +90,14 @@
public static final int MODEL_ADD_CHANGE = 2;
/** Supply at least the documents that have been added, changed, or deleted within the specified time range. */
public static final int MODEL_ADD_CHANGE_DELETE = 3;
- /** This indicates that the seeds are never complete; the previous seeds are lost and cannot be retrieved. */
- public static final int MODEL_PARTIAL = 4;
+ /** Like MODEL_ADD, except considering document discovery */
+ public static final int MODEL_CHAINED_ADD = 9;
+ /** Like MODEL_ADD_CHANGE, except considering document discovery */
+ public static final int MODEL_CHAINED_ADD_CHANGE = 10;
+ /** Like MODEL_ADD_CHANGE_DELETE, except considering document discovery */
+ public static final int MODEL_CHAINED_ADD_CHANGE_DELETE = 11;
+
// These are the job modes the connector may want to know about.
// For a once-only job, it is essential that documents that are processed by processDocuments() always queue up their child links,
// This is not true for continuous jobs, which never delete unreachable links because they never terminate.
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/JobDeleteRecord.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/JobDeleteRecord.java
new file mode 100644
index 0000000..0a8278a
--- /dev/null
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/JobDeleteRecord.java
@@ -0,0 +1,35 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.interfaces;
+
+
+/** This class is a paper object which contains a job ID.
+*/
+public class JobDeleteRecord extends JobRecord
+{
+ public static final String _rcsid = "@(#)$Id$";
+
+ /** Constructor.
+ */
+ public JobDeleteRecord(Long jobID)
+ {
+ super(jobID);
+ }
+
+}
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/JobNotifyRecord.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/JobNotifyRecord.java
new file mode 100644
index 0000000..f807be4
--- /dev/null
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/JobNotifyRecord.java
@@ -0,0 +1,35 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.interfaces;
+
+
+/** This class is a paper object which contains a job ID.
+*/
+public class JobNotifyRecord extends JobRecord
+{
+ public static final String _rcsid = "@(#)$Id$";
+
+ /** Constructor.
+ */
+ public JobNotifyRecord(Long jobID)
+ {
+ super(jobID);
+ }
+
+}
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/JobRecord.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/JobRecord.java
new file mode 100644
index 0000000..28fdf36
--- /dev/null
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/JobRecord.java
@@ -0,0 +1,63 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.interfaces;
+
+
+/** This class is a paper object which contains a job ID.
+*/
+public class JobRecord
+{
+ public static final String _rcsid = "@(#)$Id$";
+
+ /** The job id. */
+ protected final Long jobID;
+ /** Whether this job was started or not */
+ protected boolean wasStarted = false;
+
+ /** Constructor.
+ */
+ public JobRecord(Long jobID)
+ {
+ this.jobID = jobID;
+ }
+
+ /** Get the job ID.
+ *@return the id.
+ */
+ public Long getJobID()
+ {
+ return jobID;
+ }
+
+ /** Note that the job was started.
+ */
+ public void noteStarted()
+ {
+ wasStarted = true;
+ }
+
+ /** Check whether job was started.
+ *@return true if started.
+ */
+ public boolean wasStarted()
+ {
+ return wasStarted;
+ }
+
+}
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/JobSeedingRecord.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/JobSeedingRecord.java
new file mode 100644
index 0000000..ee4f1a3
--- /dev/null
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/JobSeedingRecord.java
@@ -0,0 +1,49 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.interfaces;
+
+
+/** This class is a paper object which contains a job ID and a last job start time (0 if none).
+*/
+public class JobSeedingRecord extends JobRecord
+{
+ public static final String _rcsid = "@(#)$Id$";
+
+ /** The last synch time */
+ protected final long synchTime;
+ /** Whether this job was started or not */
+ protected boolean wasStarted = false;
+
+ /** Constructor.
+ */
+ public JobSeedingRecord(Long jobID, long synchTime)
+ {
+ super(jobID);
+ this.synchTime = synchTime;
+ }
+
+ /** Get the synch time.
+ *@return the time.
+ */
+ public long getSynchTime()
+ {
+ return synchTime;
+ }
+
+}
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/JobStartRecord.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/JobStartRecord.java
index be97e68..c2a9cca 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/JobStartRecord.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/JobStartRecord.java
@@ -19,33 +19,24 @@
package org.apache.manifoldcf.crawler.interfaces;
-/** This class is a paper object which contains a job ID and a last job start time (0 if none).
+/** This class is a paper object which contains a job ID and a last job start time.
*/
-public class JobStartRecord
+public class JobStartRecord extends JobRecord
{
public static final String _rcsid = "@(#)$Id: JobStartRecord.java 988245 2010-08-23 18:39:35Z kwright $";
- /** The job id. */
- protected Long jobID;
/** The last synch time */
- protected long synchTime;
- /** Whether this job was started or not */
- protected boolean wasStarted = false;
+ protected final long synchTime;
+ /** The requestMinimum flag */
+ protected final boolean requestMinimum;
/** Constructor.
*/
- public JobStartRecord(Long jobID, long synchTime)
+ public JobStartRecord(Long jobID, long synchTime, boolean requestMinimum)
{
- this.jobID = jobID;
+ super(jobID);
this.synchTime = synchTime;
- }
-
- /** Get the job ID.
- *@return the id.
- */
- public Long getJobID()
- {
- return jobID;
+ this.requestMinimum = requestMinimum;
}
/** Get the synch time.
@@ -56,19 +47,12 @@
return synchTime;
}
- /** Note that the job was started.
+ /** Get the requestMinimum flag.
+ *@return the flag.
*/
- public void noteStarted()
+ public boolean getRequestMinimum()
{
- wasStarted = true;
+ return requestMinimum;
}
-
- /** Check whether job was started.
- *@return true if started.
- */
- public boolean wasStarted()
- {
- return wasStarted;
- }
-
+
}
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/ScheduleRecord.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/ScheduleRecord.java
index 46ab1f3..9b89b07 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/ScheduleRecord.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/ScheduleRecord.java
@@ -31,15 +31,16 @@
public static final String _rcsid = "@(#)$Id: ScheduleRecord.java 988245 2010-08-23 18:39:35Z kwright $";
// Absolute job-triggering times
- protected EnumeratedValues dayOfWeek = null;
- protected EnumeratedValues monthOfYear = null;
- protected EnumeratedValues dayOfMonth = null;
- protected EnumeratedValues year = null;
- protected EnumeratedValues hourOfDay = null;
- protected EnumeratedValues minutesOfHour = null;
- protected String timezone = null;
- protected Long duration = null;
-
+ protected final EnumeratedValues dayOfWeek;
+ protected final EnumeratedValues monthOfYear;
+ protected final EnumeratedValues dayOfMonth;
+ protected final EnumeratedValues year;
+ protected final EnumeratedValues hourOfDay;
+ protected final EnumeratedValues minutesOfHour;
+ protected final String timezone;
+ protected final Long duration;
+ protected final boolean requestMinimum;
+
/** Constructor.
*@param dayOfWeek is the day-of-week enumeration.
*@param monthOfYear is the month-of-year enumeration.
@@ -57,7 +58,8 @@
EnumeratedValues hourOfDay,
EnumeratedValues minutesOfHour,
String timezone,
- Long duration)
+ Long duration,
+ boolean requestMinimum)
{
this.dayOfWeek = dayOfWeek;
this.monthOfYear = monthOfYear;
@@ -67,6 +69,7 @@
this.minutesOfHour = minutesOfHour;
this.timezone = timezone;
this.duration = duration;
+ this.requestMinimum = requestMinimum;
}
/** Get the day of week.
@@ -133,6 +136,12 @@
return duration;
}
-
+ /** Get whether the schedule record corresponds to a minimal request or not.
+ *@return true if minimal request.
+ */
+ public boolean getRequestMinimum()
+ {
+ return requestMinimum;
+ }
}
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
index 5009aa3..77670fb 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
@@ -98,7 +98,7 @@
throws java.io.IOException, ManifoldCFException
{
// Write a version indicator
- ManifoldCF.writeDword(os,2);
+ ManifoldCF.writeDword(os,3);
// Get the job list
IJobDescription[] list = getAllJobs();
// Write the number of authorities
@@ -136,6 +136,7 @@
writeEnumeratedValues(os,sr.getMinutesOfHour());
ManifoldCF.writeString(os,sr.getTimezone());
ManifoldCF.writeLong(os,sr.getDuration());
+ ManifoldCF.writeByte(os,sr.getRequestMinimum()?1:0);
}
// Write hop count filters
@@ -174,7 +175,7 @@
throws java.io.IOException, ManifoldCFException
{
int version = ManifoldCF.readDword(is);
- if (version != 2)
+ if (version != 2 && version != 3)
throw new java.io.IOException("Unknown job configuration version: "+Integer.toString(version));
int count = ManifoldCF.readDword(is);
int i = 0;
@@ -208,9 +209,14 @@
EnumeratedValues minutesOfHour = readEnumeratedValues(is);
String timezone = ManifoldCF.readString(is);
Long duration = ManifoldCF.readLong(is);
+ boolean requestMinimum;
+ if (version >= 3)
+ requestMinimum = (ManifoldCF.readByte(is) != 0);
+ else
+ requestMinimum = false;
ScheduleRecord sr = new ScheduleRecord(dayOfWeek, monthOfYear, dayOfMonth, year,
- hourOfDay, minutesOfHour, timezone, duration);
+ hourOfDay, minutesOfHour, timezone, duration, requestMinimum);
job.addScheduleRecord(sr);
j++;
}
@@ -1343,11 +1349,12 @@
sb.append("SELECT t0.").append(jobQueue.docHashField).append(" FROM ").append(jobQueue.getTableName()).append(" t0 WHERE ")
.append(database.buildConjunctionClause(list,new ClauseDescription[]{
new MultiClause("t0."+jobQueue.docHashField,docList)})).append(" AND ")
- .append("t0.").append(jobQueue.statusField).append(" IN (?,?,?,?) AND ");
+ .append("t0.").append(jobQueue.statusField).append(" IN (?,?,?,?,?) AND ");
list.add(jobQueue.statusToString(jobQueue.STATUS_PURGATORY));
list.add(jobQueue.statusToString(jobQueue.STATUS_PENDINGPURGATORY));
list.add(jobQueue.statusToString(jobQueue.STATUS_COMPLETE));
+ list.add(jobQueue.statusToString(jobQueue.STATUS_UNCHANGED));
list.add(jobQueue.statusToString(jobQueue.STATUS_ELIGIBLEFORDELETE));
sb.append("EXISTS(SELECT 'x' FROM ").append(jobs.getTableName()).append(" t1 WHERE ")
@@ -1424,6 +1431,7 @@
sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{
new MultiClause(jobQueue.statusField,new Object[]{
jobQueue.statusToString(JobQueue.STATUS_COMPLETE),
+ jobQueue.statusToString(JobQueue.STATUS_UNCHANGED),
jobQueue.statusToString(JobQueue.STATUS_PURGATORY)}),
new UnitaryClause(jobQueue.prioritySetField,"<",new Long(currentTime))})).append(" ");
@@ -1490,6 +1498,7 @@
.append(database.buildConjunctionClause(list,new ClauseDescription[]{
new MultiClause("t1."+jobs.statusField,new Object[]{
Jobs.statusToString(Jobs.STATUS_STARTINGUP),
+ Jobs.statusToString(Jobs.STATUS_STARTINGUPMINIMAL),
Jobs.statusToString(Jobs.STATUS_ACTIVE),
Jobs.statusToString(Jobs.STATUS_ACTIVESEEDING),
Jobs.statusToString(Jobs.STATUS_ACTIVE_UNINSTALLED),
@@ -4745,13 +4754,14 @@
// We go through *all* the schedule records. The one that matches that has the latest
// end time is the one we take.
- int l = 0;
Long matchTime = null;
Long duration = null;
- while (l < thisSchedule.length)
+ boolean requestMinimum = false;
+
+ for (int l = 0; l < thisSchedule.length; l++)
{
long trialStartInterval = startInterval;
- ScheduleRecord sr = thisSchedule[l++];
+ ScheduleRecord sr = thisSchedule[l];
Long thisDuration = sr.getDuration();
if (startMethod == IJobDescription.START_WINDOWINSIDE &&
thisDuration != null)
@@ -4787,10 +4797,11 @@
if (matchTime == null || thisDuration == null ||
(duration != null && thisMatchTime.longValue() + thisDuration.longValue() >
- matchTime.longValue() + duration.longValue()))
+ matchTime.longValue() + duration.longValue()))
{
matchTime = thisMatchTime;
duration = thisDuration;
+ requestMinimum = sr.getRequestMinimum();
}
}
@@ -4823,7 +4834,7 @@
// If job was formerly "inactive", do the full startup.
// Start this job! but with no end time.
// This does not get logged because the startup thread does the logging.
- jobs.startJob(jobID,windowEnd);
+ jobs.startJob(jobID,windowEnd,requestMinimum);
jobQueue.clearFailTimes(jobID);
if (Logging.jobs.isDebugEnabled())
{
@@ -5256,6 +5267,18 @@
public void manualStart(Long jobID)
throws ManifoldCFException
{
+ manualStart(jobID,false);
+ }
+
+ /** Manually start a job. The specified job will be run REGARDLESS of the timed windows, and
+ * will not cease until complete. If the job is already running, this operation will assure that
+ * the job does not pause when its window ends. The job can be manually paused, or manually aborted.
+ *@param jobID is the ID of the job to start.
+ *@param requestMinimum is true if a minimal job run is requested.
+ */
+ public void manualStart(Long jobID, boolean requestMinimum)
+ throws ManifoldCFException
+ {
database.beginTransaction();
try
{
@@ -5284,8 +5307,9 @@
Logging.jobs.debug("Manually starting job "+jobID);
}
// Start this job! but with no end time.
- jobs.startJob(jobID,null);
+ jobs.startJob(jobID,null,requestMinimum);
jobQueue.clearFailTimes(jobID);
+
if (Logging.jobs.isDebugEnabled())
{
Logging.jobs.debug("Manual job start signal for job "+jobID+" successfully sent");
@@ -5355,12 +5379,71 @@
jobQueue.prepareDeleteScan(jobID);
}
+ /** Prepare a job to be run.
+ * This method is called regardless of the details of the job; what differs is only the flags that are passed in.
+ * The code inside will determine the appropriate procedures.
+ * (This method replaces prepareFullScan() and prepareIncrementalScan(). )
+ *@param jobID is the job id.
+ *@param legalLinkTypes are the link types allowed for the job.
+ *@param hopcountMethod describes how to handle deletions for hopcount purposes.
+ *@param connectorModel is the model used by the connector for the job.
+ *@param continuousJob is true if the job is a continuous one.
+ *@param fromBeginningOfTime is true if the job is running starting from time 0.
+ *@param requestMinimum is true if the minimal amount of work is requested for the job run.
+ */
+ public void prepareJobScan(Long jobID, String[] legalLinkTypes, int hopcountMethod,
+ int connectorModel, boolean continuousJob, boolean fromBeginningOfTime,
+ boolean requestMinimum)
+ throws ManifoldCFException
+ {
+ // (1) If the connector has MODEL_ADD_CHANGE_DELETE, then
+ // we let the connector run the show; there's no purge phase, and therefore the
+ // documents are left in a COMPLETED state if they don't show up in the list
+ // of seeds that require the attention of the connector.
+ //
+ // (2) If the connector has MODEL_ALL, then it's a full crawl no matter what, so
+ // we do a full scan initialization.
+ //
+ // (3) If the connector has some other model, we look at the start time. A start
+ // time of 0 implies a full scan, while any other start time implies an incremental
+ // scan.
+
+ // Complete connector model is told everything, so no delete phase.
+ if (connectorModel == IRepositoryConnector.MODEL_ADD_CHANGE_DELETE)
+ return;
+
+ // If the connector model is complete via chaining, then we just need to make
+ // sure discovery works to queue the changes.
+ if (connectorModel == IRepositoryConnector.MODEL_CHAINED_ADD_CHANGE_DELETE)
+ {
+ jobQueue.preparePartialScan(jobID);
+ return;
+ }
+
+ // Similarly, minimal crawl attempts no delete phase unless the connector explicitly forbids it, or unless
+ // the job criteria have changed.
+ if (requestMinimum && connectorModel != IRepositoryConnector.MODEL_ALL && !fromBeginningOfTime)
+ {
+ // If it is a chained model, do the partial prep.
+ if (connectorModel == IRepositoryConnector.MODEL_CHAINED_ADD ||
+ connectorModel == IRepositoryConnector.MODEL_CHAINED_ADD_CHANGE)
+ jobQueue.preparePartialScan(jobID);
+ return;
+ }
+
+ if (!continuousJob && connectorModel != IRepositoryConnector.MODEL_PARTIAL &&
+ (connectorModel == IRepositoryConnector.MODEL_ALL || fromBeginningOfTime))
+ prepareFullScan(jobID,legalLinkTypes,hopcountMethod);
+ else
+ jobQueue.prepareIncrementalScan(jobID);
+ }
+
/** Prepare for a full scan.
*@param jobID is the job id.
*@param legalLinkTypes are the link types allowed for the job.
*@param hopcountMethod describes how to handle deletions for hopcount purposes.
*/
- public void prepareFullScan(Long jobID, String[] legalLinkTypes, int hopcountMethod)
+ protected void prepareFullScan(Long jobID, String[] legalLinkTypes, int hopcountMethod)
throws ManifoldCFException
{
while (true)
@@ -5370,7 +5453,7 @@
database.beginTransaction(database.TRANSACTION_SERIALIZED);
try
{
- // Delete all documents that match a given criteria
+ // Delete the documents we have never fetched, including any hopcount records we've calculated.
if (legalLinkTypes.length > 0)
{
ArrayList list = new ArrayList();
@@ -5413,17 +5496,6 @@
}
}
- /** Prepare for an incremental scan.
- *@param jobID is the job id.
- *@param legalLinkTypes are the link types allowed for the job.
- *@param hopcountMethod describes how to handle deletions for hopcount purposes.
- */
- public void prepareIncrementalScan(Long jobID, String[] legalLinkTypes, int hopcountMethod)
- throws ManifoldCFException
- {
- jobQueue.prepareIncrementalScan(jobID);
- }
-
/** Manually abort a running job. The job will be permanently stopped, and will not run again until
* automatically started based on schedule, or manually started.
*@param jobID is the job to abort.
@@ -5479,8 +5551,9 @@
/** Manually restart a running job. The job will be stopped and restarted. Any schedule affinity will be lost,
* until the job finishes on its own.
*@param jobID is the job to abort.
+ *@param requestMinimum is true if a minimal job run is requested.
*/
- public void manualAbortRestart(Long jobID)
+ public void manualAbortRestart(Long jobID, boolean requestMinimum)
throws ManifoldCFException
{
if (Logging.jobs.isDebugEnabled())
@@ -5493,7 +5566,7 @@
database.beginTransaction();
try
{
- jobs.abortRestartJob(jobID);
+ jobs.abortRestartJob(jobID,requestMinimum);
database.performCommit();
break;
}
@@ -5526,6 +5599,16 @@
}
}
+ /** Manually restart a running job. The job will be stopped and restarted. Any schedule affinity will be lost,
+ * until the job finishes on its own.
+ *@param jobID is the job to abort.
+ */
+ public void manualAbortRestart(Long jobID)
+ throws ManifoldCFException
+ {
+ manualAbortRestart(jobID,false);
+ }
+
/** Abort a running job due to a fatal error condition.
*@param jobID is the job to abort.
*@param errorText is the error text.
@@ -5687,7 +5770,7 @@
*@return jobs that are active and are running in adaptive mode. These will be seeded
* based on what the connector says should be added to the queue.
*/
- public JobStartRecord[] getJobsReadyForSeeding(long currentTime)
+ public JobSeedingRecord[] getJobsReadyForSeeding(long currentTime)
throws ManifoldCFException
{
while (true)
@@ -5715,7 +5798,7 @@
IResultSet set = database.performQuery(sb.toString(),list,null,null);
// Update them all
- JobStartRecord[] rval = new JobStartRecord[set.getRowCount()];
+ JobSeedingRecord[] rval = new JobSeedingRecord[set.getRowCount()];
int i = 0;
while (i < rval.length)
{
@@ -5741,7 +5824,7 @@
Logging.jobs.debug("Marked job "+jobID+" for seeding");
}
- rval[i] = new JobStartRecord(jobID,synchTime);
+ rval[i] = new JobSeedingRecord(jobID,synchTime);
i++;
}
database.performCommit();
@@ -5775,7 +5858,7 @@
/** Get the list of jobs that are ready for deletion.
*@return jobs that were in the "readyfordelete" state.
*/
- public JobStartRecord[] getJobsReadyForDelete()
+ public JobDeleteRecord[] getJobsReadyForDelete()
throws ManifoldCFException
{
while (true)
@@ -5795,7 +5878,7 @@
IResultSet set = database.performQuery(sb.toString(),list,null,null);
// Update them all
- JobStartRecord[] rval = new JobStartRecord[set.getRowCount()];
+ JobDeleteRecord[] rval = new JobDeleteRecord[set.getRowCount()];
int i = 0;
while (i < rval.length)
{
@@ -5809,7 +5892,7 @@
Logging.jobs.debug("Marked job "+jobID+" for delete startup");
}
- rval[i] = new JobStartRecord(jobID,0L);
+ rval[i] = new JobDeleteRecord(jobID);
i++;
}
database.performCommit();
@@ -5857,10 +5940,13 @@
ArrayList list = new ArrayList();
sb.append(jobs.idField).append(",")
- .append(jobs.lastCheckTimeField)
+ .append(jobs.lastCheckTimeField).append(",")
+ .append(jobs.statusField)
.append(" FROM ").append(jobs.getTableName()).append(" WHERE ")
.append(database.buildConjunctionClause(list,new ClauseDescription[]{
- new UnitaryClause(jobs.statusField,jobs.statusToString(jobs.STATUS_READYFORSTARTUP))}))
+ new MultiClause(jobs.statusField,new Object[]{
+ jobs.statusToString(jobs.STATUS_READYFORSTARTUP),
+ jobs.statusToString(jobs.STATUS_READYFORSTARTUPMINIMAL)})}))
.append(" FOR UPDATE");
IResultSet set = database.performQuery(sb.toString(),list,null,null);
@@ -5872,18 +5958,22 @@
IResultRow row = set.getRow(i);
Long jobID = (Long)row.getValue(jobs.idField);
Long x = (Long)row.getValue(jobs.lastCheckTimeField);
+ int status = jobs.stringToStatus((String)row.getValue(jobs.statusField));
+
+ boolean requestMinimum = (status == jobs.STATUS_READYFORSTARTUPMINIMAL);
+
long synchTime = 0;
if (x != null)
synchTime = x.longValue();
// Mark status of job as "starting"
- jobs.writeStatus(jobID,jobs.STATUS_STARTINGUP);
+ jobs.writeStatus(jobID,requestMinimum?jobs.STATUS_STARTINGUPMINIMAL:jobs.STATUS_STARTINGUP);
if (Logging.jobs.isDebugEnabled())
{
Logging.jobs.debug("Marked job "+jobID+" for startup");
}
- rval[i] = new JobStartRecord(jobID,synchTime);
+ rval[i] = new JobStartRecord(jobID,synchTime,requestMinimum);
i++;
}
database.performCommit();
@@ -6151,7 +6241,15 @@
// Set the state of the job back to "ReadyForStartup"
jobs.writeStatus(jobID,jobs.STATUS_READYFORSTARTUP);
break;
+ case Jobs.STATUS_STARTINGUPMINIMAL:
+ if (Logging.jobs.isDebugEnabled())
+ Logging.jobs.debug("Setting job "+jobID+" back to 'ReadyForStartupMinimal' state");
+
+ // Set the state of the job back to "ReadyForStartupMinimal"
+ jobs.writeStatus(jobID,jobs.STATUS_READYFORSTARTUPMINIMAL);
+ break;
case Jobs.STATUS_ABORTINGSTARTINGUP:
+ case Jobs.STATUS_ABORTINGSTARTINGUPMINIMAL:
if (Logging.jobs.isDebugEnabled())
Logging.jobs.debug("Setting job "+jobID+" to 'Aborting' state");
jobs.writeStatus(jobID,jobs.STATUS_ABORTING);
@@ -6161,10 +6259,17 @@
Logging.jobs.debug("Setting job "+jobID+" to 'AbortingForRestart' state");
jobs.writeStatus(jobID,jobs.STATUS_ABORTINGFORRESTART);
break;
+ case Jobs.STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL:
+ if (Logging.jobs.isDebugEnabled())
+ Logging.jobs.debug("Setting job "+jobID+" to 'AbortingForRestartMinimal' state");
+ jobs.writeStatus(jobID,jobs.STATUS_ABORTINGFORRESTARTMINIMAL);
+ break;
case Jobs.STATUS_READYFORSTARTUP:
+ case Jobs.STATUS_READYFORSTARTUPMINIMAL:
case Jobs.STATUS_ABORTING:
case Jobs.STATUS_ABORTINGFORRESTART:
+ case Jobs.STATUS_ABORTINGFORRESTARTMINIMAL:
// ok
break;
default:
@@ -6291,8 +6396,17 @@
jobs.writeStatus(jobID,jobs.STATUS_ABORTINGFORRESTART);
break;
+ case Jobs.STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL:
+ if (Logging.jobs.isDebugEnabled())
+ Logging.jobs.debug("Setting job "+jobID+" back to 'AbortingForRestartMinimal' state");
+
+ // Set the state of the job back to "Active"
+ jobs.writeStatus(jobID,jobs.STATUS_ABORTINGFORRESTARTMINIMAL);
+ break;
+
case Jobs.STATUS_ABORTING:
case Jobs.STATUS_ABORTINGFORRESTART:
+ case Jobs.STATUS_ABORTINGFORRESTARTMINIMAL:
case Jobs.STATUS_ACTIVE:
case Jobs.STATUS_ACTIVE_UNINSTALLED:
case Jobs.STATUS_ACTIVE_NOOUTPUT:
@@ -6544,7 +6658,7 @@
/** Find the list of jobs that need to have their connectors notified of job completion.
*@return the ID's of jobs that need their output connectors notified in order to become inactive.
*/
- public JobStartRecord[] getJobsReadyForInactivity()
+ public JobNotifyRecord[] getJobsReadyForInactivity()
throws ManifoldCFException
{
while (true)
@@ -6564,7 +6678,7 @@
IResultSet set = database.performQuery(sb.toString(),list,null,null);
// Return them all
- JobStartRecord[] rval = new JobStartRecord[set.getRowCount()];
+ JobNotifyRecord[] rval = new JobNotifyRecord[set.getRowCount()];
int i = 0;
while (i < rval.length)
{
@@ -6576,7 +6690,7 @@
{
Logging.jobs.debug("Found job "+jobID+" in need of notification");
}
- rval[i++] = new JobStartRecord(jobID,0L);
+ rval[i++] = new JobNotifyRecord(jobID);
}
database.performCommit();
return rval;
@@ -6672,6 +6786,7 @@
new MultiClause(jobs.statusField,new Object[]{
jobs.statusToString(jobs.STATUS_ABORTING),
jobs.statusToString(jobs.STATUS_ABORTINGFORRESTART),
+ jobs.statusToString(jobs.STATUS_ABORTINGFORRESTARTMINIMAL),
jobs.statusToString(jobs.STATUS_PAUSING),
jobs.statusToString(jobs.STATUS_PAUSINGSEEDING),
jobs.statusToString(jobs.STATUS_ACTIVEWAITING),
@@ -7019,6 +7134,7 @@
.append(database.buildConjunctionClause(list,new ClauseDescription[]{
new MultiClause(JobQueue.statusField,new Object[]{
JobQueue.statusToString(JobQueue.STATUS_COMPLETE),
+ JobQueue.statusToString(JobQueue.STATUS_UNCHANGED),
JobQueue.statusToString(JobQueue.STATUS_PURGATORY),
JobQueue.statusToString(JobQueue.STATUS_ACTIVEPURGATORY),
JobQueue.statusToString(JobQueue.STATUS_ACTIVENEEDRESCANPURGATORY),
@@ -7131,11 +7247,15 @@
case Jobs.STATUS_ABORTING:
case Jobs.STATUS_ABORTINGSEEDING:
case Jobs.STATUS_ABORTINGSTARTINGUP:
+ case Jobs.STATUS_ABORTINGSTARTINGUPMINIMAL:
rstatus = JobStatus.JOBSTATUS_ABORTING;
break;
case Jobs.STATUS_ABORTINGFORRESTART:
+ case Jobs.STATUS_ABORTINGFORRESTARTMINIMAL:
case Jobs.STATUS_ABORTINGFORRESTARTSEEDING:
+ case Jobs.STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL:
case Jobs.STATUS_ABORTINGSTARTINGUPFORRESTART:
+ case Jobs.STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL:
rstatus = JobStatus.JOBSTATUS_RESTARTING;
break;
case Jobs.STATUS_PAUSING:
@@ -7163,7 +7283,9 @@
rstatus = JobStatus.JOBSTATUS_PAUSED;
break;
case Jobs.STATUS_STARTINGUP:
+ case Jobs.STATUS_STARTINGUPMINIMAL:
case Jobs.STATUS_READYFORSTARTUP:
+ case Jobs.STATUS_READYFORSTARTUPMINIMAL:
rstatus = JobStatus.JOBSTATUS_STARTING;
break;
case Jobs.STATUS_DELETESTARTINGUP:
@@ -7221,6 +7343,7 @@
.append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Processed'")
.append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Processed'")
.append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Processed'")
+ .append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Processed'")
.append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Being removed'")
.append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Being removed'")
.append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Being removed'")
@@ -7229,7 +7352,7 @@
.append(" END AS state,")
.append("CASE")
.append(" WHEN ")
- .append("t0.").append(jobQueue.statusField).append(" IN (?,?)")
+ .append("t0.").append(jobQueue.statusField).append(" IN (?,?,?)")
.append(" THEN 'Inactive'")
.append(" WHEN ")
.append("t0.").append(jobQueue.statusField).append(" IN (?,?)")
@@ -7289,6 +7412,7 @@
list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY));
list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY));
list.add(jobQueue.statusToString(jobQueue.STATUS_COMPLETE));
+ list.add(jobQueue.statusToString(jobQueue.STATUS_UNCHANGED));
list.add(jobQueue.statusToString(jobQueue.STATUS_PURGATORY));
list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED));
list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED));
@@ -7296,6 +7420,7 @@
list.add(jobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVED));
list.add(jobQueue.statusToString(jobQueue.STATUS_COMPLETE));
+ list.add(jobQueue.statusToString(jobQueue.STATUS_UNCHANGED));
list.add(jobQueue.statusToString(jobQueue.STATUS_PURGATORY));
list.add(jobQueue.statusToString(jobQueue.STATUS_PENDING));
@@ -7378,7 +7503,7 @@
sb.append(" AS idbucket,")
.append("CASE")
.append(" WHEN ")
- .append(jobQueue.statusField).append(" IN (?,?)")
+ .append(jobQueue.statusField).append(" IN (?,?,?)")
.append(" THEN 1 ELSE 0")
.append(" END")
.append(" AS inactive,")
@@ -7450,6 +7575,7 @@
sb.append(" FROM ").append(jobQueue.getTableName());
list.add(jobQueue.statusToString(jobQueue.STATUS_COMPLETE));
+ list.add(jobQueue.statusToString(jobQueue.STATUS_UNCHANGED));
list.add(jobQueue.statusToString(jobQueue.STATUS_PURGATORY));
list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVE));
@@ -7576,6 +7702,7 @@
jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED),
jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED),
jobQueue.statusToString(jobQueue.STATUS_COMPLETE),
+ jobQueue.statusToString(jobQueue.STATUS_UNCHANGED),
jobQueue.statusToString(jobQueue.STATUS_PURGATORY)})}));
break;
case DOCSTATE_OUTOFSCOPE:
@@ -7602,6 +7729,7 @@
sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{
new MultiClause(fieldPrefix+jobQueue.statusField,new Object[]{
jobQueue.statusToString(jobQueue.STATUS_COMPLETE),
+ jobQueue.statusToString(jobQueue.STATUS_UNCHANGED),
jobQueue.statusToString(jobQueue.STATUS_PURGATORY)})}));
break;
case DOCSTATUS_PROCESSING:
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
index 2f32bc3..60f0f07 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
@@ -60,15 +60,17 @@
public final static int STATUS_PENDING = 0;
public final static int STATUS_ACTIVE = 1;
public final static int STATUS_COMPLETE = 2;
- public final static int STATUS_PENDINGPURGATORY = 3;
- public final static int STATUS_ACTIVEPURGATORY = 4;
- public final static int STATUS_PURGATORY = 5;
- public final static int STATUS_BEINGDELETED = 6;
- public final static int STATUS_ACTIVENEEDRESCAN = 7;
- public final static int STATUS_ACTIVENEEDRESCANPURGATORY = 8;
- public final static int STATUS_BEINGCLEANED = 9;
- public final static int STATUS_ELIGIBLEFORDELETE = 10;
- public final static int STATUS_HOPCOUNTREMOVED = 11;
+ public final static int STATUS_UNCHANGED = 3;
+ public final static int STATUS_PENDINGPURGATORY = 4;
+ public final static int STATUS_ACTIVEPURGATORY = 5;
+ public final static int STATUS_PURGATORY = 6;
+ public final static int STATUS_BEINGDELETED = 7;
+ public final static int STATUS_ACTIVENEEDRESCAN = 8;
+ public final static int STATUS_ACTIVENEEDRESCANPURGATORY = 9;
+ public final static int STATUS_BEINGCLEANED = 10;
+ public final static int STATUS_ELIGIBLEFORDELETE = 11;
+ public final static int STATUS_HOPCOUNTREMOVED = 12;
+
// Action values
public final static int ACTION_RESCAN = 0;
public final static int ACTION_REMOVE = 1;
@@ -121,6 +123,7 @@
statusMap.put("P",new Integer(STATUS_PENDING));
statusMap.put("A",new Integer(STATUS_ACTIVE));
statusMap.put("C",new Integer(STATUS_COMPLETE));
+ statusMap.put("U",new Integer(STATUS_UNCHANGED));
statusMap.put("G",new Integer(STATUS_PENDINGPURGATORY));
statusMap.put("F",new Integer(STATUS_ACTIVEPURGATORY));
statusMap.put("Z",new Integer(STATUS_PURGATORY));
@@ -533,6 +536,7 @@
new MultiClause(statusField,new Object[]{
statusToString(STATUS_PENDINGPURGATORY),
statusToString(STATUS_COMPLETE),
+ statusToString(STATUS_UNCHANGED),
statusToString(STATUS_PURGATORY)})});
performUpdate(map,"WHERE "+query,list,null);
@@ -583,6 +587,7 @@
new UnitaryClause(jobIDField,jobID),
new MultiClause(statusField,new Object[]{
statusToString(STATUS_PENDINGPURGATORY),
+ statusToString(STATUS_UNCHANGED),
statusToString(STATUS_COMPLETE)})});
performUpdate(map,"WHERE "+query,list,null);
@@ -592,6 +597,34 @@
unconditionallyAnalyzeTables();
}
+ /** Prepare for a "partial" job. This is called ONLY when the job is inactive.
+ *
+ * This method maps all COMPLETE entries to UNCHANGED. The purpose is to
+ * allow discovery to find the documents that need to be processed. If they were
+ * marked as COMPLETE that would stop them from being queued.
+ *@param jobID is the job identifier.
+ */
+ public void preparePartialScan(Long jobID)
+ throws ManifoldCFException
+ {
+ // Map COMPLETE to UNCHANGED.
+ HashMap map = new HashMap();
+ map.put(statusField,statusToString(STATUS_UNCHANGED));
+ // Do not reset priorities here! They should all be blank at this point.
+ map.put(checkTimeField,new Long(0L));
+ map.put(checkActionField,actionToString(ACTION_RESCAN));
+ map.put(failTimeField,null);
+ map.put(failCountField,null);
+ ArrayList list = new ArrayList();
+ String query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(jobIDField,jobID),
+ new UnitaryClause(statusField,statusToString(STATUS_COMPLETE))});
+ performUpdate(map,"WHERE "+query,list,null);
+ noteModifications(0,1,0);
+ // Do an analyze, otherwise our plans are going to be crap right off the bat
+ unconditionallyAnalyzeTables();
+ }
+
/** Prepare for an "incremental" job. This is called ONLY when the job is inactive;
* that is, there should be no ACTIVE or ACTIVEPURGATORY entries at all.
*
@@ -603,7 +636,7 @@
public void prepareIncrementalScan(Long jobID)
throws ManifoldCFException
{
- // Delete PENDING and ACTIVE entries
+ // Map COMPLETE to PENDINGPURGATORY.
HashMap map = new HashMap();
map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
// Do not reset priorities here! They should all be blank at this point.
@@ -614,7 +647,9 @@
ArrayList list = new ArrayList();
String query = buildConjunctionClause(list,new ClauseDescription[]{
new UnitaryClause(jobIDField,jobID),
- new UnitaryClause(statusField,statusToString(STATUS_COMPLETE))});
+ new MultiClause(statusField,new Object[]{
+ statusToString(STATUS_COMPLETE),
+ statusToString(STATUS_UNCHANGED)})});
performUpdate(map,"WHERE "+query,list,null);
noteModifications(0,1,0);
// Do an analyze, otherwise our plans are going to be crap right off the bat
@@ -1001,6 +1036,7 @@
break;
case STATUS_COMPLETE:
+ case STATUS_UNCHANGED:
case STATUS_PURGATORY:
// Set the status and time both
map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
@@ -1278,6 +1314,7 @@
switch (currentStatus)
{
case STATUS_PURGATORY:
+ case STATUS_UNCHANGED:
// Set the status and time both
map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
map.put(checkTimeField,new Long(desiredExecuteTime));
@@ -1506,6 +1543,8 @@
return "A";
case STATUS_COMPLETE:
return "C";
+ case STATUS_UNCHANGED:
+ return "U";
case STATUS_PENDINGPURGATORY:
return "G";
case STATUS_ACTIVEPURGATORY:
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
index f11e84b..418017f 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
@@ -92,17 +92,23 @@
public static final int STATUS_RESUMINGSEEDING = 17; // In the process of resuming a paused or waited job, seeding process active too; will enter STATUS_ACTIVESEEDING when done.
public static final int STATUS_ABORTING = 18; // Aborting (not yet aborted because documents still being processed)
public static final int STATUS_STARTINGUP = 19; // Loading the queue (will go into ACTIVE if successful, or INACTIVE if not)
- public static final int STATUS_ABORTINGSTARTINGUP = 20; // Will abort once the queue loading is complete
- public static final int STATUS_READYFORSTARTUP = 21; // Job is marked for startup; startup thread has not taken it yet.
- public static final int STATUS_READYFORDELETE = 22; // Job is marked for delete; delete thread has not taken it yet.
- public static final int STATUS_ABORTINGSEEDING = 23; // Same as aborting, but seeding process is currently active also.
- public static final int STATUS_ABORTINGFORRESTART = 24; // Same as aborting, except after abort is complete startup will happen.
- public static final int STATUS_ABORTINGFORRESTARTSEEDING = 25; // Seeding version of aborting for restart
- public static final int STATUS_ABORTINGSTARTINGUPFORRESTART = 26; // Starting up version of aborting for restart
- public static final int STATUS_READYFORNOTIFY = 27; // Job is ready to be notified of completion
- public static final int STATUS_NOTIFYINGOFCOMPLETION = 28; // Notifying connector of terminating job (either aborted, or finished)
- public static final int STATUS_DELETING = 29; // The job is deleting.
- public static final int STATUS_DELETESTARTINGUP = 30; // The delete is starting up.
+ public static final int STATUS_STARTINGUPMINIMAL = 20; // Loading the queue for minimal job run (will go into ACTIVE if successful, or INACTIVE if not)
+ public static final int STATUS_ABORTINGSTARTINGUP = 21; // Will abort once the queue loading is complete
+ public static final int STATUS_ABORTINGSTARTINGUPMINIMAL = 22; // Will abort once the queue loading is complete
+ public static final int STATUS_READYFORSTARTUP = 23; // Job is marked for minimal startup; startup thread has not taken it yet.
+ public static final int STATUS_READYFORSTARTUPMINIMAL = 24; // Job is marked for startup; startup thread has not taken it yet.
+ public static final int STATUS_READYFORDELETE = 25; // Job is marked for delete; delete thread has not taken it yet.
+ public static final int STATUS_ABORTINGSEEDING = 26; // Same as aborting, but seeding process is currently active also.
+ public static final int STATUS_ABORTINGFORRESTART = 27; // Same as aborting, except after abort is complete startup will happen.
+ public static final int STATUS_ABORTINGFORRESTARTMINIMAL = 28; // Same as aborting, except after abort is complete startup will happen.
+ public static final int STATUS_ABORTINGFORRESTARTSEEDING = 29; // Seeding version of aborting for restart
+ public static final int STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL = 30; // Seeding version of aborting for restart
+ public static final int STATUS_ABORTINGSTARTINGUPFORRESTART = 31; // Starting up version of aborting for restart
+ public static final int STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL = 32; // Starting up version of aborting for restart
+ public static final int STATUS_READYFORNOTIFY = 33; // Job is ready to be notified of completion
+ public static final int STATUS_NOTIFYINGOFCOMPLETION = 34; // Notifying connector of terminating job (either aborted, or finished)
+ public static final int STATUS_DELETING = 35; // The job is deleting.
+ public static final int STATUS_DELETESTARTINGUP = 36; // The delete is starting up.
// These statuses have to do with whether a job has an installed underlying connector or not.
// There are two reasons to have a special state here: (1) if the behavior of the crawler differs, or (2) if the
@@ -113,13 +119,13 @@
// But, since there is no indication in the jobs table of an uninstalled connector for such jobs, the code which starts
// jobs up (or otherwise would enter any state that has a corresponding special state) must check to see if the underlying
// connector exists before deciding what state to put the job into.
- public static final int STATUS_ACTIVE_UNINSTALLED = 31; // Active, but repository connector not installed
- public static final int STATUS_ACTIVESEEDING_UNINSTALLED = 32; // Active and seeding, but repository connector not installed
- public static final int STATUS_ACTIVE_NOOUTPUT = 33; // Active, but output connector not installed
- public static final int STATUS_ACTIVESEEDING_NOOUTPUT = 34; // Active and seeding, but output connector not installed
- public static final int STATUS_ACTIVE_NEITHER = 35; // Active, but neither repository connector nor output connector installed
- public static final int STATUS_ACTIVESEEDING_NEITHER = 36; // Active and seeding, but neither repository connector nor output connector installed
- public static final int STATUS_DELETING_NOOUTPUT = 37; // Job is being deleted but there's no output connector installed
+ public static final int STATUS_ACTIVE_UNINSTALLED = 37; // Active, but repository connector not installed
+ public static final int STATUS_ACTIVESEEDING_UNINSTALLED = 38; // Active and seeding, but repository connector not installed
+ public static final int STATUS_ACTIVE_NOOUTPUT = 39; // Active, but output connector not installed
+ public static final int STATUS_ACTIVESEEDING_NOOUTPUT = 40; // Active and seeding, but output connector not installed
+ public static final int STATUS_ACTIVE_NEITHER = 41; // Active, but neither repository connector nor output connector installed
+ public static final int STATUS_ACTIVESEEDING_NEITHER = 42; // Active and seeding, but neither repository connector nor output connector installed
+ public static final int STATUS_DELETING_NOOUTPUT = 43; // Job is being deleted but there's no output connector installed
// Type field values
public static final int TYPE_CONTINUOUS = IJobDescription.TYPE_CONTINUOUS;
@@ -194,13 +200,18 @@
statusMap.put("Z",new Integer(STATUS_PAUSEDWAIT));
statusMap.put("X",new Integer(STATUS_ABORTING));
statusMap.put("B",new Integer(STATUS_STARTINGUP));
+ statusMap.put("b",new Integer(STATUS_STARTINGUPMINIMAL));
statusMap.put("Q",new Integer(STATUS_ABORTINGSTARTINGUP));
+ statusMap.put("q",new Integer(STATUS_ABORTINGSTARTINGUPMINIMAL));
statusMap.put("C",new Integer(STATUS_READYFORSTARTUP));
+ statusMap.put("c",new Integer(STATUS_READYFORSTARTUPMINIMAL));
statusMap.put("E",new Integer(STATUS_READYFORDELETE));
statusMap.put("V",new Integer(STATUS_DELETESTARTINGUP));
statusMap.put("e",new Integer(STATUS_DELETING));
statusMap.put("Y",new Integer(STATUS_ABORTINGFORRESTART));
+ statusMap.put("M",new Integer(STATUS_ABORTINGFORRESTARTMINIMAL));
statusMap.put("T",new Integer(STATUS_ABORTINGSTARTINGUPFORRESTART));
+ statusMap.put("t",new Integer(STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL));
statusMap.put("a",new Integer(STATUS_ACTIVESEEDING));
statusMap.put("x",new Integer(STATUS_ABORTINGSEEDING));
@@ -208,6 +219,7 @@
statusMap.put("w",new Integer(STATUS_ACTIVEWAITSEEDING));
statusMap.put("z",new Integer(STATUS_PAUSEDWAITSEEDING));
statusMap.put("y",new Integer(STATUS_ABORTINGFORRESTARTSEEDING));
+ statusMap.put("m",new Integer(STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL));
statusMap.put("H",new Integer(STATUS_ACTIVEWAITING));
statusMap.put("h",new Integer(STATUS_ACTIVEWAITINGSEEDING));
@@ -842,6 +854,15 @@
map.put(statusField,statusToString(STATUS_READYFORSTARTUP));
performUpdate(map,"WHERE "+query,list,invKey);
+ // Starting up or aborting starting up goes back to just being ready
+ list.clear();
+ query = buildConjunctionClause(list,new ClauseDescription[]{
+ new MultiClause(statusField,new Object[]{
+ statusToString(STATUS_STARTINGUPMINIMAL),
+ statusToString(STATUS_ABORTINGSTARTINGUPMINIMAL)})});
+ map.put(statusField,statusToString(STATUS_READYFORSTARTUPMINIMAL));
+ performUpdate(map,"WHERE "+query,list,invKey);
+
// Aborting starting up for restart state goes to ABORTINGFORRESTART
list.clear();
query = buildConjunctionClause(list,new ClauseDescription[]{
@@ -849,6 +870,13 @@
map.put(statusField,statusToString(STATUS_ABORTINGFORRESTART));
performUpdate(map,"WHERE "+query,list,invKey);
+ // Aborting starting up for restart state goes to ABORTINGFORRESTART
+ list.clear();
+ query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(statusField,statusToString(STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL))});
+ map.put(statusField,statusToString(STATUS_ABORTINGFORRESTARTMINIMAL));
+ performUpdate(map,"WHERE "+query,list,invKey);
+
// All seeding values return to pre-seeding values
list.clear();
query = buildConjunctionClause(list,new ClauseDescription[]{
@@ -887,6 +915,11 @@
performUpdate(map,"WHERE "+query,list,invKey);
list.clear();
query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(statusField,statusToString(STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL))});
+ map.put(statusField,statusToString(STATUS_ABORTINGFORRESTARTMINIMAL));
+ performUpdate(map,"WHERE "+query,list,invKey);
+ list.clear();
+ query = buildConjunctionClause(list,new ClauseDescription[]{
new UnitaryClause(statusField,statusToString(STATUS_PAUSEDSEEDING))});
map.put(statusField,statusToString(STATUS_PAUSED));
performUpdate(map,"WHERE "+query,list,invKey);
@@ -1163,7 +1196,7 @@
int status = stringToStatus(statusValue);
// Any active state in the lifecycle will do: seeding, active, active_seeding
return (status == STATUS_ACTIVE || status == STATUS_ACTIVESEEDING ||
- status == STATUS_STARTINGUP);
+ status == STATUS_STARTINGUP || status == STATUS_STARTINGUPMINIMAL);
}
/** Reset delete startup worker thread status.
@@ -1209,6 +1242,7 @@
HashMap map = new HashMap();
String query;
+ list.clear();
query = buildConjunctionClause(list,new ClauseDescription[]{
new UnitaryClause(statusField,statusToString(STATUS_STARTINGUP))});
map.put(statusField,statusToString(STATUS_READYFORSTARTUP));
@@ -1216,7 +1250,15 @@
list.clear();
query = buildConjunctionClause(list,new ClauseDescription[]{
- new UnitaryClause(statusField,statusToString(STATUS_ABORTINGSTARTINGUP))});
+ new UnitaryClause(statusField,statusToString(STATUS_STARTINGUPMINIMAL))});
+ map.put(statusField,statusToString(STATUS_READYFORSTARTUPMINIMAL));
+ performUpdate(map,"WHERE "+query,list,new StringSet(getJobStatusKey()));
+
+ list.clear();
+ query = buildConjunctionClause(list,new ClauseDescription[]{
+ new MultiClause(statusField,new Object[]{
+ statusToString(STATUS_ABORTINGSTARTINGUP),
+ statusToString(STATUS_ABORTINGSTARTINGUPMINIMAL)})});
map.put(statusField,statusToString(STATUS_ABORTING));
performUpdate(map,"WHERE "+query,list,new StringSet(getJobStatusKey()));
@@ -1226,6 +1268,12 @@
map.put(statusField,statusToString(STATUS_ABORTINGFORRESTART));
performUpdate(map,"WHERE "+query,list,new StringSet(getJobStatusKey()));
+ list.clear();
+ query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(statusField,statusToString(STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL))});
+ map.put(statusField,statusToString(STATUS_ABORTINGFORRESTARTMINIMAL));
+ performUpdate(map,"WHERE "+query,list,new StringSet(getJobStatusKey()));
+
}
/** Reset as part of restoring seeding worker threads.
@@ -1278,6 +1326,11 @@
performUpdate(map,"WHERE "+query,list,invKey);
list.clear();
query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(statusField,statusToString(STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL))});
+ map.put(statusField,statusToString(STATUS_ABORTINGFORRESTARTMINIMAL));
+ performUpdate(map,"WHERE "+query,list,invKey);
+ list.clear();
+ query = buildConjunctionClause(list,new ClauseDescription[]{
new UnitaryClause(statusField,statusToString(STATUS_PAUSEDSEEDING))});
map.put(statusField,statusToString(STATUS_PAUSED));
performUpdate(map,"WHERE "+query,list,invKey);
@@ -1329,15 +1382,16 @@
* when the job enters the "active" state.)
*@param jobID is the job identifier.
*@param windowEnd is the window end time, if any
+ *@param requestMinimum is true if a minimal job run is requested
*/
- public void startJob(Long jobID, Long windowEnd)
+ public void startJob(Long jobID, Long windowEnd, boolean requestMinimum)
throws ManifoldCFException
{
ArrayList list = new ArrayList();
String query = buildConjunctionClause(list,new ClauseDescription[]{
new UnitaryClause(idField,jobID)});
HashMap map = new HashMap();
- map.put(statusField,statusToString(STATUS_READYFORSTARTUP));
+ map.put(statusField,statusToString(requestMinimum?STATUS_READYFORSTARTUPMINIMAL:STATUS_READYFORSTARTUP));
map.put(endTimeField,null);
// Make sure error is removed (from last time)
map.put(errorField,null);
@@ -1488,6 +1542,7 @@
switch (status)
{
case STATUS_STARTINGUP:
+ case STATUS_STARTINGUPMINIMAL:
if (connectionMgr.checkConnectorExists((String)row.getValue(connectionNameField)))
{
if (outputMgr.checkConnectorExists((String)row.getValue(outputNameField)))
@@ -1504,11 +1559,15 @@
}
break;
case STATUS_ABORTINGSTARTINGUP:
+ case STATUS_ABORTINGSTARTINGUPMINIMAL:
newStatus = STATUS_ABORTING;
break;
case STATUS_ABORTINGSTARTINGUPFORRESTART:
newStatus = STATUS_ABORTINGFORRESTART;
break;
+ case STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL:
+ newStatus = STATUS_ABORTINGFORRESTARTMINIMAL;
+ break;
default:
// Complain!
throw new ManifoldCFException("Unexpected job status encountered: "+Integer.toString(status));
@@ -1600,6 +1659,9 @@
case STATUS_ABORTINGFORRESTARTSEEDING:
newStatus = STATUS_ABORTINGFORRESTART;
break;
+ case STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL:
+ newStatus = STATUS_ABORTINGFORRESTARTMINIMAL;
+ break;
default:
throw new ManifoldCFException("Unexpected job status encountered: "+Integer.toString(status));
}
@@ -1677,7 +1739,8 @@
throw new ManifoldCFException("Job does not exist: "+jobID);
IResultRow row = set.getRow(0);
int status = stringToStatus(row.getValue(statusField).toString());
- if (status == STATUS_ABORTING || status == STATUS_ABORTINGSEEDING || status == STATUS_ABORTINGSTARTINGUP)
+ if (status == STATUS_ABORTING || status == STATUS_ABORTINGSEEDING ||
+ status == STATUS_ABORTINGSTARTINGUP || status == STATUS_ABORTINGSTARTINGUPMINIMAL)
return false;
int newStatus;
switch (status)
@@ -1686,7 +1749,12 @@
case STATUS_ABORTINGSTARTINGUPFORRESTART:
newStatus = STATUS_ABORTINGSTARTINGUP;
break;
+ case STATUS_STARTINGUPMINIMAL:
+ case STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL:
+ newStatus = STATUS_ABORTINGSTARTINGUPMINIMAL;
+ break;
case STATUS_READYFORSTARTUP:
+ case STATUS_READYFORSTARTUPMINIMAL:
case STATUS_ACTIVE:
case STATUS_ACTIVE_UNINSTALLED:
case STATUS_ACTIVE_NOOUTPUT:
@@ -1698,6 +1766,7 @@
case STATUS_PAUSED:
case STATUS_PAUSEDWAIT:
case STATUS_ABORTINGFORRESTART:
+ case STATUS_ABORTINGFORRESTARTMINIMAL:
newStatus = STATUS_ABORTING;
break;
case STATUS_ACTIVESEEDING:
@@ -1711,6 +1780,7 @@
case STATUS_PAUSEDSEEDING:
case STATUS_PAUSEDWAITSEEDING:
case STATUS_ABORTINGFORRESTARTSEEDING:
+ case STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL:
newStatus = STATUS_ABORTINGSEEDING;
break;
default:
@@ -1726,8 +1796,9 @@
/** Restart a job. Finish off what's currently happening, and then start the job up again.
*@param jobID is the job id.
+ *@param requestMinimum is true if the minimal job run is requested.
*/
- public void abortRestartJob(Long jobID)
+ public void abortRestartJob(Long jobID, boolean requestMinimum)
throws ManifoldCFException
{
// Get the current job status
@@ -1740,15 +1811,20 @@
throw new ManifoldCFException("Job does not exist: "+jobID);
IResultRow row = set.getRow(0);
int status = stringToStatus(row.getValue(statusField).toString());
- if (status == STATUS_ABORTINGFORRESTART || status == STATUS_ABORTINGFORRESTARTSEEDING || status == STATUS_ABORTINGSTARTINGUPFORRESTART)
+ if (status == STATUS_ABORTINGFORRESTART || status == STATUS_ABORTINGFORRESTARTSEEDING ||
+ status == STATUS_ABORTINGSTARTINGUPFORRESTART ||
+ status == STATUS_ABORTINGFORRESTARTMINIMAL || status == STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL ||
+ status == STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL)
return;
int newStatus;
switch (status)
{
case STATUS_STARTINGUP:
- newStatus = STATUS_ABORTINGSTARTINGUPFORRESTART;
+ case STATUS_STARTINGUPMINIMAL:
+ newStatus = requestMinimum?STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL:STATUS_ABORTINGSTARTINGUPFORRESTART;
break;
case STATUS_READYFORSTARTUP:
+ case STATUS_READYFORSTARTUPMINIMAL:
case STATUS_ACTIVE:
case STATUS_ACTIVE_UNINSTALLED:
case STATUS_ACTIVE_NOOUTPUT:
@@ -1759,7 +1835,7 @@
case STATUS_PAUSINGWAITING:
case STATUS_PAUSED:
case STATUS_PAUSEDWAIT:
- newStatus = STATUS_ABORTINGFORRESTART;
+ newStatus = requestMinimum?STATUS_ABORTINGFORRESTARTMINIMAL:STATUS_ABORTINGFORRESTART;
break;
case STATUS_ACTIVESEEDING:
case STATUS_ACTIVESEEDING_UNINSTALLED:
@@ -1771,7 +1847,7 @@
case STATUS_PAUSINGWAITINGSEEDING:
case STATUS_PAUSEDSEEDING:
case STATUS_PAUSEDWAITSEEDING:
- newStatus = STATUS_ABORTINGFORRESTARTSEEDING;
+ newStatus = requestMinimum?STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL:STATUS_ABORTINGFORRESTARTSEEDING;
break;
default:
throw new ManifoldCFException("Job "+jobID+" is not restartable");
@@ -2061,6 +2137,13 @@
map.put(errorField,null);
map.put(windowEndField,null);
break;
+ case STATUS_ABORTINGFORRESTARTMINIMAL:
+ map.put(statusField,statusToString(STATUS_READYFORSTARTUPMINIMAL));
+ map.put(endTimeField,null);
+ // Make sure error is removed (from last time)
+ map.put(errorField,null);
+ map.put(windowEndField,null);
+ break;
case STATUS_PAUSING:
map.put(statusField,statusToString(STATUS_PAUSED));
break;
@@ -2295,14 +2378,24 @@
return "X";
case STATUS_ABORTINGFORRESTART:
return "Y";
+ case STATUS_ABORTINGFORRESTARTMINIMAL:
+ return "M";
case STATUS_STARTINGUP:
return "B";
+ case STATUS_STARTINGUPMINIMAL:
+ return "b";
case STATUS_ABORTINGSTARTINGUP:
return "Q";
+ case STATUS_ABORTINGSTARTINGUPMINIMAL:
+ return "q";
case STATUS_ABORTINGSTARTINGUPFORRESTART:
return "T";
+ case STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL:
+ return "t";
case STATUS_READYFORSTARTUP:
return "C";
+ case STATUS_READYFORSTARTUPMINIMAL:
+ return "c";
case STATUS_READYFORDELETE:
return "E";
case STATUS_DELETESTARTINGUP:
@@ -2321,6 +2414,8 @@
return "z";
case STATUS_ABORTINGFORRESTARTSEEDING:
return "y";
+ case STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL:
+ return "m";
case STATUS_ACTIVE_UNINSTALLED:
return "R";
case STATUS_ACTIVESEEDING_UNINSTALLED:
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/ScheduleManager.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/ScheduleManager.java
index 42a665f..2f1f6df 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/ScheduleManager.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/ScheduleManager.java
@@ -41,6 +41,7 @@
* <tr><td>minutesofhour</td><td>VARCHAR(255)</td><td></td></tr>
* <tr><td>timezone</td><td>VARCHAR(32)</td><td></td></tr>
* <tr><td>windowlength</td><td>BIGINT</td><td></td></tr>
+* <tr><td>requestminimum</td><td>CHAR(1)</td><td></td></tr>
* </table>
* <br><br>
*
@@ -60,7 +61,7 @@
public final static String minutesOfHourField = "minutesofhour";
public final static String timezoneField = "timezone";
public final static String windowDurationField = "windowlength";
-
+ public final static String requestMinimumField = "requestminimum";
/** Constructor.
*@param threadContext is the thread context.
@@ -96,6 +97,7 @@
map.put(minutesOfHourField,new ColumnDescription("VARCHAR(255)",false,true,null,null,false));
map.put(timezoneField,new ColumnDescription("VARCHAR(32)",false,true,null,null,false));
map.put(windowDurationField,new ColumnDescription("BIGINT",false,true,null,null,false));
+ map.put(requestMinimumField,new ColumnDescription("CHAR(1)",false,true,null,null,false));
performCreate(map,null);
}
else
@@ -113,6 +115,14 @@
performAlter(null,null,list,null);
}
+
+ if (existing.get(requestMinimumField) == null)
+ {
+ HashMap map = new HashMap();
+ map.put(requestMinimumField,new ColumnDescription("CHAR(1)",false,true,null,null,false));
+ performAlter(map,null,null,null);
+ }
+
}
// Index management
@@ -171,7 +181,8 @@
stringToEnumeratedValue((String)row.getValue(hourOfDayField)),
stringToEnumeratedValue((String)row.getValue(minutesOfHourField)),
(String)row.getValue(timezoneField),
- (Long)row.getValue(windowDurationField));
+ (Long)row.getValue(windowDurationField),
+ stringToRequestMinimumValue((String)row.getValue(requestMinimumField)));
((JobDescription)returnValues.get(ownerID)).addScheduleRecord(sr);
i++;
}
@@ -209,7 +220,8 @@
stringToEnumeratedValue((String)row.getValue(hourOfDayField)),
stringToEnumeratedValue((String)row.getValue(minutesOfHourField)),
(String)row.getValue(timezoneField),
- (Long)row.getValue(windowDurationField));
+ (Long)row.getValue(windowDurationField),
+ stringToRequestMinimumValue((String)row.getValue(requestMinimumField)));
ArrayList theList = (ArrayList)returnValues.get(ownerID);
if (theList == null)
{
@@ -245,6 +257,7 @@
map.put(minutesOfHourField,enumeratedValueToString(record.getMinutesOfHour()));
map.put(timezoneField,record.getTimezone());
map.put(windowDurationField,record.getDuration());
+ map.put(requestMinimumField,requestMinimumValueToString(record.getRequestMinimum()));
map.put(ownerIDField,ownerID);
map.put(ordinalField,new Long((long)i));
performInsert(map,null);
@@ -340,5 +353,22 @@
return rval.toString();
}
-
+ public static String requestMinimumValueToString(boolean requestMinimum)
+ {
+ return requestMinimum?"T":"F";
+ }
+
+ public static boolean stringToRequestMinimumValue(String requestMinimum)
+ throws ManifoldCFException
+ {
+ if (requestMinimum == null)
+ return false;
+ else if (requestMinimum.equals("T"))
+ return true;
+ else if (requestMinimum.equals("F"))
+ return false;
+ else
+ throw new ManifoldCFException("Bad requestminimum value: '"+requestMinimum+"'");
+ }
+
}
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobNotificationThread.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobNotificationThread.java
index ea8b1ce..0a775e4 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobNotificationThread.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobNotificationThread.java
@@ -66,7 +66,7 @@
// Before we begin, conditionally reset
resetManager.waitForReset(threadContext);
- JobStartRecord[] jobsNeedingNotification = jobManager.getJobsReadyForInactivity();
+ JobNotifyRecord[] jobsNeedingNotification = jobManager.getJobsReadyForInactivity();
try
{
HashMap connectionNames = new HashMap();
@@ -74,7 +74,7 @@
int k = 0;
while (k < jobsNeedingNotification.length)
{
- JobStartRecord jsr = jobsNeedingNotification[k++];
+ JobNotifyRecord jsr = jobsNeedingNotification[k++];
Long jobID = jsr.getJobID();
IJobDescription job = jobManager.load(jobID,true);
if (job != null)
@@ -144,7 +144,7 @@
k = 0;
while (k < jobsNeedingNotification.length)
{
- JobStartRecord jsr = jobsNeedingNotification[k++];
+ JobNotifyRecord jsr = jobsNeedingNotification[k++];
Long jobID = jsr.getJobID();
IJobDescription job = jobManager.load(jobID,true);
if (job != null)
@@ -170,7 +170,7 @@
int i = 0;
while (i < jobsNeedingNotification.length)
{
- JobStartRecord jsr = jobsNeedingNotification[i++];
+ JobNotifyRecord jsr = jobsNeedingNotification[i++];
if (!jsr.wasStarted())
{
// Clean up from failed start.
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
index af5c70f..da97f63 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
@@ -3118,13 +3118,13 @@
/** Start a job.
*/
- protected static int apiWriteStartJob(IThreadContext tc, Configuration output, Long jobID)
+ protected static int apiWriteStartJob(IThreadContext tc, Configuration output, Long jobID, boolean requestMinimum)
throws ManifoldCFException
{
try
{
IJobManager jobManager = JobManagerFactory.make(tc);
- jobManager.manualStart(jobID);
+ jobManager.manualStart(jobID,requestMinimum);
return WRITERESULT_CREATED;
}
catch (ManifoldCFException e)
@@ -3154,13 +3154,13 @@
/** Restart a job.
*/
- protected static int apiWriteRestartJob(IThreadContext tc, Configuration output, Long jobID)
+ protected static int apiWriteRestartJob(IThreadContext tc, Configuration output, Long jobID, boolean requestMinimum)
throws ManifoldCFException
{
try
{
IJobManager jobManager = JobManagerFactory.make(tc);
- jobManager.manualAbortRestart(jobID);
+ jobManager.manualAbortRestart(jobID,requestMinimum);
return WRITERESULT_CREATED;
}
catch (ManifoldCFException e)
@@ -3379,7 +3379,12 @@
if (path.startsWith("start/"))
{
Long jobID = new Long(path.substring("start/".length()));
- return apiWriteStartJob(tc,output,jobID);
+ return apiWriteStartJob(tc,output,jobID,false);
+ }
+ else if (path.startsWith("startminimal/"))
+ {
+ Long jobID = new Long(path.substring("startminimal/".length()));
+ return apiWriteStartJob(tc,output,jobID,true);
}
else if (path.startsWith("abort/"))
{
@@ -3389,7 +3394,12 @@
else if (path.startsWith("restart/"))
{
Long jobID = new Long(path.substring("restart/".length()));
- return apiWriteRestartJob(tc,output,jobID);
+ return apiWriteRestartJob(tc,output,jobID,false);
+ }
+ else if (path.startsWith("restartminimal/"))
+ {
+ Long jobID = new Long(path.substring("restartminimal/".length()));
+ return apiWriteRestartJob(tc,output,jobID,true);
}
else if (path.startsWith("pause/"))
{
@@ -3580,6 +3590,7 @@
protected static final String JOBNODE_SCHEDULE = "schedule";
protected static final String JOBNODE_LINKTYPE = "link_type";
protected static final String JOBNODE_COUNT = "count";
+ protected static final String JOBNODE_REQUESTMINIMUM = "requestminimum";
protected static final String JOBNODE_TIMEZONE = "timezone";
protected static final String JOBNODE_DURATION = "duration";
protected static final String JOBNODE_DAYOFWEEK = "dayofweek";
@@ -3732,6 +3743,7 @@
// Create a schedule record.
String timezone = null;
Long duration = null;
+ boolean requestMinimum = false;
EnumeratedValues dayOfWeek = null;
EnumeratedValues monthOfYear = null;
EnumeratedValues dayOfMonth = null;
@@ -3745,6 +3757,10 @@
{
ConfigurationNode scheduleField = child.findChild(q++);
String fieldType = scheduleField.getType();
+ if (fieldType.equals(JOBNODE_REQUESTMINIMUM))
+ {
+ requestMinimum = scheduleField.getValue().equals("true");
+ }
if (fieldType.equals(JOBNODE_TIMEZONE))
{
timezone = scheduleField.getValue();
@@ -3780,7 +3796,7 @@
else
throw new ManifoldCFException("Unrecognized field in schedule record: '"+fieldType+"'");
}
- ScheduleRecord sr = new ScheduleRecord(dayOfWeek,monthOfYear,dayOfMonth,year,hourOfDay,minutesOfHour,timezone,duration);
+ ScheduleRecord sr = new ScheduleRecord(dayOfWeek,monthOfYear,dayOfMonth,year,hourOfDay,minutesOfHour,timezone,duration,requestMinimum);
// Add the schedule record to the job.
jobDescription.addScheduleRecord(sr);
}
@@ -3931,6 +3947,11 @@
child = new ConfigurationNode(JOBNODE_SCHEDULE);
ConfigurationNode recordChild;
+ // requestminimum
+ recordChild = new ConfigurationNode(JOBNODE_REQUESTMINIMUM);
+ recordChild.setValue(sr.getRequestMinimum()?"true":"false");
+ child.addChild(child.getChildCount(),recordChild);
+
// timezone
if (sr.getTimezone() != null)
{
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java
index be44940..6147c58 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java
@@ -91,7 +91,7 @@
Logging.threads.debug("Seeding thread woke up");
// Grab active, adaptive jobs (and set their state to xxxSEEDING as a side effect)
- JobStartRecord[] seedJobs = jobManager.getJobsReadyForSeeding(currentTime);
+ JobSeedingRecord[] seedJobs = jobManager.getJobsReadyForSeeding(currentTime);
// Process these jobs, and do the seeding. The seeding is based on what came back
// in the job start record for sync time. If there's an interruption, we just go on
@@ -113,7 +113,7 @@
int i = 0;
while (i < seedJobs.length)
{
- JobStartRecord jsr = seedJobs[i++];
+ JobSeedingRecord jsr = seedJobs[i++];
Long jobID = jsr.getJobID();
try
{
@@ -203,7 +203,7 @@
int i = 0;
while (i < seedJobs.length)
{
- JobStartRecord jsr = seedJobs[i++];
+ JobSeedingRecord jsr = seedJobs[i++];
if (!jsr.wasStarted())
{
if (Logging.threads.isDebugEnabled())
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartDeleteThread.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartDeleteThread.java
index 5f80949..b70888c 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartDeleteThread.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartDeleteThread.java
@@ -80,7 +80,7 @@
// See if there are any starting jobs.
// Note: Since this following call changes the job state, we must be careful to reset it on any kind of failure.
- JobStartRecord[] deleteJobs = jobManager.getJobsReadyForDelete();
+ JobDeleteRecord[] deleteJobs = jobManager.getJobsReadyForDelete();
try
{
@@ -100,7 +100,7 @@
int i = 0;
while (i < deleteJobs.length)
{
- JobStartRecord jsr = deleteJobs[i++];
+ JobDeleteRecord jsr = deleteJobs[i++];
Long jobID = jsr.getJobID();
try
{
@@ -128,7 +128,7 @@
int i = 0;
while (i < deleteJobs.length)
{
- JobStartRecord jsr = deleteJobs[i++];
+ JobDeleteRecord jsr = deleteJobs[i++];
if (!jsr.wasStarted())
{
// Clean up from failed start.
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java
index c2c4ec0..2fbfa16 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java
@@ -135,38 +135,15 @@
// Get the number of link types.
String[] legalLinkTypes = connector.getRelationshipTypes();
- // The old logic here looked at the model, and if it was incomplete, either
- // performed a complete crawl initialization, or an incremental crawl initialization.
- // The fact is that we can now determine automatically what kind of crawl should be
- // done, based on the model the connector states and on the starting time that we
- // would be feeding it. (The starting time is reset to 0 when the document specification
- // is changed - that's a crucial consideration.)
- //
- // The new logic does this:
- //
- // (1) If the connector has MODEL_ADD_CHANGE_DELETE, then
- // we let the connector run the show; there's no purge phase, and therefore the
- // documents are left in a COMPLETED state if they don't show up in the list
- // of seeds that require the attention of the connector.
- //
- // (2) If the connector has MODEL_ALL, then it's a full crawl no matter what, so
- // we do a full scan initialization.
- //
- // (3) If the connector has some other model, we look at the start time. A start
- // time of 0 implies a full scan, while any other start time implies an incremental
- // scan.
-
- if (model != connector.MODEL_ADD_CHANGE_DELETE)
- {
- if (Logging.threads.isDebugEnabled())
- Logging.threads.debug("Preparing job "+jobID.toString()+" for execution...");
- if (jobType != IJobDescription.TYPE_CONTINUOUS && model != connector.MODEL_PARTIAL && (model == connector.MODEL_ALL || lastJobTime == 0L))
- jobManager.prepareFullScan(jobID,legalLinkTypes,hopcountMethod);
- else
- jobManager.prepareIncrementalScan(jobID,legalLinkTypes,hopcountMethod);
- if (Logging.threads.isDebugEnabled())
- Logging.threads.debug("Prepared job "+jobID.toString()+" for execution.");
- }
+ boolean requestMinimum = jsr.getRequestMinimum();
+
+ if (Logging.threads.isDebugEnabled())
+ Logging.threads.debug("Preparing job "+jobID.toString()+" for execution...");
+ jobManager.prepareJobScan(jobID,legalLinkTypes,hopcountMethod,
+ model,jobType == IJobDescription.TYPE_CONTINUOUS,lastJobTime == 0L,
+ requestMinimum);
+ if (Logging.threads.isDebugEnabled())
+ Logging.threads.debug("Prepared job "+jobID.toString()+" for execution.");
try
{
diff --git a/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties b/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties
index 98f435f..bee28cb 100644
--- a/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties
+++ b/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties
@@ -34,10 +34,6 @@
editjob.HopFilters=Hop Filters
editjob.am=am
editjob.pm=pm
-editjob.st=st
-editjob.nd=nd
-editjob.rd=rd
-editjob.th=th
index.WelcomeToApacheManifoldFC=Welcome to Apache ManifoldCF
@@ -341,6 +337,14 @@
editjob.Add=Add
editjob.Addforcedmetadata=Add forced metadata
editjob.ForcedMetadataNameMustNotBeNull=Forced metadata name must not be null
+editjob.st=st
+editjob.nd=nd
+editjob.rd=rd
+editjob.th=th
+editjob.dayofmonth=day of month
+editjob.JobInvocationColon=Job invocation:
+editjob.Minimal=Minimal
+editjob.Complete=Complete
editjob.tab=tab
@@ -357,6 +361,39 @@
showjobstatus.PleaseTryAgainLater=This page is unavailable due to maintenance operations. Please try again later.
showjobstatus.StatusOfJobs=Status of Jobs
+showjobstatus.Notyetrun=Not yet run
+showjobstatus.Running=Running
+showjobstatus.Runningnoconnector=Running, no connector
+showjobstatus.Aborting=Aborting
+showjobstatus.Restarting=Restarting
+showjobstatus.Stopping=Stopping
+showjobstatus.Resuming=Resuming
+showjobstatus.Paused=Paused
+showjobstatus.Done=Done
+showjobstatus.Waiting=Waiting
+showjobstatus.Startingup=Starting up
+showjobstatus.Cleaningup=Cleaning up
+showjobstatus.Terminating=Terminating
+showjobstatus.Endnotification=End notification
+showjobstatus.ErrorColon=Error:
+showjobstatus.Unknown=Unknown
+showjobstatus.Notstarted=Not started
+showjobstatus.Aborted=Aborted
+showjobstatus.Neverrun=Never run
+showjobstatus.Start=Start
+showjobstatus.Startminimal=Start minimal
+showjobstatus.Restart=Restart
+showjobstatus.Restartminimal=Restart minimal
+showjobstatus.Pause=Pause
+showjobstatus.Abort=Abort
+showjobstatus.Resume=Resume
+showjobstatus.Startjob=Start job
+showjobstatus.minimally=minimally
+showjobstatus.Restartjob=Restart job
+showjobstatus.Pausejob=Pause job
+showjobstatus.Abortjob=Abort job
+showjobstatus.Resumejob=Resume job
+
documentstatus.ApacheManifoldCFDocumentStatus=Apache ManifoldCF: Document Status
documentstatus.DocumentStatus=Document Status
documentstatus.Connection=Connection:
@@ -686,7 +723,7 @@
viewjob.am=am
viewjob.pm=pm
viewjob.plus=plus
-viewjob.inJanuary=in January
+viewjob.ineverymonthofyear=in every month of the year
viewjob.in=in
viewjob.January=January
viewjob.February=February
@@ -713,6 +750,9 @@
viewjob.nd=nd
viewjob.rd=rd
viewjob.th=th
+viewjob.JobInvocationColon=Job invocation:
+viewjob.Minimal=Minimal
+viewjob.Complete=Complete
viewauthority.ViewAuthorityConnectionStatus=View Authority Connection Status
viewauthority.NameColon=Name:
diff --git a/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties b/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties
index 90b849a..4f0e3c6 100644
--- a/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties
+++ b/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties
@@ -34,10 +34,6 @@
editjob.HopFilters=ホップフィルタ
editjob.am=午前
editjob.pm=午後
-editjob.st=日
-editjob.nd=日
-editjob.rd=日
-editjob.th=日
index.WelcomeToApacheManifoldFC=Apache ManifoldCFへようこそ
@@ -341,6 +337,14 @@
editjob.Add=Add
editjob.Addforcedmetadata=Add forced metadata
editjob.ForcedMetadataNameMustNotBeNull=Forced metadata name must not be null
+editjob.st=日
+editjob.nd=日
+editjob.rd=日
+editjob.th=日
+editjob.dayofmonth=day of month
+editjob.JobInvocationColon=Job invocation:
+editjob.Minimal=Minimal
+editjob.Complete=Complete
editjob.tab=tab
@@ -357,6 +361,39 @@
showjobstatus.PleaseTryAgainLater=保守処理中です。少々お待ちください。
showjobstatus.StatusOfJobs=ジョブの状態
+showjobstatus.Notyetrun=Not yet run
+showjobstatus.Running=Running
+showjobstatus.Runningnoconnector=Running, no connector
+showjobstatus.Aborting=Aborting
+showjobstatus.Restarting=Restarting
+showjobstatus.Stopping=Stopping
+showjobstatus.Resuming=Resuming
+showjobstatus.Paused=Paused
+showjobstatus.Done=Done
+showjobstatus.Waiting=Waiting
+showjobstatus.Startingup=Starting up
+showjobstatus.Cleaningup=Cleaning up
+showjobstatus.Terminating=Terminating
+showjobstatus.Endnotification=End notification
+showjobstatus.ErrorColon=Error:
+showjobstatus.Unknown=Unknown
+showjobstatus.Notstarted=Not started
+showjobstatus.Aborted=Aborted
+showjobstatus.Neverrun=Never run
+showjobstatus.Start=Start
+showjobstatus.Startminimal=Start minimal
+showjobstatus.Restart=Restart
+showjobstatus.Restartminimal=Restart minimal
+showjobstatus.Pause=Pause
+showjobstatus.Abort=Abort
+showjobstatus.Resume=Resume
+showjobstatus.Startjob=Start job
+showjobstatus.minimally=minimally
+showjobstatus.Restartjob=Restart job
+showjobstatus.Pausejob=Pause job
+showjobstatus.Abortjob=Abort job
+showjobstatus.Resumejob=Resume job
+
documentstatus.ApacheManifoldCFDocumentStatus=Apache ManifoldCF：コンテンツの状態
documentstatus.DocumentStatus=コンテンツの状態
documentstatus.Connection=コネクション：
@@ -673,34 +710,34 @@
viewjob.Startatbeginningofschedulewindow=Start at beginning of schedule window
viewjob.Startinsideschedulewindow=Start inside schedule window
viewjob.Dontautomaticallystart=Don't automatically start
-viewjob.Anydayoftheweek=Any day of week
-viewjob.Sundays=Sundays
-viewjob.Mondays=Mondays
-viewjob.Tuesdays=Tuesdays
-viewjob.Wednesdays=Wednesdays
-viewjob.Thursdays=Thursdays
-viewjob.Fridays=Fridays
-viewjob.Saturdays=Saturdays
+viewjob.Anydayoftheweek=すべての曜日
+viewjob.Sundays=日
+viewjob.Mondays=月
+viewjob.Tuesdays=火
+viewjob.Wednesdays=水
+viewjob.Thursdays=木
+viewjob.Fridays=金
+viewjob.Saturdays=土
viewjob.oneveryhour=on every hour
viewjob.atmidnight=at midnight
viewjob.at=at
-viewjob.am=am
-viewjob.pm=pm
+viewjob.am=午前
+viewjob.pm=午後
viewjob.plus=plus
-viewjob.inJanuary=in January
-viewjob.in=in
-viewjob.January=January
-viewjob.February=February
-viewjob.March=March
-viewjob.April=April
-viewjob.May=May
-viewjob.June=June
-viewjob.July=July
-viewjob.August=August
-viewjob.September=September
-viewjob.October=October
-viewjob.November=November
-viewjob.December=December
+viewjob.ineverymonthofyear=すべての月
+viewjob.in=
+viewjob.January=１月
+viewjob.February=２月
+viewjob.March=３月
+viewjob.April=４月
+viewjob.May=５月
+viewjob.June=６月
+viewjob.July=７月
+viewjob.August=８月
+viewjob.September=９月
+viewjob.October=１０月
+viewjob.November=１１月
+viewjob.December=１２月
viewjob.onthe1stofthemonth=on the 1st of the month
viewjob.onthe=on the
viewjob.ofthemonth=of the month
@@ -710,10 +747,13 @@
viewjob.Deleteunreachabledocuments=Delete unreachable documents
viewjob.Nodeletesfornow=No deletes, for now
viewjob.Nodeletesforever=No deletes, forever
-viewjob.st=st
-viewjob.nd=nd
-viewjob.rd=rd
-viewjob.th=th
+viewjob.st=日
+viewjob.nd=日
+viewjob.rd=日
+viewjob.th=日
+viewjob.JobInvocationColon=Job invocation:
+viewjob.Minimal=Minimal
+viewjob.Complete=Complete
viewauthority.ViewAuthorityConnectionStatus=権限コネクション状態の表示
viewauthority.NameColon=名前：
diff --git a/site/src/documentation/content/xdocs/en_US/end-user-documentation.xml b/site/src/documentation/content/xdocs/en_US/end-user-documentation.xml
index 8338b33..551d8a7 100644
--- a/site/src/documentation/content/xdocs/en_US/end-user-documentation.xml
+++ b/site/src/documentation/content/xdocs/en_US/end-user-documentation.xml
@@ -284,15 +284,20 @@
status column. Allowed actions you may see at one point or another include:</p>
<ul>
<li>Start (start the job)</li>
+ <li>Start minimal (start the job, but do only the minimal work possible)</li>
<li>Abort(abort the job)</li>
<li>Pause (pause the job)</li>
<li>Resume (resume the job)</li>
<li>Restart (equivalent to aborting the job, and starting it all over again)</li>
+ <li>Restart minimal (equivalent to aborting the job, and starting it all over again, doing only the minimal work possible)</li>
</ul>
<br/>
<p>The columns "Documents", "Active", and "Processed" have very specific means as far as documents in the job's queue are concerned. The "Documents" column counts all the documents
that belong to the job. The "Active" column counts all of the documents for that job that are queued up for processing. The "Processed" column counts all documents that are on the
queue for the job that have been processed at least once in the past.</p>
+ <p>Using the "minimal" variant of the listed actions will perform the minimum possible amount of work, given the model that the connection type for the job uses. In some
+ cases, this will mean that additions and modifications are indexed, but deletions are not detected. A complete job run is usually necessary to fully synchronize the target
+ index with the repository contents.</p>
</section>
<section id="statusreports">
<title>Status Reports</title>
diff --git a/site/src/documentation/content/xdocs/en_US/programmatic-operation.xml b/site/src/documentation/content/xdocs/en_US/programmatic-operation.xml
index d9b1870..87fb638 100644
--- a/site/src/documentation/content/xdocs/en_US/programmatic-operation.xml
+++ b/site/src/documentation/content/xdocs/en_US/programmatic-operation.xml
@@ -30,7 +30,9 @@
<section>
<title>Programmatic Operation</title>
<p></p>
- <p>A certain subset of ManifoldCF users want to think of ManifoldCF as an engine that they can poke from whatever other system they are developing. While ManifoldCF is not precisely a document indexing engine per se, it can certainly be controlled programmatically. Right now, there are three principle ways of achieving this control.</p>
+ <p>A certain subset of ManifoldCF users want to think of ManifoldCF as an engine that they can poke from whatever other system they are developing. While
+ ManifoldCF is not precisely a document indexing engine per se, it can certainly be controlled programmatically. Right now, there are three principle ways of
+ achieving this control.</p>
<p></p>
<section>
<title>Control by Servlet API</title>
@@ -46,17 +48,17 @@
<p>http[s]://<em><server_and_port></em>/mcf-api-service/json/<em><resource></em></p>
<p></p>
<p>The servlet ignores request data, except when the PUT or POST verb is used. In that case, the request data is presumed to be a JSON object. The servlet
- responds either with an error response code (either 400 or 500) with an appropriate explanatory message, or with a 200 (OK), 201 (CREATED), or 404 (NOT FOUND)
- response code along with a response JSON object.</p>
+ responds either with an error response code (either 400 or 500) with an appropriate explanatory message, or with a 200 (OK), 201 (CREATED), or
+ 404 (NOT FOUND) response code along with a response JSON object.</p>
<p></p>
</section>
<section>
<title>JSON equivalents for ManifoldCF</title>
<p></p>
- <p>ManifoldCF treats certain JSON forms as equivalent, for the purposes of readability. For example, the array form <strong>"foo" : [ { ... } ]</strong> is treated equivalently to
- <strong>"foo" : { }</strong>, whenever there is only one array element. This gives a coder some flexibility as to how s/he encodes JSON in requests. Please also be aware that
- similar compressions will occur in the JSON responses from the API servlet, and your code must be able to deal with this possibility. The following table
- describes some of the equivalences:</p>
+ <p>ManifoldCF treats certain JSON forms as equivalent, for the purposes of readability. For example, the array form <strong>"foo" : [ { ... } ]</strong> is
+ treated equivalently to <strong>"foo" : { }</strong>, whenever there is only one array element. This gives a coder some flexibility as to how s/he encodes
+ JSON in requests. Please also be aware that similar compressions will occur in the JSON responses from the API servlet, and your code must be able to deal
+ with this possibility. The following table describes some of the equivalences:</p>
<p></p>
<p></p>
<p></p>
@@ -107,8 +109,10 @@
<tr><td>jobstatuses/<em><job_id></em></td><td>GET</td><td>Get a specific job's status</td><td>N/A</td><td>{"jobstatus":<em><job_status_object></em>} <strong>OR</strong> { } <strong>OR</strong> {"error":<em><error_text></em>} </td></tr>
<tr><td>jobstatusesnocounts/<em><job_id></em></td><td>GET</td><td>Get a specific job's status, returning '0' for all counts</td><td>N/A</td><td>{"jobstatus":<em><job_status_object></em>} <strong>OR</strong> { } <strong>OR</strong> {"error":<em><error_text></em>} </td></tr>
<tr><td>start/<em><job_id></em></td><td>PUT</td><td>Start a specified job manually</td><td>N/A</td><td>{ } <strong>OR</strong> {"error":<em><error_text></em>}</td></tr>
+ <tr><td>startminimal/<em><job_id></em></td><td>PUT</td><td>Start a specified job manually, minimal run requested</td><td>N/A</td><td>{ } <strong>OR</strong> {"error":<em><error_text></em>}</td></tr>
<tr><td>abort/<em><job_id></em></td><td>PUT</td><td>Abort a specified job</td><td>N/A</td><td>{ } <strong>OR</strong> {"error":<em><error_text></em>}</td></tr>
<tr><td>restart/<em><job_id></em></td><td>PUT</td><td>Stop and start a specified job</td><td>N/A</td><td>{ } <strong>OR</strong> {"error":<em><error_text></em>}</td></tr>
+ <tr><td>restartminimal/<em><job_id></em></td><td>PUT</td><td>Stop and start a specified job, minimal run requested</td><td>N/A</td><td>{ } <strong>OR</strong> {"error":<em><error_text></em>}</td></tr>
<tr><td>pause/<em><job_id></em></td><td>PUT</td><td>Pause a specified job</td><td>N/A</td><td>{ } <strong>OR</strong> {"error":<em><error_text></em>}</td></tr>
<tr><td>resume/<em><job_id></em></td><td>PUT</td><td>Resume a specified job</td><td>N/A</td><td>{ } <strong>OR</strong> {"error":<em><error_text></em>}</td></tr>
@@ -343,6 +347,7 @@
<tr><td>"year"</td><td>The optional year enumeration object</td></tr>
<tr><td>"hourofday"</td><td>The optional hour-of-the-day enumeration object</td></tr>
<tr><td>"minutesofhour"</td><td>The optional minutes-of-the-hour enumeration object</td></tr>
+ <tr><td>"requestminimum"</td><td>Optional flag indicating whether the job run will be minimal or not ("true" means minimal)</td></tr>
</table>
<p></p>
<p>Each enumeration object describes an array of integers using the form:</p>
@@ -434,16 +439,21 @@
<section>
<title>Control by direct code</title>
<p></p>
- <p>Control by direct java code is quite a reasonable thing to do. The sources of the above commands should give a pretty clear idea how to proceed, if that's what you want to do.</p>
+ <p>Control by direct java code is quite a reasonable thing to do. The sources of the above commands should give a pretty clear idea how to proceed, if that's what you
+ want to do.</p>
<p></p>
<p></p>
</section>
<section>
<title>Caveats</title>
<p></p>
- <p>The above commands know nothing about the differences between connection types. Instead, they deal with configuration and specification information in the form of XML documents. Normally, these XML documents are hidden from a system integrator, unless they happen to look into the database with a tool such as psql. But the API commands above often will require such XML documents to be included as part of the command execution.</p>
+ <p>The above commands know nothing about the differences between connection types. Instead, they deal with configuration and specification information in the
+ form of XML documents. Normally, these XML documents are hidden from a system integrator, unless they happen to look into the database with a tool such as
+ psql. But the API commands above often will require such XML documents to be included as part of the command execution.</p>
<p></p>
- <p>This has one major consequence. Any application that would manipulate connections and jobs directly cannot be connection-type independent - these applications must know the proper form of XML to submit to the command. So, it is not possible to use these command APIs to write one's own UI wrapper, without sacrificing some of the repository independence that ManifoldCF by itself maintains.</p>
+ <p>This has one major consequence. Any application that would manipulate connections and jobs directly cannot be connection-type independent - these
+ applications must know the proper form of XML to submit to the command. So, it is not possible to use these command APIs to write one's own UI wrapper,
+ without sacrificing some of the repository independence that ManifoldCF by itself maintains.</p>
</section>
</section>
</body>
diff --git a/site/src/documentation/content/xdocs/en_US/writing-repository-connectors.xml b/site/src/documentation/content/xdocs/en_US/writing-repository-connectors.xml
index e8572d1..d7b9b39 100644
--- a/site/src/documentation/content/xdocs/en_US/writing-repository-connectors.xml
+++ b/site/src/documentation/content/xdocs/en_US/writing-repository-connectors.xml
@@ -73,8 +73,9 @@
<li>LiveLink (demonstrates use of local keystore infrastructure)</li>
<li>Meridio (local keystore, web services, result sets)</li>
<li>SharePoint (local keystore, web services)</li>
- <li>RSS (local keystore, binning)</li>
- <li>Web (local database schema, local keystore, binning, events and prerequisites, cache management)</li>
+ <li>RSS (local keystore, binning, fuzzy xml parsing)</li>
+ <li>Web (local database schema, local keystore, binning, events and prerequisites, cache management, fuzzy xml parsing)</li>
+ <li>Wiki (binning, rigorous xml parsing)</li>
</ul>
<p></p>
<p>You will also note that all of these connectors extend a framework-provided repository connector base class, found at <em>org.apache.manifoldcf.crawler.connectors.BaseRepositoryConnector</em>. This base class furnishes some basic bookkeeping logic for managing the connector pool, as well as default implementations of some of the less typical functionality a connector may have. For example, connectors are allowed to have database tables of their own, which are instantiated when the connector is registered, and are torn down when the connector is removed. This is, however, not very typical, and the base implementation reflects that.</p>
@@ -109,15 +110,24 @@
<p></p>
<table>
<tr><th>Model</th><th>Description</th></tr>
+ <tr><td><em>MODEL_ALL</em></td><td>The <strong>addSeedDocuments()</strong> method supplies all specified documents on each call</td></tr>
+ <tr><td><em>MODEL_PARTIAL</em></td><td>The <strong>addSeedDocuments()</strong> does not return a complete list of documents that match the criteria and time interval, because some of those documents are no longer discoverable</td></tr>
<tr><td><em>MODEL_ADD</em></td><td>The <strong>addSeedDocuments()</strong> method supplies at least all the matching documents that have been added to the repository, within the specified time interval</td></tr>
<tr><td><em>MODEL_ADD_CHANGE</em></td><td>The <strong>addSeedDocuments()</strong> method supplies at least those matching documents that have been added or changed in the repository, within the specified time interval</td></tr>
<tr><td><em>MODEL_ADD_CHANGE_DELETE</em></td><td>The <strong>addSeedDocuments()</strong> method supplies at least those matching documents that have been added, changed, or removed in the repository, within the specified time interval</td></tr>
- <tr><td><em>MODEL_PARTIAL</em></td><td>The <strong>addSeedDocuments()</strong> does not return a complete list of documents that match the criteria and time interval, because some of those documents are no longer discoverable</td></tr>
+ <tr><td><em>MODEL_CHAINED_ADD</em></td><td>The <strong>addSeedDocuments()</strong> method, plus documents reachable by discovery from seeds, supplies at least all the matching documents that have been added to the repository, within the specified time interval</td></tr>
+ <tr><td><em>MODEL_CHAINED_ADD_CHANGE</em></td><td>The <strong>addSeedDocuments()</strong> method, plus documents reachable by discovery from seeds, supplies at least those matching documents that have been added or changed in the repository, within the specified time interval</td></tr>
+ <tr><td><em>MODEL_CHAINED_ADD_CHANGE_DELETE</em></td><td>The <strong>addSeedDocuments()</strong> method, plus documents reachable by discovery from seeds, supplies at least those matching documents that have been added, changed, or removed in the repository, within the specified time interval</td></tr>
</table>
<p></p>
- <p>Note that the choice of model is actually much more subtle than the above description might indicate. It may, for one thing, be affected by characteristics of the repository, such as whether the repository considers a document to have been changed if its security information was changed. This would mean that, even though most document changes are picked up and thus one might be tempted to declare the connector to be <em>MODEL_ADD_CHANGE</em>, the correct choice would in fact be <em>MODEL_ADD</em>.</p>
+ <p>Note that the choice of model is actually much more subtle than the above description might indicate. It may, for one thing, be affected by characteristics of
+ the repository, such as whether the repository considers a document to have been changed if its security information was changed. This would mean that,
+ even though most document changes are picked up and thus one might be tempted to declare the connector to be <em>MODEL_ADD_CHANGE</em>, the
+ correct choice would in fact be <em>MODEL_ADD</em>.</p>
<p></p>
- <p>Another subtle point is what documents the connector is actually supposed to return by means of the <strong>addSeedDocuments()</strong> method. The start time and end time parameters handed to the method do not have to be strictly adhered to, for instance; it is always okay to return more documents. It is never okay for the connector to return fewer documents than were requested, on the other hand.</p>
+ <p>Another subtle point is what documents the connector is actually supposed to return by means of the <strong>addSeedDocuments()</strong> method. The
+ start time and end time parameters handed to the method do not have to be strictly adhered to, for instance; it is always okay to return more documents. It is never
+ okay for the connector to return fewer documents than were requested, on the other hand.</p>
<p></p>
</section>
<section>
@@ -139,19 +149,34 @@
<section>
<title>Choosing the form of the document version string</title>
<p></p>
- <p>The document version string is used by ManifoldCF to determine whether or not the document or configuration changed in such a way as to require that the document be reprocessed. ManifoldCF therefore requests the version string for any document that is ready for processing, and usually does not process the document again if the returned version string agrees with the version string it has stored.</p>
+ <p>The document version string is used by ManifoldCF to determine whether or not the document or configuration changed in such a way as to require that the document
+ be reprocessed. ManifoldCF therefore requests the version string for any document that is ready for processing, and usually does not process the document again if the
+ returned version string agrees with the version string it has stored.</p>
<p></p>
- <p>Thinking about it more carefully, it is clear that what a connector writer needs to do is include everything in the version string that could potentially affect how the document gets processed. That may include the version of the document in the repository, bits of configuration information, metadata, and even access tokens (if the underlying repository versions these things independently from the document itself). Storing all of that information in the version string seems like a lot - but the string is unlimited in length, and it actually serves another useful purpose to do it that way. Specifically, when it comes time to do the actual processing, it's often the correct thing to do to obtain the necessary data out of the version string, rather than calculating it or fetching it anew. That way of working guarantees that the document processing was done in a manner that agrees with its recorded version string, thus eliminating any chance of ManifoldCF getting confused.</p>
+ <p>Thinking about it more carefully, it is clear that what a connector writer needs to do is include everything in the version string that could potentially affect how the
+ document gets processed. That may include the version of the document in the repository, bits of configuration information, metadata, and even access tokens (if the
+ underlying repository versions these things independently from the document itself). Storing all of that information in the version string seems like a lot - but the string
+ is unlimited in length, and it actually serves another useful purpose to do it that way. Specifically, when it comes time to do the actual processing, it's often the correct
+ thing to do to obtain the necessary data out of the version string, rather than calculating it or fetching it anew. That way of working guarantees that the document
+ processing was done in a manner that agrees with its recorded version string, thus eliminating any chance of ManifoldCF getting confused.</p>
<p></p>
- <p>For longer data that needs to persist between the <strong>getDocumentVersions()</strong> method call and the <strong>processDocuments()</strong> method call, the connector is welcome to save this information in a temporary disk file. To help make sure nothing leaks which this approach is used, the IRepositoryConnector interface has a method that will be called to clean up any temporary files that might have been created in the handling of a given document identifier.</p>
+ <p>For longer data that needs to persist between the <strong>getDocumentVersions()</strong> method call and the <strong>processDocuments()</strong> method
+ call, the connector is welcome to save this information in a temporary disk file. To help make sure nothing leaks which this approach is used, the IRepositoryConnector
+ interface has a method that will be called to clean up any temporary files that might have been created in the handling of a given document identifier.</p>
<p></p>
</section>
<section>
<title>Notes on connector UI methods</title>
<p></p>
- <p>The crawler UI uses a tabbed layout structure, and thus each of these elements must properly implement the tabbed model. This means that the "header" methods above must add the desired tab names to a specified array, and the "body" methods must provide appropriate HTML which handles both the case where a tab is displayed, and where it is not displayed. Also, it makes sense to use the appropriate css definitions, so that the connector UI pages have a similar look-and-feel to the rest of ManifoldCF's crawler ui. We strongly suggest starting with one of the supplied connector's UI code, both for a description of the arguments to each page, and for some decent ideas of ways to organize your connector's UI code. </p>
+ <p>The crawler UI uses a tabbed layout structure, and thus each of these elements must properly implement the tabbed model. This means that the "header" methods
+ above must add the desired tab names to a specified array, and the "body" methods must provide appropriate HTML which handles both the case where a tab is
+ displayed, and where it is not displayed. Also, it makes sense to use the appropriate css definitions, so that the connector UI pages have a similar look-and-feel to the
+ rest of ManifoldCF's crawler ui. We strongly suggest starting with one of the supplied connector's UI code, both for a description of the arguments to each page, and
+ for some decent ideas of ways to organize your connector's UI code. </p>
<p></p>
- <p>Please also note that it is good practice to name the form fields in your HTML in such a way that they cannot collide with form fields that may come from the framework's HTML or any specific output connector's HTML. The <em>DocumentSpecification</em> editing HTML especially may be prone to collisions, because within any given job, this HTML is included in the same page as HTML from the chosen output connector.</p>
+ <p>Please also note that it is good practice to name the form fields in your HTML in such a way that they cannot collide with form fields that may come from the
+ framework's HTML or any specific output connector's HTML. The <em>DocumentSpecification</em> editing HTML especially may be prone to collisions, because
+ within any given job, this HTML is included in the same page as HTML from the chosen output connector.</p>
<p></p>
<p></p>
</section>
@@ -159,7 +184,8 @@
<section>
<title>Implementation support provided by the framework</title>
<p></p>
- <p>ManifoldCF's framework provides a number of helpful services designed to make the creation of a connector easier. These services are summarized below. (This is not an exhaustive list, by any means.)</p>
+ <p>ManifoldCF's framework provides a number of helpful services designed to make the creation of a connector easier. These services are summarized below.
+ (This is not an exhaustive list, by any means.)</p>
<p></p>
<ul>
<li>Lock management and synchronization (see <em>org.apache.manifoldcf.core.interfaces.LockManagerFactory</em>)</li>
@@ -180,7 +206,8 @@
<section>
<title>DO's and DON'T DO's</title>
<p></p>
- <p>It's always a good idea to make use of an existing infrastructure component, if it's meant for that purpose, rather than inventing your own. There are, however, some limitations we recommend you adhere to.</p>
+ <p>It's always a good idea to make use of an existing infrastructure component, if it's meant for that purpose, rather than inventing your own. There are, however,
+ some limitations we recommend you adhere to.</p>
<p></p>
<ul>
<li>DO make use of infrastructure components described in the section above</li>
@@ -188,7 +215,8 @@
<li>NEVER write connector code that directly uses framework database tables, other than the ones installed and managed by your connector</li>
</ul>
<p></p>
- <p>If you are tempted to violate these rules, it may well mean you don't understand something important. At the very least, we'd like to know why. Send email to dev@manifoldcf.apache.org with a description of your problem and how you are tempted to solve it.</p>
+ <p>If you are tempted to violate these rules, it may well mean you don't understand something important. At the very least, we'd like to know why. Send email
+ to dev@manifoldcf.apache.org with a description of your problem and how you are tempted to solve it.</p>
</section>
</section>
</body>
diff --git a/site/src/documentation/content/xdocs/ja_JP/programmatic-operation.xml b/site/src/documentation/content/xdocs/ja_JP/programmatic-operation.xml
index d9b1870..87fb638 100644
--- a/site/src/documentation/content/xdocs/ja_JP/programmatic-operation.xml
+++ b/site/src/documentation/content/xdocs/ja_JP/programmatic-operation.xml
@@ -30,7 +30,9 @@
<section>
<title>Programmatic Operation</title>
<p></p>
- <p>A certain subset of ManifoldCF users want to think of ManifoldCF as an engine that they can poke from whatever other system they are developing. While ManifoldCF is not precisely a document indexing engine per se, it can certainly be controlled programmatically. Right now, there are three principle ways of achieving this control.</p>
+ <p>A certain subset of ManifoldCF users want to think of ManifoldCF as an engine that they can poke from whatever other system they are developing. While
+ ManifoldCF is not precisely a document indexing engine per se, it can certainly be controlled programmatically. Right now, there are three principle ways of
+ achieving this control.</p>
<p></p>
<section>
<title>Control by Servlet API</title>
@@ -46,17 +48,17 @@
<p>http[s]://<em><server_and_port></em>/mcf-api-service/json/<em><resource></em></p>
<p></p>
<p>The servlet ignores request data, except when the PUT or POST verb is used. In that case, the request data is presumed to be a JSON object. The servlet
- responds either with an error response code (either 400 or 500) with an appropriate explanatory message, or with a 200 (OK), 201 (CREATED), or 404 (NOT FOUND)
- response code along with a response JSON object.</p>
+ responds either with an error response code (either 400 or 500) with an appropriate explanatory message, or with a 200 (OK), 201 (CREATED), or
+ 404 (NOT FOUND) response code along with a response JSON object.</p>
<p></p>
</section>
<section>
<title>JSON equivalents for ManifoldCF</title>
<p></p>
- <p>ManifoldCF treats certain JSON forms as equivalent, for the purposes of readability. For example, the array form <strong>"foo" : [ { ... } ]</strong> is treated equivalently to
- <strong>"foo" : { }</strong>, whenever there is only one array element. This gives a coder some flexibility as to how s/he encodes JSON in requests. Please also be aware that
- similar compressions will occur in the JSON responses from the API servlet, and your code must be able to deal with this possibility. The following table
- describes some of the equivalences:</p>
+ <p>ManifoldCF treats certain JSON forms as equivalent, for the purposes of readability. For example, the array form <strong>"foo" : [ { ... } ]</strong> is
+ treated equivalently to <strong>"foo" : { }</strong>, whenever there is only one array element. This gives a coder some flexibility as to how s/he encodes
+ JSON in requests. Please also be aware that similar compressions will occur in the JSON responses from the API servlet, and your code must be able to deal
+ with this possibility. The following table describes some of the equivalences:</p>
<p></p>
<p></p>
<p></p>
@@ -107,8 +109,10 @@
<tr><td>jobstatuses/<em><job_id></em></td><td>GET</td><td>Get a specific job's status</td><td>N/A</td><td>{"jobstatus":<em><job_status_object></em>} <strong>OR</strong> { } <strong>OR</strong> {"error":<em><error_text></em>} </td></tr>
<tr><td>jobstatusesnocounts/<em><job_id></em></td><td>GET</td><td>Get a specific job's status, returning '0' for all counts</td><td>N/A</td><td>{"jobstatus":<em><job_status_object></em>} <strong>OR</strong> { } <strong>OR</strong> {"error":<em><error_text></em>} </td></tr>
<tr><td>start/<em><job_id></em></td><td>PUT</td><td>Start a specified job manually</td><td>N/A</td><td>{ } <strong>OR</strong> {"error":<em><error_text></em>}</td></tr>
+ <tr><td>startminimal/<em><job_id></em></td><td>PUT</td><td>Start a specified job manually, minimal run requested</td><td>N/A</td><td>{ } <strong>OR</strong> {"error":<em><error_text></em>}</td></tr>
<tr><td>abort/<em><job_id></em></td><td>PUT</td><td>Abort a specified job</td><td>N/A</td><td>{ } <strong>OR</strong> {"error":<em><error_text></em>}</td></tr>
<tr><td>restart/<em><job_id></em></td><td>PUT</td><td>Stop and start a specified job</td><td>N/A</td><td>{ } <strong>OR</strong> {"error":<em><error_text></em>}</td></tr>
+ <tr><td>restartminimal/<em><job_id></em></td><td>PUT</td><td>Stop and start a specified job, minimal run requested</td><td>N/A</td><td>{ } <strong>OR</strong> {"error":<em><error_text></em>}</td></tr>
<tr><td>pause/<em><job_id></em></td><td>PUT</td><td>Pause a specified job</td><td>N/A</td><td>{ } <strong>OR</strong> {"error":<em><error_text></em>}</td></tr>
<tr><td>resume/<em><job_id></em></td><td>PUT</td><td>Resume a specified job</td><td>N/A</td><td>{ } <strong>OR</strong> {"error":<em><error_text></em>}</td></tr>
@@ -343,6 +347,7 @@
<tr><td>"year"</td><td>The optional year enumeration object</td></tr>
<tr><td>"hourofday"</td><td>The optional hour-of-the-day enumeration object</td></tr>
<tr><td>"minutesofhour"</td><td>The optional minutes-of-the-hour enumeration object</td></tr>
+ <tr><td>"requestminimum"</td><td>Optional flag indicating whether the job run will be minimal or not ("true" means minimal)</td></tr>
</table>
<p></p>
<p>Each enumeration object describes an array of integers using the form:</p>
@@ -434,16 +439,21 @@
<section>
<title>Control by direct code</title>
<p></p>
- <p>Control by direct java code is quite a reasonable thing to do. The sources of the above commands should give a pretty clear idea how to proceed, if that's what you want to do.</p>
+ <p>Control by direct java code is quite a reasonable thing to do. The sources of the above commands should give a pretty clear idea how to proceed, if that's what you
+ want to do.</p>
<p></p>
<p></p>
</section>
<section>
<title>Caveats</title>
<p></p>
- <p>The above commands know nothing about the differences between connection types. Instead, they deal with configuration and specification information in the form of XML documents. Normally, these XML documents are hidden from a system integrator, unless they happen to look into the database with a tool such as psql. But the API commands above often will require such XML documents to be included as part of the command execution.</p>
+ <p>The above commands know nothing about the differences between connection types. Instead, they deal with configuration and specification information in the
+ form of XML documents. Normally, these XML documents are hidden from a system integrator, unless they happen to look into the database with a tool such as
+ psql. But the API commands above often will require such XML documents to be included as part of the command execution.</p>
<p></p>
- <p>This has one major consequence. Any application that would manipulate connections and jobs directly cannot be connection-type independent - these applications must know the proper form of XML to submit to the command. So, it is not possible to use these command APIs to write one's own UI wrapper, without sacrificing some of the repository independence that ManifoldCF by itself maintains.</p>
+ <p>This has one major consequence. Any application that would manipulate connections and jobs directly cannot be connection-type independent - these
+ applications must know the proper form of XML to submit to the command. So, it is not possible to use these command APIs to write one's own UI wrapper,
+ without sacrificing some of the repository independence that ManifoldCF by itself maintains.</p>
</section>
</section>
</body>
diff --git a/site/src/documentation/content/xdocs/ja_JP/writing-repository-connectors.xml b/site/src/documentation/content/xdocs/ja_JP/writing-repository-connectors.xml
index e8572d1..d7b9b39 100644
--- a/site/src/documentation/content/xdocs/ja_JP/writing-repository-connectors.xml
+++ b/site/src/documentation/content/xdocs/ja_JP/writing-repository-connectors.xml
@@ -73,8 +73,9 @@
<li>LiveLink (demonstrates use of local keystore infrastructure)</li>
<li>Meridio (local keystore, web services, result sets)</li>
<li>SharePoint (local keystore, web services)</li>
- <li>RSS (local keystore, binning)</li>
- <li>Web (local database schema, local keystore, binning, events and prerequisites, cache management)</li>
+ <li>RSS (local keystore, binning, fuzzy xml parsing)</li>
+ <li>Web (local database schema, local keystore, binning, events and prerequisites, cache management, fuzzy xml parsing)</li>
+ <li>Wiki (binning, rigorous xml parsing)</li>
</ul>
<p></p>
<p>You will also note that all of these connectors extend a framework-provided repository connector base class, found at <em>org.apache.manifoldcf.crawler.connectors.BaseRepositoryConnector</em>. This base class furnishes some basic bookkeeping logic for managing the connector pool, as well as default implementations of some of the less typical functionality a connector may have. For example, connectors are allowed to have database tables of their own, which are instantiated when the connector is registered, and are torn down when the connector is removed. This is, however, not very typical, and the base implementation reflects that.</p>
@@ -109,15 +110,24 @@
<p></p>
<table>
<tr><th>Model</th><th>Description</th></tr>
+ <tr><td><em>MODEL_ALL</em></td><td>The <strong>addSeedDocuments()</strong> method supplies all specified documents on each call</td></tr>
+ <tr><td><em>MODEL_PARTIAL</em></td><td>The <strong>addSeedDocuments()</strong> does not return a complete list of documents that match the criteria and time interval, because some of those documents are no longer discoverable</td></tr>
<tr><td><em>MODEL_ADD</em></td><td>The <strong>addSeedDocuments()</strong> method supplies at least all the matching documents that have been added to the repository, within the specified time interval</td></tr>
<tr><td><em>MODEL_ADD_CHANGE</em></td><td>The <strong>addSeedDocuments()</strong> method supplies at least those matching documents that have been added or changed in the repository, within the specified time interval</td></tr>
<tr><td><em>MODEL_ADD_CHANGE_DELETE</em></td><td>The <strong>addSeedDocuments()</strong> method supplies at least those matching documents that have been added, changed, or removed in the repository, within the specified time interval</td></tr>
- <tr><td><em>MODEL_PARTIAL</em></td><td>The <strong>addSeedDocuments()</strong> does not return a complete list of documents that match the criteria and time interval, because some of those documents are no longer discoverable</td></tr>
+ <tr><td><em>MODEL_CHAINED_ADD</em></td><td>The <strong>addSeedDocuments()</strong> method, plus documents reachable by discovery from seeds, supplies at least all the matching documents that have been added to the repository, within the specified time interval</td></tr>
+ <tr><td><em>MODEL_CHAINED_ADD_CHANGE</em></td><td>The <strong>addSeedDocuments()</strong> method, plus documents reachable by discovery from seeds, supplies at least those matching documents that have been added or changed in the repository, within the specified time interval</td></tr>
+ <tr><td><em>MODEL_CHAINED_ADD_CHANGE_DELETE</em></td><td>The <strong>addSeedDocuments()</strong> method, plus documents reachable by discovery from seeds, supplies at least those matching documents that have been added, changed, or removed in the repository, within the specified time interval</td></tr>
</table>
<p></p>
- <p>Note that the choice of model is actually much more subtle than the above description might indicate. It may, for one thing, be affected by characteristics of the repository, such as whether the repository considers a document to have been changed if its security information was changed. This would mean that, even though most document changes are picked up and thus one might be tempted to declare the connector to be <em>MODEL_ADD_CHANGE</em>, the correct choice would in fact be <em>MODEL_ADD</em>.</p>
+ <p>Note that the choice of model is actually much more subtle than the above description might indicate. It may, for one thing, be affected by characteristics of
+ the repository, such as whether the repository considers a document to have been changed if its security information was changed. This would mean that,
+ even though most document changes are picked up and thus one might be tempted to declare the connector to be <em>MODEL_ADD_CHANGE</em>, the
+ correct choice would in fact be <em>MODEL_ADD</em>.</p>
<p></p>
- <p>Another subtle point is what documents the connector is actually supposed to return by means of the <strong>addSeedDocuments()</strong> method. The start time and end time parameters handed to the method do not have to be strictly adhered to, for instance; it is always okay to return more documents. It is never okay for the connector to return fewer documents than were requested, on the other hand.</p>
+ <p>Another subtle point is what documents the connector is actually supposed to return by means of the <strong>addSeedDocuments()</strong> method. The
+ start time and end time parameters handed to the method do not have to be strictly adhered to, for instance; it is always okay to return more documents. It is never
+ okay for the connector to return fewer documents than were requested, on the other hand.</p>
<p></p>
</section>
<section>
@@ -139,19 +149,34 @@
<section>
<title>Choosing the form of the document version string</title>
<p></p>
- <p>The document version string is used by ManifoldCF to determine whether or not the document or configuration changed in such a way as to require that the document be reprocessed. ManifoldCF therefore requests the version string for any document that is ready for processing, and usually does not process the document again if the returned version string agrees with the version string it has stored.</p>
+ <p>The document version string is used by ManifoldCF to determine whether or not the document or configuration changed in such a way as to require that the document
+ be reprocessed. ManifoldCF therefore requests the version string for any document that is ready for processing, and usually does not process the document again if the
+ returned version string agrees with the version string it has stored.</p>
<p></p>
- <p>Thinking about it more carefully, it is clear that what a connector writer needs to do is include everything in the version string that could potentially affect how the document gets processed. That may include the version of the document in the repository, bits of configuration information, metadata, and even access tokens (if the underlying repository versions these things independently from the document itself). Storing all of that information in the version string seems like a lot - but the string is unlimited in length, and it actually serves another useful purpose to do it that way. Specifically, when it comes time to do the actual processing, it's often the correct thing to do to obtain the necessary data out of the version string, rather than calculating it or fetching it anew. That way of working guarantees that the document processing was done in a manner that agrees with its recorded version string, thus eliminating any chance of ManifoldCF getting confused.</p>
+ <p>Thinking about it more carefully, it is clear that what a connector writer needs to do is include everything in the version string that could potentially affect how the
+ document gets processed. That may include the version of the document in the repository, bits of configuration information, metadata, and even access tokens (if the
+ underlying repository versions these things independently from the document itself). Storing all of that information in the version string seems like a lot - but the string
+ is unlimited in length, and it actually serves another useful purpose to do it that way. Specifically, when it comes time to do the actual processing, it's often the correct
+ thing to do to obtain the necessary data out of the version string, rather than calculating it or fetching it anew. That way of working guarantees that the document
+ processing was done in a manner that agrees with its recorded version string, thus eliminating any chance of ManifoldCF getting confused.</p>
<p></p>
- <p>For longer data that needs to persist between the <strong>getDocumentVersions()</strong> method call and the <strong>processDocuments()</strong> method call, the connector is welcome to save this information in a temporary disk file. To help make sure nothing leaks which this approach is used, the IRepositoryConnector interface has a method that will be called to clean up any temporary files that might have been created in the handling of a given document identifier.</p>
+ <p>For longer data that needs to persist between the <strong>getDocumentVersions()</strong> method call and the <strong>processDocuments()</strong> method
+ call, the connector is welcome to save this information in a temporary disk file. To help make sure nothing leaks which this approach is used, the IRepositoryConnector
+ interface has a method that will be called to clean up any temporary files that might have been created in the handling of a given document identifier.</p>
<p></p>
</section>
<section>
<title>Notes on connector UI methods</title>
<p></p>
- <p>The crawler UI uses a tabbed layout structure, and thus each of these elements must properly implement the tabbed model. This means that the "header" methods above must add the desired tab names to a specified array, and the "body" methods must provide appropriate HTML which handles both the case where a tab is displayed, and where it is not displayed. Also, it makes sense to use the appropriate css definitions, so that the connector UI pages have a similar look-and-feel to the rest of ManifoldCF's crawler ui. We strongly suggest starting with one of the supplied connector's UI code, both for a description of the arguments to each page, and for some decent ideas of ways to organize your connector's UI code. </p>
+ <p>The crawler UI uses a tabbed layout structure, and thus each of these elements must properly implement the tabbed model. This means that the "header" methods
+ above must add the desired tab names to a specified array, and the "body" methods must provide appropriate HTML which handles both the case where a tab is
+ displayed, and where it is not displayed. Also, it makes sense to use the appropriate css definitions, so that the connector UI pages have a similar look-and-feel to the
+ rest of ManifoldCF's crawler ui. We strongly suggest starting with one of the supplied connector's UI code, both for a description of the arguments to each page, and
+ for some decent ideas of ways to organize your connector's UI code. </p>
<p></p>
- <p>Please also note that it is good practice to name the form fields in your HTML in such a way that they cannot collide with form fields that may come from the framework's HTML or any specific output connector's HTML. The <em>DocumentSpecification</em> editing HTML especially may be prone to collisions, because within any given job, this HTML is included in the same page as HTML from the chosen output connector.</p>
+ <p>Please also note that it is good practice to name the form fields in your HTML in such a way that they cannot collide with form fields that may come from the
+ framework's HTML or any specific output connector's HTML. The <em>DocumentSpecification</em> editing HTML especially may be prone to collisions, because
+ within any given job, this HTML is included in the same page as HTML from the chosen output connector.</p>
<p></p>
<p></p>
</section>
@@ -159,7 +184,8 @@
<section>
<title>Implementation support provided by the framework</title>
<p></p>
- <p>ManifoldCF's framework provides a number of helpful services designed to make the creation of a connector easier. These services are summarized below. (This is not an exhaustive list, by any means.)</p>
+ <p>ManifoldCF's framework provides a number of helpful services designed to make the creation of a connector easier. These services are summarized below.
+ (This is not an exhaustive list, by any means.)</p>
<p></p>
<ul>
<li>Lock management and synchronization (see <em>org.apache.manifoldcf.core.interfaces.LockManagerFactory</em>)</li>
@@ -180,7 +206,8 @@
<section>
<title>DO's and DON'T DO's</title>
<p></p>
- <p>It's always a good idea to make use of an existing infrastructure component, if it's meant for that purpose, rather than inventing your own. There are, however, some limitations we recommend you adhere to.</p>
+ <p>It's always a good idea to make use of an existing infrastructure component, if it's meant for that purpose, rather than inventing your own. There are, however,
+ some limitations we recommend you adhere to.</p>
<p></p>
<ul>
<li>DO make use of infrastructure components described in the section above</li>
@@ -188,7 +215,8 @@
<li>NEVER write connector code that directly uses framework database tables, other than the ones installed and managed by your connector</li>
</ul>
<p></p>
- <p>If you are tempted to violate these rules, it may well mean you don't understand something important. At the very least, we'd like to know why. Send email to dev@manifoldcf.apache.org with a description of your problem and how you are tempted to solve it.</p>
+ <p>If you are tempted to violate these rules, it may well mean you don't understand something important. At the very least, we'd like to know why. Send email
+ to dev@manifoldcf.apache.org with a description of your problem and how you are tempted to solve it.</p>
</section>
</section>
</body>
diff --git a/site/src/documentation/resources/images/en_US/job-status.PNG b/site/src/documentation/resources/images/en_US/job-status.PNG
index 8c85e7a..9b71346 100644
--- a/site/src/documentation/resources/images/en_US/job-status.PNG
+++ b/site/src/documentation/resources/images/en_US/job-status.PNG
Binary files differ
diff --git a/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityTester.java b/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityTester.java
index 3c24428..0407efe 100644
--- a/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityTester.java
+++ b/tests/filesystem/src/test/java/org/apache/manifoldcf/filesystem_tests/SanityTester.java
@@ -131,11 +131,11 @@
if (status.getDocumentsProcessed() != 5)
throw new ManifoldCFException("Wrong number of documents processed - expected 5, saw "+new Long(status.getDocumentsProcessed()).toString());
- // Add a file and recrawl
+ // Add a file and recrawl using minimal crawl
FileHelper.createFile(new File("testdata/testdir/test4.txt"),"Added file");
// Now, start the job, and wait until it completes.
- jobManager.manualStart(job.getID());
+ jobManager.manualStart(job.getID(),true);
instance.waitJobInactiveNative(jobManager,job.getID(),120000L);
status = jobManager.getStatus(job.getID());
@@ -143,11 +143,11 @@
if (status.getDocumentsProcessed() != 6)
throw new ManifoldCFException("Wrong number of documents processed after add - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString());
- // Change a file, and recrawl
+ // Change a file, and recrawl, once again using minimal
FileHelper.changeFile(new File("testdata/test1.txt"),"Modified contents");
// Now, start the job, and wait until it completes.
- jobManager.manualStart(job.getID());
+ jobManager.manualStart(job.getID(),true);
instance.waitJobInactiveNative(jobManager,job.getID(),120000L);
status = jobManager.getStatus(job.getID());
@@ -159,8 +159,17 @@
// Delete a file, and recrawl
FileHelper.removeFile(new File("testdata/test2.txt"));
-
- // Now, start the job, and wait until it completes.
+
+ // Do a minimal recrawl first; the delete should not be picked up.
+ jobManager.manualStart(job.getID(),true);
+ instance.waitJobInactiveNative(jobManager,job.getID(),120000L);
+
+ status = jobManager.getStatus(job.getID());
+ // The test data area has 4 documents and one directory, and we have to count the root directory too.
+ if (status.getDocumentsProcessed() != 6)
+ throw new ManifoldCFException("Wrong number of documents processed after delete with minimal crawl - expected 6, saw "+new Long(status.getDocumentsProcessed()).toString());
+
+ // Now, do a complete crawl - the delete should be found now.
jobManager.manualStart(job.getID());
instance.waitJobInactiveNative(jobManager,job.getID(),120000L);
