Port tracking code from CONNECTORS-590 to a more modern branch. 

git-svn-id: https://svn.apache.org/repos/asf/manifoldcf/branches/CONNECTORS-590-2@1464874 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
index 77670fb..f45cb0f 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
@@ -645,12 +645,14 @@
         // Clean up carrydown stuff
         carryDown.reset();
         database.performCommit();
+        TrackerClass.noteCommit();
         Logging.jobs.debug("Reset complete");
         break;
       }
       catch (ManifoldCFException e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
         {
           if (Logging.perf.isDebugEnabled())
@@ -663,6 +665,7 @@
       catch (Error e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         throw e;
       }
       finally
@@ -687,11 +690,13 @@
       {
         jobQueue.resetDocumentWorkerStatus();
         database.performCommit();
+        TrackerClass.noteCommit();
         break;
       }
       catch (ManifoldCFException e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
         {
           if (Logging.perf.isDebugEnabled())
@@ -704,6 +709,7 @@
       catch (Error e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         throw e;
       }
       finally
@@ -732,6 +738,7 @@
   {
     Logging.jobs.debug("Resetting doc deleting status");
     jobQueue.resetDocDeleteWorkerStatus();
+    TrackerClass.noteCommit();
     Logging.jobs.debug("Reset complete");
   }
 
@@ -742,6 +749,7 @@
   {
     Logging.jobs.debug("Resetting doc cleaning status");
     jobQueue.resetDocCleanupWorkerStatus();
+    TrackerClass.noteCommit();
     Logging.jobs.debug("Reset complete");
   }
 
@@ -997,6 +1005,7 @@
         }
         
         database.performCommit();
+        TrackerClass.noteCommit();
         
         if (Logging.perf.isDebugEnabled())
           Logging.perf.debug("Done pruning unindexable docs after "+new Long(System.currentTimeMillis()-startTime).toString()+" ms.");
@@ -1007,11 +1016,13 @@
       catch (Error e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         throw e;
       }
       catch (ManifoldCFException e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
         {
           if (Logging.perf.isDebugEnabled())
@@ -1258,6 +1269,7 @@
         }
 
         database.performCommit();
+        TrackerClass.noteCommit();
         
         if (Logging.perf.isDebugEnabled())
           Logging.perf.debug("Done pruning unindexable docs after "+new Long(System.currentTimeMillis()-startTime).toString()+" ms.");
@@ -1268,11 +1280,13 @@
       catch (Error e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         throw e;
       }
       catch (ManifoldCFException e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
         {
           if (Logging.perf.isDebugEnabled())
@@ -1821,6 +1835,7 @@
         }
 
         database.performCommit();
+        TrackerClass.noteCommit();
         
         return new DocumentSetAndFlags(rval, rvalBoolean);
 
@@ -1828,6 +1843,7 @@
       catch (ManifoldCFException e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
         {
           if (Logging.perf.isDebugEnabled())
@@ -1840,6 +1856,7 @@
       catch (Error e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         throw e;
       }
       finally
@@ -2442,11 +2459,13 @@
           i++;
         }
         database.performCommit();
+        TrackerClass.noteCommit();
         break;
       }
       catch (ManifoldCFException e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
         {
           if (Logging.perf.isDebugEnabled())
@@ -2460,6 +2479,7 @@
       catch (Error e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         throw e;
       }
       finally
@@ -2607,11 +2627,13 @@
         // we don't delete them here.
         
         database.performCommit();
+        TrackerClass.noteCommit();
         return rval;
       }
       catch (ManifoldCFException e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
         {
           if (Logging.perf.isDebugEnabled())
@@ -2625,6 +2647,7 @@
       catch (Error e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         throw e;
       }
       finally
@@ -2987,16 +3010,19 @@
         }
 
         database.performCommit();
+        TrackerClass.noteCommit();
         break;
       }
       catch (Error e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         throw e;
       }
       catch (ManifoldCFException e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
         {
           if (Logging.perf.isDebugEnabled())
@@ -3194,11 +3220,13 @@
         }
 
         database.performCommit();
+        TrackerClass.noteCommit();
         break;
       }
       catch (ManifoldCFException e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
         {
           if (Logging.perf.isDebugEnabled())
@@ -3211,6 +3239,7 @@
       catch (Error e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         throw e;
       }
       finally
@@ -3289,11 +3318,13 @@
         }
 
         database.performCommit();
+        TrackerClass.noteCommit();
         break;
       }
       catch (ManifoldCFException e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
         {
           if (Logging.perf.isDebugEnabled())
@@ -3306,6 +3337,7 @@
       catch (Error e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         throw e;
       }
       finally
@@ -3524,6 +3556,7 @@
           hopCount.recordSeedReferences(jobID,legalLinkTypes,reorderedDocIDHashes,hopcountMethod);
 
         database.performCommit();
+        TrackerClass.noteCommit();
         
         if (Logging.perf.isDebugEnabled())
           Logging.perf.debug("Took "+new Long(System.currentTimeMillis()-startTime).toString()+" ms to add "+Integer.toString(reorderedDocIDHashes.length)+
@@ -3544,6 +3577,7 @@
       catch (ManifoldCFException e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
         {
           if (Logging.perf.isDebugEnabled())
@@ -3557,6 +3591,7 @@
       catch (Error e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         throw e;
       }
       finally
@@ -4154,6 +4189,7 @@
           jobQueue.reactivateHopcountRemovedRecords(jobID);
 
         database.performCommit();
+        TrackerClass.noteCommit();
         
         if (Logging.perf.isDebugEnabled())
           Logging.perf.debug("Took "+new Long(System.currentTimeMillis()-startTime).toString()+" ms to add "+Integer.toString(reorderedDocIDHashes.length)+
@@ -4173,6 +4209,7 @@
       catch (ManifoldCFException e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
         {
           sleepAmt = getRandomAmount();
@@ -4186,6 +4223,7 @@
       catch (Error e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         throw e;
       }
       finally
@@ -5377,6 +5415,7 @@
     // No special treatment needed for hopcount or carrydown, since these all get deleted at once
     // at the end of the job delete process.
     jobQueue.prepareDeleteScan(jobID);
+    TrackerClass.noteCommit();
   }
   
   /** Prepare a job to be run.
@@ -5469,11 +5508,13 @@
 
         jobQueue.prepareFullScan(jobID);
         database.performCommit();
+        TrackerClass.noteCommit();
         break;
       }
       catch (ManifoldCFException e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
         {
           if (Logging.perf.isDebugEnabled())
@@ -5486,6 +5527,7 @@
       catch (Error e)
       {
         database.signalRollback();
+        TrackerClass.noteRollback();
         throw e;
       }
       finally
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
index 60f0f07..b59a918 100644
--- a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
@@ -390,6 +390,8 @@
     // Reindex the jobqueue table, since we've probably made lots of bad tuples doing the above operations.
     reindexTable();
     unconditionallyAnalyzeTables();
+
+    TrackerClass.noteGlobalEvent("Restart");
   }
 
   /** Flip all records for a job that have status HOPCOUNTREMOVED back to PENDING.
@@ -407,6 +409,8 @@
       new UnitaryClause(jobIDField,jobID),
       new UnitaryClause(statusField,statusToString(STATUS_HOPCOUNTREMOVED))});
     performUpdate(map,"WHERE "+query,list,null);
+    
+    TrackerClass.noteJobEvent(jobID,"Map HOPCOUNTREMOVED to PENDING");
   }
 
   /** Delete all records for a job that have status HOPCOUNTREMOVED.
@@ -464,6 +468,8 @@
         statusToString(STATUS_ACTIVEPURGATORY),
         statusToString(STATUS_ACTIVENEEDRESCANPURGATORY)})});
     performUpdate(map,"WHERE "+query,list,null);
+        
+    TrackerClass.noteGlobalEvent("Reset document worker status");
   }
 
   /** Reset doc delete worker status.
@@ -479,6 +485,8 @@
     String query = buildConjunctionClause(list,new ClauseDescription[]{
       new UnitaryClause(statusField,statusToString(STATUS_BEINGDELETED))});
     performUpdate(map,"WHERE "+query,list,null);
+      
+    TrackerClass.noteGlobalEvent("Reset document delete worker status");
   }
 
   /** Reset doc cleaning worker status.
@@ -494,6 +502,8 @@
     String query = buildConjunctionClause(list,new ClauseDescription[]{
       new UnitaryClause(statusField,statusToString(STATUS_BEINGCLEANED))});
     performUpdate(map,"WHERE "+query,list,null);
+      
+    TrackerClass.noteGlobalEvent("Reset document cleanup worker status");
   }
 
   /** Prepare for a job delete pass.  This will not be called
@@ -545,6 +555,7 @@
     // Do an analyze, otherwise our plans are going to be crap right off the bat
     unconditionallyAnalyzeTables();
 
+    TrackerClass.noteJobEvent(jobID,"Prepare delete scan");
   }
   
   /** Prepare for a "full scan" job.  This will not be called
@@ -595,6 +606,8 @@
     noteModifications(0,2,0);
     // Do an analyze, otherwise our plans are going to be crap right off the bat
     unconditionallyAnalyzeTables();
+        
+    TrackerClass.noteJobEvent(jobID,"Prepare full scan");
   }
 
   /** Prepare for a "partial" job.  This is called ONLY when the job is inactive.
@@ -654,6 +667,8 @@
     noteModifications(0,1,0);
     // Do an analyze, otherwise our plans are going to be crap right off the bat
     unconditionallyAnalyzeTables();
+      
+    TrackerClass.noteJobEvent(jobID,"Prepare incremental scan");
   }
 
   /** Delete ingested document identifiers (as part of deleting the owning job).
@@ -767,6 +782,7 @@
       // Leave doc priority unchanged.
       break;
     default:
+      TrackerClass.printForensics(recID, currentStatus);
       throw new ManifoldCFException("Unexpected jobqueue status - record id "+recID.toString()+", expecting active status, saw "+Integer.toString(currentStatus));
     }
 
@@ -779,6 +795,8 @@
     String query = buildConjunctionClause(list,new ClauseDescription[]{
       new UnitaryClause(idField,recID)});
     performUpdate(map,"WHERE "+query,list,null);
+      
+    TrackerClass.noteRecordEvent(recID, newStatus, "Note completion");
   }
 
   /** Either mark a record as hopcountremoved, or set status to "rescan", depending on the
@@ -814,6 +832,7 @@
       // Leave doc priority unchanged.
       break;
     default:
+      TrackerClass.printForensics(recID, currentStatus);
       throw new ManifoldCFException("Unexpected jobqueue status - record id "+recID.toString()+", expecting active status, saw "+Integer.toString(currentStatus));
     }
 
@@ -826,6 +845,7 @@
     String query = buildConjunctionClause(list,new ClauseDescription[]{
       new UnitaryClause(idField,recID)});
     performUpdate(map,"WHERE "+query,list,null);
+    TrackerClass.noteRecordEvent(recID, newStatus, "Update or hopcount remove");
     return rval;
   }
 
@@ -846,6 +866,7 @@
       newStatus = STATUS_ACTIVEPURGATORY;
       break;
     default:
+      TrackerClass.printForensics(id, currentStatus);
       throw new ManifoldCFException("Unexpected status value for jobqueue record "+id.toString()+"; got "+Integer.toString(currentStatus));
     }
 
@@ -856,6 +877,7 @@
       new UnitaryClause(idField,id)});
     performUpdate(map,"WHERE "+query,list,null);
     noteModifications(0,1,0);
+    TrackerClass.noteRecordEvent(id, newStatus, "Make active");
   }
 
   /** Set the status on a record, including check time and priority.
@@ -887,6 +909,7 @@
       new UnitaryClause(idField,id)});
     performUpdate(map,"WHERE "+query,list,null);
     noteModifications(0,1,0);
+    TrackerClass.noteRecordEvent(id, status, "Set status");
   }
 
   /** Set the status of a document to "being deleted".
@@ -901,6 +924,7 @@
       new UnitaryClause(idField,id)});
     performUpdate(map,"WHERE "+query,list,null);
     noteModifications(0,1,0);
+    TrackerClass.noteRecordEvent(id, STATUS_BEINGDELETED, "Set deleting status");
   }
 
   /** Set the status of a document to be "no longer deleting" */
@@ -918,6 +942,7 @@
       new UnitaryClause(idField,id)});
     performUpdate(map,"WHERE "+query,list,null);
     noteModifications(0,1,0);
+    TrackerClass.noteRecordEvent(id, STATUS_ELIGIBLEFORDELETE, "Set undeleting status");
   }
 
   /** Set the status of a document to "being cleaned".
@@ -932,6 +957,7 @@
       new UnitaryClause(idField,id)});
     performUpdate(map,"WHERE "+query,list,null);
     noteModifications(0,1,0);
+    TrackerClass.noteRecordEvent(id, STATUS_BEINGCLEANED, "Set cleaning status");
   }
 
   /** Set the status of a document to be "no longer cleaning" */
@@ -949,6 +975,7 @@
       new UnitaryClause(idField,id)});
     performUpdate(map,"WHERE "+query,list,null);
     noteModifications(0,1,0);
+    TrackerClass.noteRecordEvent(id, STATUS_PURGATORY, "Set uncleaning status");
   }
 
   /** Remove multiple records entirely.
@@ -1040,6 +1067,7 @@
     case STATUS_PURGATORY:
       // Set the status and time both
       map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
+      TrackerClass.noteRecordEvent(recordID, STATUS_PENDINGPURGATORY, "Update existing record initial");
       if (desiredExecuteTime == -1L)
         map.put(checkTimeField,new Long(0L));
       else
@@ -1130,6 +1158,7 @@
     performInsert(map,null);
     prereqEventManager.addRows(recordID,prereqEvents);
     noteModifications(1,0,0);
+    TrackerClass.noteRecordEvent(recordID, STATUS_PENDING, "Create initial");
   }
 
   /** Note the remaining documents that do NOT need to be queued.  These are noted so that the
@@ -1317,6 +1346,7 @@
     case STATUS_UNCHANGED:
       // Set the status and time both
       map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
+      TrackerClass.noteRecordEvent(recordID, STATUS_PENDINGPURGATORY, "Update existing");
       map.put(checkTimeField,new Long(desiredExecuteTime));
       map.put(checkActionField,actionToString(ACTION_RESCAN));
       map.put(failTimeField,null);
@@ -1334,6 +1364,7 @@
         // The document has been processed before, so it has to go into PENDINGPURGATORY.
         // Set the status and time both
         map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
+        TrackerClass.noteRecordEvent(recordID, STATUS_PENDINGPURGATORY, "Update existing");
         map.put(checkTimeField,new Long(desiredExecuteTime));
         map.put(checkActionField,actionToString(ACTION_RESCAN));
         map.put(failTimeField,null);
@@ -1363,6 +1394,7 @@
         // Flip the state to the new one, and set the document priority at this time too - it will be preserved when the
         // processing is completed.
         map.put(statusField,statusToString(STATUS_ACTIVENEEDRESCAN));
+        TrackerClass.noteRecordEvent(recordID, STATUS_ACTIVENEEDRESCAN, "Update existing");
         map.put(checkTimeField,new Long(desiredExecuteTime));
         map.put(checkActionField,actionToString(ACTION_RESCAN));
         map.put(failTimeField,null);
@@ -1387,6 +1419,7 @@
         // Flip the state to the new one, and set the document priority at this time too - it will be preserved when the
         // processing is completed.
         map.put(statusField,statusToString(STATUS_ACTIVENEEDRESCANPURGATORY));
+        TrackerClass.noteRecordEvent(recordID, STATUS_ACTIVENEEDRESCANPURGATORY, "Update existing");
         map.put(checkTimeField,new Long(desiredExecuteTime));
         map.put(checkActionField,actionToString(ACTION_RESCAN));
         map.put(failTimeField,null);
@@ -1454,6 +1487,8 @@
     performInsert(map,null);
     prereqEventManager.addRows(recordID,prereqEvents);
     noteModifications(1,0,0);
+    TrackerClass.noteRecordEvent(recordID, STATUS_PENDING, "Create new");
+
   }
 
   // Methods to convert status strings to integers and back
diff --git a/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/TrackerClass.java b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/TrackerClass.java
new file mode 100644
index 0000000..90c1c2a
--- /dev/null
+++ b/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/TrackerClass.java
@@ -0,0 +1,287 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.manifoldcf.crawler.jobs;
+
+import java.util.*;
+import java.io.*;
+
+/** Debugging class to keep track of recent modifications to the jobqueue table,
+* along with context as to where it occurred.  If a jobqueue state error occurs,
+* we can then print out all of the pertinent history and find the culprit.
+*/
+public class TrackerClass
+{
+  // The goal of this class is to keep track of at least some of the history
+  // potentially affecting each record.
+  protected final static long HISTORY_LENGTH = 60000L * 15;     // 15 minutes
+
+  // Active transaction
+  protected final static Map<String,TransactionData> transactionData = new HashMap<String,TransactionData>();
+  
+  // Modification history
+  protected final static List<TransactionData> history = new ArrayList<TransactionData>();
+  
+  // Place where we keep track of individual modifications
+  private TrackerClass()
+  {
+  }
+  
+  /** Add a single record event, as yet uncommitted */
+  public static void noteRecordEvent(Long recordID, int newStatus, String description)
+  {
+    addEvent(new RecordEvent(recordID, newStatus, new Exception(description)));
+  }
+  
+  /** Add a global event, as yet uncommitted, which has the potential
+  * to affect any record's state in a given job.
+  */
+  public static void noteJobEvent(Long jobID, String description)
+  {
+    addEvent(new JobEvent(jobID, new Exception(description)));
+  }
+  
+  /** Add a global event, as yet uncommitted, which has the potential
+  * to affect the state of any record.
+  */
+  public static void noteGlobalEvent(String description)
+  {
+    addEvent(new GlobalEvent(new Exception(description)));
+  }
+  
+  protected static void addEvent(HistoryRecord hr)
+  {
+    String threadName = Thread.currentThread().getName();
+    TransactionData td;
+    synchronized (transactionData)
+    {
+      td = transactionData.get(threadName);
+      if (td == null)
+      {
+        td = new TransactionData();
+        transactionData.put(threadName,td);
+      }
+    }
+    td.addEvent(hr);
+  }
+  
+  /** Note a commit operation.
+  */
+  public static void noteCommit()
+  {
+    long currentTime = System.currentTimeMillis();
+    String threadName = Thread.currentThread().getName();
+    TransactionData td;
+    synchronized (transactionData)
+    {
+      td = transactionData.get(threadName);
+      transactionData.remove(threadName);
+    }
+    if (td == null)
+      return;
+    // Only keep stuff around for an hour
+    long removalCutoff = currentTime - HISTORY_LENGTH;
+    synchronized (history)
+    {
+      history.add(td);
+      // Clean out older records
+      while (history.size() > 0)
+      {
+        TransactionData td2 = history.get(0);
+        if (td2.isFlushable(removalCutoff))
+          history.remove(0);
+        else
+          break;
+      }
+    }
+    
+  }
+  
+  /** Note a rollback operation.
+  */
+  public static void noteRollback()
+  {
+    String threadName = Thread.currentThread().getName();
+    synchronized (transactionData)
+    {
+      transactionData.remove(threadName);
+    }
+  }
+  
+  public static void printForensics(Long recordID, int existingStatus)
+  {
+    synchronized (transactionData)
+    {
+      synchronized (history)
+      {
+        System.err.println("---- Forensics for record "+recordID+", current status: "+existingStatus+" ----");
+        System.err.println("--Current stack trace--");
+        StringWriter sw = new StringWriter();
+        new Exception("Unexpected jobqueue status").printStackTrace(new PrintWriter(sw,true));
+        System.err.print(sw.toString());
+        System.err.println("--Active transactions--");
+        for (String threadName : transactionData.keySet())
+        {
+          for (HistoryRecord hr : transactionData.get(threadName).getEvents())
+          {
+            if (hr.applies(recordID))
+            {
+              System.err.println("Thread '"+threadName+"' was active:");
+              hr.print();
+            }
+          }
+        }
+        System.err.println("--Pertinent History--");
+        for (TransactionData td : history)
+        {
+          for (HistoryRecord hr : td.getEvents())
+          {
+            if (hr.applies(recordID))
+            {
+              hr.print();
+            }
+          }
+        }
+      }
+    }
+        
+  }
+  
+  protected static class TransactionData
+  {
+    protected final List<HistoryRecord> transactionEvents = new ArrayList<HistoryRecord>();
+    protected long timestamp;
+    
+    public TransactionData()
+    {
+      timestamp = System.currentTimeMillis();
+    }
+    
+    public void addEvent(HistoryRecord event)
+    {
+      transactionEvents.add(event);
+    }
+    
+    public List<HistoryRecord> getEvents()
+    {
+      return transactionEvents;
+    }
+    
+    public boolean isFlushable(long cutoffTime)
+    {
+      return cutoffTime > timestamp;
+    }
+  }
+  
+  protected abstract static class HistoryRecord
+  {
+    protected long timestamp;
+    protected Exception trace;
+    
+    public HistoryRecord(Exception trace)
+    {
+      this.trace = trace;
+      this.timestamp = System.currentTimeMillis();
+    }
+    
+    public void print()
+    {
+      System.err.println("  at "+new Long(timestamp)+", location: ");
+      StringWriter sw = new StringWriter();
+      trace.printStackTrace(new PrintWriter(sw,true));
+      System.err.print(sw.toString());
+    }
+    
+    public abstract boolean applies(Long recordID);
+    
+  }
+  
+  protected static class RecordEvent extends HistoryRecord
+  {
+    protected Long recordID;
+    protected int newStatus;
+    
+    public RecordEvent(Long recordID, int newStatus, Exception trace)
+    {
+      super(trace);
+      this.recordID = recordID;
+      this.newStatus = newStatus;
+    }
+    
+    @Override
+    public void print()
+    {
+      System.err.println("Record "+recordID+" status modified to "+newStatus);
+      super.print();
+    }
+    
+    @Override
+    public boolean applies(Long recordID)
+    {
+      return this.recordID.equals(recordID);
+    }
+
+  }
+  
+  protected static class JobEvent extends HistoryRecord
+  {
+    protected Long jobID;
+    
+    public JobEvent(Long jobID, Exception trace)
+    {
+      super(trace);
+      this.jobID = jobID;
+    }
+    
+    @Override
+    public void print()
+    {
+      System.err.println("All job related records modified for job "+jobID);
+      super.print();
+    }
+    
+    @Override
+    public boolean applies(Long recordID)
+    {
+      return true;
+    }
+  }
+  
+  protected static class GlobalEvent extends HistoryRecord
+  {
+    public GlobalEvent(Exception trace)
+    {
+      super(trace);
+    }
+    
+    @Override
+    public void print()
+    {
+      System.err.println("All records modified");
+      super.print();
+    }
+
+    @Override
+    public boolean applies(Long recordID)
+    {
+      return true;
+    }
+  }
+  
+}