SLING-5882 ThreadExpiringThreadPoolTest fails 8% of time. Fixed. Now < 0.1%

git-svn-id: https://svn.apache.org/repos/asf/sling/trunk@1753662 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/src/test/java/org/apache/sling/commons/threads/impl/ThreadExpiringThreadPoolTest.java b/src/test/java/org/apache/sling/commons/threads/impl/ThreadExpiringThreadPoolTest.java
index 80c0e4f..28db33b 100644
--- a/src/test/java/org/apache/sling/commons/threads/impl/ThreadExpiringThreadPoolTest.java
+++ b/src/test/java/org/apache/sling/commons/threads/impl/ThreadExpiringThreadPoolTest.java
@@ -51,11 +51,98 @@
 
     private static final Logger LOG = LoggerFactory.getLogger(ThreadExpiringThreadPoolTest.class);
 
-    private static final int MAX_THREAD_AGE_MS = 15; // let threads expire after this many ms
+    private static final int MAX_THREAD_AGE_MS = 90; // let threads expire after this many ms
 
     @Rule
     public ThreadPoolContext context = new ThreadPoolContext();
 
+
+    /**
+     * Attempts to isolate failures that happen > 0.2% of the time related to the
+     * way in which the underlying thread pool behaves. This is not normally run as
+     * a test , but use it if you want to isolate a rare failure.
+     */
+    //@Test
+    public void shouldLetMultipleThreadsDieAfterExpiryMulti() {
+        int fail = 0;
+        int success = 0;
+        for (int i = 0; i < 500; i++) {
+            try {
+                LOG.info("Running {} ", i);
+                context = new ThreadPoolContext();
+                context.before();
+                shouldLetMultipleThreadsDieAfterExpiry();
+                success++;
+            } catch (Throwable e) {
+                LOG.error("Failed ", e);
+                fail++;
+                fail("Race condition encountered");
+            } finally {
+                context.after();
+            }
+        }
+        LOG.info("Failed {} sucess {}", fail, success);
+        assertEquals(0, fail);
+    }
+    /**
+     * Attempts to isolate failures that happen > 0.2% of the time related to the
+     * way in which the underlying thread pool behaves. This is not normally run as
+     * a test, but use it if you want to isolate a rare failure.
+     */
+    // @Test
+    public void shouldCreateNewThreadAfterExpiryMulti() {
+
+        int fail = 0;
+        int success = 0;
+        for (int i = 0; i < 500; i++) {
+            try {
+                LOG.info("Running {} ", i);
+                context = new ThreadPoolContext();
+                context.before();
+                shouldCreateNewThreadAfterExpiry();
+                success++;
+            } catch (Throwable e ) {
+                LOG.error("Failed ",e);
+                fail++;
+                fail("Race condition encountered");
+            } finally {
+                context.after();
+            }
+        }
+        LOG.info("Failed {} sucess {}", fail, success);
+        assertEquals(0, fail);
+    }
+    /**
+     * Attempts to isolate failures that happen > 0.2% of the time related to the
+     * way in which the underlying thread pool behaves. This is not normally run as
+     * a test, but use it if you want to isolate a rare failure.
+     */
+    // @Test
+    public void shouldCreateNewThreadAfterExpiryForFailingTasksMulti() {
+
+        int fail = 0;
+        int success = 0;
+        for (int i = 0; i < 500; i++) {
+            try {
+                LOG.info("Running {} ", i);
+                context = new ThreadPoolContext();
+                context.before();
+                shouldCreateNewThreadAfterExpiryForFailingTasks();
+                success++;
+            } catch (Throwable e ) {
+                LOG.error("Failed ",e);
+                fail++;
+                fail("Race condition encountered");
+            } finally {
+                context.after();
+            }
+        }
+        LOG.info("Failed {} sucess {}", fail, success);
+        assertEquals(0, fail);
+
+    }
+
+
     @Test
     public void shouldCreateNewThreadAfterExpiry() throws InterruptedException, ExecutionException {
         final TrackingThreadFactory threadFactory = context.getThreadFactory();
@@ -249,6 +336,7 @@
         }
 
         public Set<String> getActiveThreads() {
+            letThreadsDie();
             final HashSet<String> active = new HashSet<String>();
             for (final Thread thread : threadHistory) {
                 if (thread.isAlive()) {
@@ -259,6 +347,7 @@
         }
 
         public Set<String> getExpiredThreads() {
+            letThreadsDie();
             final HashSet<String> expired = new HashSet<String>();
             for (final Thread thread : threadHistory) {
                 if (!thread.isAlive()) {
@@ -268,6 +357,19 @@
             return expired;
         }
 
+        /**
+         * This avoids a race condition where a thread has been evicted from the pool but is still alive becuase it evicted itself.
+         * JDK8 java.util.concurrent.ThreadPoolExecutor does this. The 15ms assumes the process takes no more than 15ms to complete.
+         * That is OS and VM dependent.
+         */
+        public void letThreadsDie() {
+            try {
+                Thread.sleep(15);
+            } catch ( Exception e) {
+                LOG.debug(e.getMessage(),e);
+            }
+        }
+
         @Override
         public Thread newThread(final Runnable r) {
             final Thread thread = new Thread(group, r, "test-thread-" + threadCount.getAndIncrement());