Merge branch 'bidir-deadlock' into 'ibm-trunk'

Bidir deadlock

See merge request !49
diff --git a/yoko-core/src/main/java/org/apache/yoko/orb/OB/GIOPConnection.java b/yoko-core/src/main/java/org/apache/yoko/orb/OB/GIOPConnection.java
index cea456e..a095a6f 100644
--- a/yoko-core/src/main/java/org/apache/yoko/orb/OB/GIOPConnection.java
+++ b/yoko-core/src/main/java/org/apache/yoko/orb/OB/GIOPConnection.java
@@ -23,6 +23,7 @@
 import org.omg.SendingContext.CodeBase;
 
 abstract public class GIOPConnection implements DowncallEmitter, UpcallReturn {
+    static final java.util.logging.Logger logger = java.util.logging.Logger.getLogger(GIOPConnection.class.getName());
     // ----------------------------------------------------------------
     // Inner classes
     // ----------------------------------------------------------------
@@ -1440,8 +1441,10 @@
     public void setState(int newState) {
         synchronized (this) {
             if (state_ == newState
-                    || (state_ != State.Holding && newState < state_))
+                    || (state_ != State.Holding && newState < state_)) {
+                logger.fine("No state change from " +state_  + " to "  + newState);
                 return;
+            }
 
             //
             // make sure to update the state since some of the actions
diff --git a/yoko-core/src/main/java/org/apache/yoko/orb/OB/GIOPConnectionThreaded.java b/yoko-core/src/main/java/org/apache/yoko/orb/OB/GIOPConnectionThreaded.java
index 5f80336..269edf5 100644
--- a/yoko-core/src/main/java/org/apache/yoko/orb/OB/GIOPConnectionThreaded.java
+++ b/yoko-core/src/main/java/org/apache/yoko/orb/OB/GIOPConnectionThreaded.java
@@ -17,81 +17,55 @@
 
 package org.apache.yoko.orb.OB;
 
+import static java.util.concurrent.TimeUnit.SECONDS;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
 public final class GIOPConnectionThreaded extends GIOPConnection {
     static final Logger logger = Logger.getLogger(GIOPConnectionThreaded.class.getName());
+    
     // ----------------------------------------------------------------
     // Inner helper classes
     // ----------------------------------------------------------------
 
-    //
-    // thread to handle connection shutdown
-    // 
-    public final class ShutdownThread extends Thread {
-        private GIOPConnectionThreaded parent_;
 
-        ShutdownThread(ThreadGroup group, GIOPConnectionThreaded parent) {
-            super(group, "Yoko:GIOPConnectionThreaded:ShutdownThread");
-            parent_ = parent;
-        }
+    public final class Shutdown implements Runnable {
 
         public void run() {
             try {
-                parent_.execShutdown();
+                execShutdown();
             } catch (RuntimeException ex) {
                 Assert._OB_assert(ex);
             }
-
-            //
-            // break cyclic dependency with parent
-            //
-            parent_ = null;
         }
     }
 
-    //
-    // thread to handle reception of messages
-    // 
-    public final class ReceiverThread extends Thread {
-        private GIOPConnectionThreaded parent_;
-
-        ReceiverThread(ThreadGroup group, GIOPConnectionThreaded parent) {
-            super(group, "Yoko:GIOPConnectionThreaded:ReceiverThread");
-            parent_ = parent;
+    public final class Receiver implements Runnable {
+        Receiver() {
+            receiverLock.readLock().lock();
         }
-
+        
         public void run() {
             try {
-                parent_.execReceive();
+                execReceive();
             } catch (RuntimeException ex) {
                 Assert._OB_assert(ex);
+            } finally {
+                receiverLock.readLock().unlock();
             }
-
-            //
-            // break cyclic dependency with parent
-            //
-            parent_ = null;
         }
     }
-
     // ----------------------------------------------------------------
     // Member data
     // ----------------------------------------------------------------
     // 
 
     //
-    // the shutdown thread handle
-    //
-    protected Thread shutdownThread_ = null;
-
-    //
-    // the list of receiver threads
-    //
-    protected java.util.LinkedList receiverThreads_ = new java.util.LinkedList();
-
-    //
     // the holding monitor to pause the receiver threads
     //
     protected java.lang.Object holdingMonitor_ = new java.lang.Object();
@@ -105,6 +79,9 @@
     // sending mutex to prevent multiple threads from sending at once
     //
     protected java.lang.Object sendMutex_ = new java.lang.Object();
+    
+    private boolean shuttingDown;
+    private final ReentrantReadWriteLock receiverLock = new ReentrantReadWriteLock(true);
 
     // ----------------------------------------------------------------
     // Protected Methods
@@ -115,45 +92,9 @@
     // Assumes 'this' is synchronized on entry
     //
     protected void addReceiverThread() {
-        //
-        // Retrieve the thread group
-        //
-        ThreadGroup group;
-        if ((properties_ & Property.CreatedByClient) != 0) {
-            group = orbInstance_.getClientWorkerGroup();
-        }
-        else {
-            group = orbInstance_.getServerWorkerGroup();
-        }
-
-        //
-        // Start receiver thread
-        //
-        Thread thr = new ReceiverThread(group, this);
-        thr.setDaemon(true); 
-        thr.start();
-
-        //
-        // add the thread to our list of threads
-        //
-        receiverThreads_.addLast(thr);
+        getExecutor().submit(new Receiver());
     }
 
-    //
-    // clean up any dead receiver threads
-    // assumes 'this' is synchronized on entry
-    //
-    protected void cleanupDeadReceiverThreads() {
-        java.util.ListIterator i = receiverThreads_.listIterator(0);
-
-        while (i.hasNext()) {
-            Thread thr = (Thread) i.next();
-
-            if (!thr.isAlive()) {
-                i.remove();
-            }
-        }
-    }
 
     //
     // pause a thread on a holding monitor if turned on
@@ -229,6 +170,7 @@
                 .describeTransient(org.apache.yoko.orb.OB.MinorCodes.MinorForcedShutdown),
                 org.apache.yoko.orb.OB.MinorCodes.MinorForcedShutdown,
                 org.omg.CORBA.CompletionStatus.COMPLETED_MAYBE), false);
+        arrive();
 
     }
 
@@ -244,8 +186,11 @@
         //
         // don't shutdown if there are pending upcalls
         // 
-        if (upcallsInProgress_ > 0 || state_ != State.Closing)
+        if (upcallsInProgress_ > 0 || state_ != State.Closing) {
+            logger.fine("pending upcalls: " + upcallsInProgress_ + " state: " + state_);
+         
             return;
+        }
 
         //
         // send a CloseConnection if we can
@@ -266,38 +211,44 @@
                     org.omg.GIOP.MsgType_1_1.CloseConnection, false, 0);
 
             messageQueue_.add(orbInstance_, out._OB_buffer());
+        } else {
+            logger.fine("could not send close connection message");
         }
 
         //
-        // now create the startup thread
+        // now create the shutdown thread
         //
         try {
-            if (shutdownThread_ != null)
+            if (shuttingDown)
                 return;
 
-            //
-            // Retrieve the thread group
-            //
-            ThreadGroup group;
-            if ((properties_ & Property.CreatedByClient) != 0)
-                group = orbInstance_.getClientWorkerGroup();
-            else
-                group = orbInstance_.getServerWorkerGroup();
-
+            shuttingDown = true;
             //
             // start the shutdown thread
             //
-            shutdownThread_ = new ShutdownThread(group, this);
-            shutdownThread_.setDaemon(true);
-            shutdownThread_.start();
+            try {
+                getExecutor().submit(new Shutdown());
+            } catch (RejectedExecutionException ree) {
+                logger.log(Level.WARNING, "Could not submit shutdown task", ree);
+            }
         } catch (OutOfMemoryError ex) {
             processException(State.Closed, new org.omg.CORBA.IMP_LIMIT(
                     org.apache.yoko.orb.OB.MinorCodes.describeImpLimit(org.apache.yoko.orb.OB.MinorCodes.MinorThreadLimit),
                     org.apache.yoko.orb.OB.MinorCodes.MinorThreadLimit,
                     org.omg.CORBA.CompletionStatus.COMPLETED_NO), false);
+        } finally {
+            arrive();
         }
     }
 
+
+    private void arrive() {
+        if ((properties_ & Property.CreatedByClient) != 0)
+            orbInstance_.getClientPhaser().arrive();
+        else
+            orbInstance_.getServerPhaser().arrive();
+    }
+
     // ----------------------------------------------------------------
     // Public Methods
     // ----------------------------------------------------------------
@@ -308,6 +259,7 @@
     public GIOPConnectionThreaded(ORBInstance orbInstance,
             org.apache.yoko.orb.OCI.Transport transport, GIOPClient client) {
         super(orbInstance, transport, client);
+        orbInstance.getClientPhaser().register();
         start();
     }
 
@@ -317,6 +269,14 @@
     public GIOPConnectionThreaded(ORBInstance orbInstance,
             org.apache.yoko.orb.OCI.Transport transport, OAInterface oa) {
         super(orbInstance, transport, oa);
+        orbInstance.getServerPhaser().register();
+    }
+    
+    private ExecutorService getExecutor() {
+        if ((properties_ & Property.CreatedByClient) != 0)
+            return orbInstance_.getClientExecutor();
+        else
+            return orbInstance_.getServerExecutor();
     }
 
     //
@@ -351,49 +311,38 @@
 
         // 
         // shutdown the transport
+        // synchronization on sendMutex_ is needed to avoid a deadlock in some oracle and ibm jdks between send and shutdown
+        // https://bugs.openjdk.java.net/browse/JDK-8013809 deadlock in SSLSocketImpl between between write and close 
         // 
-        transport_.shutdown();
+        synchronized (sendMutex_) {
+            transport_.shutdown();
+        }
 
         //
         // Shutdown the receiver threads. There may not be a receiver
         // thread if the transport is SendOnly.
         //
-        if (transport_.mode() == org.apache.yoko.orb.OCI.SendReceiveMode.SendReceive
-                || transport_.mode() == org.apache.yoko.orb.OCI.SendReceiveMode.ReceiveOnly) {
-            int timeout = shutdownTimeout_ * 1000;
-
-            synchronized (this) {
-                java.util.ListIterator i = receiverThreads_.listIterator();
-
-                while (i.hasNext()) {
-                    Thread t = (Thread) i.next();
-
-                    try {
-                        if (timeout > 0) {
-                            t.join(timeout);
-                        }
-                        else {
-                            t.join();
-                        }
-                    } catch (InterruptedException ex) {
-                        continue;
-                    }
-
-                    i.remove();
-                }
-            }
+        try {
+            receiverLock.writeLock().tryLock(shutdownTimeout_, SECONDS);
+        } catch (InterruptedException e) {
         }
 
-        //
-        // We now close the connection actively, since it may still be
-        // open under certain circumstances. For example, the reciver
-        // thread may not have terminated yet or the receive thread might
-        // set the state to GIOPState::Error before termination.
-        //
-        processException(State.Closed, new org.omg.CORBA.TRANSIENT(org.apache.yoko.orb.OB.MinorCodes
-                .describeTransient(org.apache.yoko.orb.OB.MinorCodes.MinorForcedShutdown),
-                org.apache.yoko.orb.OB.MinorCodes.MinorForcedShutdown,
-                org.omg.CORBA.CompletionStatus.COMPLETED_MAYBE), false);
+        try {
+            //
+            // We now close the connection actively, since it may still be
+            // open under certain circumstances. For example, the receiver
+            // thread may not have terminated yet or the receive thread might
+            // set the state to GIOPState::Error before termination.
+            //
+            processException(State.Closed, new org.omg.CORBA.TRANSIENT(org.apache.yoko.orb.OB.MinorCodes
+                    .describeTransient(org.apache.yoko.orb.OB.MinorCodes.MinorForcedShutdown),
+                    org.apache.yoko.orb.OB.MinorCodes.MinorForcedShutdown,
+                    org.omg.CORBA.CompletionStatus.COMPLETED_MAYBE), false);
+        } finally {
+            if (receiverLock.isWriteLockedByCurrentThread()) {
+                receiverLock.writeLock().unlock();
+            }
+        }
     }
 
     //
@@ -514,9 +463,7 @@
                 // case)
                 // 
                 if (haveBidirSCL) {
-                    synchronized (this) {
-                        addReceiverThread();
-                    }
+                    addReceiverThread();
                 }
 
                 upcall.invoke();
@@ -814,12 +761,15 @@
         //
         if (transport_.mode() != org.apache.yoko.orb.OCI.SendReceiveMode.SendOnly) {
             try {
-                synchronized (this) {
-                    if (receiverThreads_.size() > 0) {
-                        return;
+                // If the write lock is obtainable there are no receivers outstanding.
+                // We can then add a receiver, which implicitly obtains a read lock.
+                // ReentrantReadWriteLock explicitly allows downgrading a write lock to a read lock.
+                if(receiverLock.writeLock().tryLock()) {
+                    try {
+                        addReceiverThread();
+                    } finally {
+                        receiverLock.writeLock().unlock();
                     }
-
-                    addReceiverThread();
                 }
             } catch (OutOfMemoryError ex) {
                 synchronized (this) {
@@ -851,10 +801,6 @@
         }
 
         synchronized (this) {
-            //
-            // cleanup any defunct receiver threads now
-            // 
-            cleanupDeadReceiverThreads();
 
             //
             // if we can't write messages then don't bother to proceed
diff --git a/yoko-core/src/main/java/org/apache/yoko/orb/OB/GIOPServerStarter.java b/yoko-core/src/main/java/org/apache/yoko/orb/OB/GIOPServerStarter.java
index c54a153..2cb4be4 100644
--- a/yoko-core/src/main/java/org/apache/yoko/orb/OB/GIOPServerStarter.java
+++ b/yoko-core/src/main/java/org/apache/yoko/orb/OB/GIOPServerStarter.java
@@ -17,11 +17,8 @@
 
 package org.apache.yoko.orb.OB;
 
-import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.apache.yoko.orb.PortableServer.*;
-
 abstract class GIOPServerStarter {
     static final Logger logger = Logger.getLogger(GIOPServerStarter.class.getName());
     
diff --git a/yoko-core/src/main/java/org/apache/yoko/orb/OB/GIOPServerStarterThreaded.java b/yoko-core/src/main/java/org/apache/yoko/orb/OB/GIOPServerStarterThreaded.java
index ad6e9b7..fb2097c 100644
--- a/yoko-core/src/main/java/org/apache/yoko/orb/OB/GIOPServerStarterThreaded.java
+++ b/yoko-core/src/main/java/org/apache/yoko/orb/OB/GIOPServerStarterThreaded.java
@@ -17,21 +17,18 @@
 
 package org.apache.yoko.orb.OB;
 
+import java.util.concurrent.ExecutorService;
+import java.util.logging.Level;
+
 final class GIOPServerStarterThreaded extends GIOPServerStarter {
     //
     // The starter thread
     //
-    protected final class StarterThread extends Thread {
-        private GIOPServerStarterThreaded starter_;
-
-        StarterThread(ThreadGroup group, GIOPServerStarterThreaded starter) {
-            super(group, "Yoko:Server:StarterThread");
-            starter_ = starter;
-        }
+    protected final class Starter implements Runnable {
 
         public void run() {
             try {
-                starter_.starterRun();
+                starterRun();
             } catch (RuntimeException ex) {
                 Assert._OB_assert(ex);
             }
@@ -41,8 +38,8 @@
             // Shutdown the acceptor so that no further connections are
             // accepted
             //
-            starter_.logCloseAcceptor();
-            starter_.acceptor_.shutdown();
+            logCloseAcceptor();
+            acceptor_.shutdown();
 
             //
             // Accept all connections which might have queued up in the
@@ -51,7 +48,7 @@
                 org.apache.yoko.orb.OCI.Transport transport = null;
 
                 try {
-                    transport = starter_.acceptor_.accept(false);
+                    transport = acceptor_.accept(false);
                 } catch (org.omg.CORBA.SystemException ex) {
                 }
 
@@ -62,8 +59,8 @@
 
                 try {
                     GIOPConnection connection = new GIOPConnectionThreaded(
-                            starter_.orbInstance_, transport,
-                            starter_.oaInterface_);
+                            orbInstance_, transport,
+                            oaInterface_);
 
                     connection.setState(GIOPConnection.State.Closing);
                 } catch (org.omg.CORBA.SystemException ex) {
@@ -74,17 +71,11 @@
             //
             // Close the acceptor
             //
-            starter_.acceptor_.close();
+            acceptor_.close();
 
-            //
-            // Break cyclic object dependency
-            //
-            starter_ = null;
         }
     }
 
-    protected Thread starterThread_;
-
     // ----------------------------------------------------------------------
     // GIOPServerStarterThreaded package member implementation
     // ----------------------------------------------------------------------
@@ -98,14 +89,12 @@
             //
             // Retrieve the thread group for the servers
             //
-            ThreadGroup group = orbInstance_.getServerWorkerGroup();
+            ExecutorService executor = orbInstance_.getServerExecutor();
 
             //
             // Start starter thread
             //
-            starterThread_ = new StarterThread(group, this);
-            starterThread_.setDaemon(true); 
-            starterThread_.start();
+            executor.submit(new Starter());
         } catch (OutOfMemoryError ex) {
             acceptor_.close();
             state_ = StateClosed;
@@ -249,11 +238,13 @@
                             // StateClosing for proper connection shutdown
                             //
                             Assert._OB_assert(state_ == StateClosed);
-
+                            logger.fine("Processing an inbound connection because state is closed"); 
                             GIOPConnection connection = new GIOPConnectionThreaded(
                                     orbInstance_, transport, oaInterface_);
+                            logger.fine("Created connection " + connection); 
 
                             connection.setState(GIOPConnection.State.Closing);
+                            logger.fine("set connection state to closing"); 
                         }
                     } catch (org.omg.CORBA.SystemException ex) {
                         String msg = "can't accept connection\n" + ex.getMessage();
diff --git a/yoko-core/src/main/java/org/apache/yoko/orb/OB/ORBControl.java b/yoko-core/src/main/java/org/apache/yoko/orb/OB/ORBControl.java
index 6197632..3a0ae1a 100644
--- a/yoko-core/src/main/java/org/apache/yoko/orb/OB/ORBControl.java
+++ b/yoko-core/src/main/java/org/apache/yoko/orb/OB/ORBControl.java
@@ -25,6 +25,11 @@
 import static org.omg.CORBA.CompletionStatus.COMPLETED_NO;
 
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Phaser;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.locks.ReadWriteLock;
 
 import org.apache.yoko.orb.OBPortableServer.POAManagerFactory_impl;
 import org.apache.yoko.orb.OBPortableServer.POA_impl;
@@ -63,6 +68,9 @@
     //
     private volatile Thread mainThread_;
 
+    private long shutdownTimeout_ = 2;//seconds
+
+
     // ----------------------------------------------------------------------
     // ORBControl private and protected member implementations
     // ----------------------------------------------------------------------
@@ -105,65 +113,34 @@
     }
 
     private void waitForServerThreads() {
-        ThreadGroup group = orbInstance_.getServerWorkerGroup();
-        synchronized (group) {
-            int timeOuts = 0;
+        shutdownExecutor(orbInstance_.getServerPhaser(), orbInstance_.getServerExecutor());
 
-            // it's possible that a thread will get stalled in a read(), which seems 
-            // to happen most often with SSLSockets.  We'll do some retry loops here 
-            // and interrupt any threads that seem to be taking an excessively long time 
-            // to clean up. 
-            int count = group.activeCount();
-            while (count > 0) {
-                try {
-                    group.wait(200);
-                } catch (InterruptedException ex) {
-                }
-                int newCount = group.activeCount();
-                // we woke up because of a timeout.  
-                if (newCount == count) {
-                    timeOuts++;
-                    // after 2 timeouts, interrupt any remaining threads in the 
-                    // group. 
-                    if (timeOuts == 2) {
-                        group.interrupt();
-                    }
-                    // we've waited a full second, and we still have active threads. 
-                    // time to give up waiting. 
-                    if (timeOuts >= 5) {
-                        break;
-                    }
-                }
-                count = newCount;
-            }
+        //
+        // Get the DispatchStrategyFactory implementation and
+        // destroy it. It must be destroyed here so that the
+        // thread pools get destroyed before OCI::Current_impl
+        // gets destroyed by the destruction of the Root
+        // POA. Otherwise, thread specific data for the thread
+        // pool threads will not get released.
+        //
+        DispatchStrategyFactory dsFactory = orbInstance_.getDispatchStrategyFactory();
 
-            //
-            // Get the DispatchStrategyFactory implementation and
-            // destroy it. It must be destroyed here so that the
-            // thread pools get destroyed before OCI::Current_impl
-            // gets destroyed by the destruction of the Root
-            // POA. Otherwise, thread specific data for the thread
-            // pool threads will not get released.
-            //
-            DispatchStrategyFactory dsFactory = orbInstance_.getDispatchStrategyFactory();
+        DispatchStrategyFactory_impl dsFactoryImpl = (DispatchStrategyFactory_impl) dsFactory;
 
-            DispatchStrategyFactory_impl dsFactoryImpl = (DispatchStrategyFactory_impl) dsFactory;
+        dsFactoryImpl._OB_destroy();
 
-            dsFactoryImpl._OB_destroy();
+        //
+        // Mark the server side state as shutdown and notify any
+        // waiting threads
+        //
+        state = State.SERVER_SHUTDOWN;
 
-            //
-            // Mark the server side state as shutdown and notify any
-            // waiting threads
-            //
-            state = State.SERVER_SHUTDOWN;
-
-            //
-            // Destroy the root POA
-            //
-            if (rootPOA_ != null) {
-                rootPOA_.destroy(true, true);
-                rootPOA_ = null;
-            }
+        //
+        // Destroy the root POA
+        //
+        if (rootPOA_ != null) {
+            rootPOA_.destroy(true, true);
+            rootPOA_ = null;
         }
     }
 
@@ -443,15 +420,7 @@
             // Wait for all the threads in the client worker group to
             // terminate
             //
-            ThreadGroup group = orbInstance_.getClientWorkerGroup();
-            synchronized (group) {
-                while (group.activeCount() > 0) {
-                    try {
-                        group.wait();
-                    } catch (InterruptedException ex) {
-                    }
-                }
-            }
+            shutdownExecutor(orbInstance_.getClientPhaser(), orbInstance_.getClientExecutor());
         }
 
         //
@@ -462,6 +431,20 @@
         notifyAll();
     }
 
+    private void shutdownExecutor(Phaser phaser, ExecutorService executor) {
+        int phase = phaser.arrive();//release the system's "lock"
+        //phaser advances after all GIOPConnectionThreaded have shut down (gracefully or abort)
+        try {
+            phaser.awaitAdvanceInterruptibly(phase, shutdownTimeout_, TimeUnit.SECONDS);
+        } catch (InterruptedException e1) {
+            Thread.currentThread().interrupt();
+        } catch (TimeoutException e) {
+        } finally {
+            phaser.forceTermination();
+        }
+        executor.shutdownNow();
+    }
+
     //
     // Initialize the Root POA
     //
diff --git a/yoko-core/src/main/java/org/apache/yoko/orb/OB/ORBInstance.java b/yoko-core/src/main/java/org/apache/yoko/orb/OB/ORBInstance.java
index 9842688..a7493ef 100644
--- a/yoko-core/src/main/java/org/apache/yoko/orb/OB/ORBInstance.java
+++ b/yoko-core/src/main/java/org/apache/yoko/orb/OB/ORBInstance.java
@@ -17,6 +17,16 @@
 
 package org.apache.yoko.orb.OB;
 
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Phaser;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
 import org.apache.yoko.orb.OB.BootManager;
 import org.apache.yoko.orb.OB.DispatchStrategyFactory;
 import org.apache.yoko.orb.OB.Logger;
@@ -88,9 +98,11 @@
 
     private RecursiveMutex orbSyncMutex_ = new RecursiveMutex();
 
-    private ThreadGroup serverWorkerGroup_;
+    private ExecutorService serverExecutor_;
+    private Phaser serverPhaser = new Phaser(1);
 
-    private ThreadGroup clientWorkerGroup_;
+    private ExecutorService clientExecutor_;
+    private Phaser clientPhaser = new Phaser(1);
 
     private org.apache.yoko.orb.OCI.ConFactoryRegistry conFactoryRegistry_;
 
@@ -171,10 +183,11 @@
         defaultWcs_ = defaultWcs;
 
         //
-        // Create the server and client worker groups
+        // Create the server and client executors
+        // TODO why are these separate?
         //
-        clientWorkerGroup_ = new ThreadGroup("ClientWorkers");
-        serverWorkerGroup_ = new ThreadGroup("ServerWorkers");
+        clientExecutor_ = Executors.newCachedThreadPool();
+        serverExecutor_ = Executors.newCachedThreadPool();
 
         //
         // Use the TypeCode cache?
@@ -303,23 +316,8 @@
         // coreTraceLevels_.destroy();
         // coreTraceLevels_ = null;
 
-        try {
-            //
-            // Destroy the client and server worker groups
-            //
-            serverWorkerGroup_.destroy();
-        } catch (IllegalThreadStateException ex) {
-            // we ignore this...occasionally, it is necessary 
-            // to kick the threads to force them to shutdown. 
-        }
-
-        try {
-            clientWorkerGroup_.destroy();
-        } catch (IllegalThreadStateException ex) {
-            // we ignore this...occasionally, it is necessary 
-            // to kick the threads to force them to shutdown. 
-        }
-
+        // Client and server executors shut down in the ORBControl
+        
         //
         // Destroy the ConFactoryRegistry
         //
@@ -424,12 +422,20 @@
         return orbSyncMutex_;
     }
 
-    public ThreadGroup getServerWorkerGroup() {
-        return serverWorkerGroup_;
+    public ExecutorService getServerExecutor() {
+        return serverExecutor_;
     }
 
-    public ThreadGroup getClientWorkerGroup() {
-        return clientWorkerGroup_;
+    public Phaser getServerPhaser() {
+        return serverPhaser;
+    }
+    
+    public ExecutorService getClientExecutor() {
+        return clientExecutor_;
+    }
+    
+    public Phaser getClientPhaser() {
+        return clientPhaser;
     }
 
     public org.apache.yoko.orb.OCI.ConFactoryRegistry getConFactoryRegistry() {
@@ -478,4 +484,5 @@
     public OrbAsyncHandler getAsyncHandler() {
         return asyncHandler_;
     }
+
 }
diff --git a/yoko-core/src/main/java/org/apache/yoko/orb/OCI/IIOP/Acceptor_impl.java b/yoko-core/src/main/java/org/apache/yoko/orb/OCI/IIOP/Acceptor_impl.java
index 75bcefe..1793988 100644
--- a/yoko-core/src/main/java/org/apache/yoko/orb/OCI/IIOP/Acceptor_impl.java
+++ b/yoko-core/src/main/java/org/apache/yoko/orb/OCI/IIOP/Acceptor_impl.java
@@ -71,7 +71,7 @@
     }
 
     public void close() {
-        logger.fine("Closing connection to host=" + localAddress_ + ", port=" + port_);
+        logger.log(Level.FINE, "Closing server socket with host=" + localAddress_ + ", port=" + port_, new Exception("Stack trace"));
         //
         // Destroy the info object
         //
@@ -83,7 +83,9 @@
         try {
             socket_.close();
             socket_ = null;
+            logger.log(Level.FINE, "Closed server socket with host=" + localAddress_ + ", port=" + port_);
         } catch (java.io.IOException ex) {
+            logger.log(Level.FINE, "Exception closing server socket with host=" + localAddress_ + ", port=" + port_, ex);
         }
     }