///////////////////////////////////////////////////////////////////////////////
//
// @@@ START COPYRIGHT @@@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
//
// @@@ END COPYRIGHT @@@
//
///////////////////////////////////////////////////////////////////////////////

#include <iostream>

using namespace std;

#include <stdio.h>
#include <stdlib.h>
#include <setjmp.h>
#include <signal.h>
#include <fcntl.h>
#include <netdb.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <sys/epoll.h>
#include <sys/file.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <errno.h>
#include <limits.h>
#include <unistd.h>

#include "localio.h"
#include "mlio.h"
#include "monlogging.h"
#include "monsonar.h"
#include "montrace.h"
#include "redirector.h"
#include "healthcheck.h"
#include "config.h"
#include "device.h"
#include "cluster.h"
#include "monitor.h"

#include "replicate.h"

#include "clusterconf.h"
#include "lnode.h"
#include "pnode.h"
#include "reqqueue.h"
#include "zclient.h"
#include "commaccept.h"
#include "meas.h"
#ifdef NAMESERVER_PROCESS
#include "nscommacceptmon.h"
#else
#include "nameserver.h"
#endif

extern bool IAmIntegrating;
extern bool IAmIntegrated;
extern bool IsRealCluster;
extern bool IsAgentMode;
extern bool IsMaster;
extern bool IsMPIChild;
extern char MasterMonitorName[MAX_PROCESS_PATH];
extern char Node_name[MPI_MAX_PROCESSOR_NAME];
extern bool ZClientEnabled;
extern char IntegratingMonitorPort[MPI_MAX_PORT_NAME];
extern char MyCommPort[MPI_MAX_PORT_NAME];
extern char MyMPICommPort[MPI_MAX_PORT_NAME];
extern char MySyncPort[MPI_MAX_PORT_NAME];
#ifdef NAMESERVER_PROCESS
extern CCommAcceptMon CommAcceptMon;
extern char MyMon2NsPort[MPI_MAX_PORT_NAME];
#else
extern CNameServer *NameServer;
extern bool NameServerEnabled;
extern char MyPtPPort[MPI_MAX_PORT_NAME];
#endif
extern bool SMSIntegrating;
extern int CreatorShellPid;
extern Verifier_t CreatorShellVerifier;
extern CommType_t CommType;

extern int MyPNID;

extern CReqQueue ReqQueue;

extern CMonitor *Monitor;
extern CNodeContainer *Nodes;
extern CConfigContainer *Config;
#ifndef NAMESERVER_PROCESS
extern CDeviceContainer *Devices;
#endif
extern CNode *MyNode;
extern CMonStats *MonStats;
#ifndef NAMESERVER_PROCESS
extern CRedirector Redirector;
#endif
extern CMonLog *MonLog;
extern CHealthCheck HealthCheck;
extern CCommAccept CommAccept;
extern CZClient    *ZClient;
extern CMeas Meas;

extern long next_test_delay;
extern CReplicate Replicator;

extern char *ErrorMsg (int error_code);

extern const char *ProcessTypeString( PROCESSTYPE type );

const char *JoiningPhaseString( JOINING_PHASE phase);
const char *StateString( STATE state);
#ifndef NAMESERVER_PROCESS
const char *SyncStateString( SyncState state);
#endif
const char *EpollEventString( __uint32_t events );
const char *EpollOpString( int op );
const char *NodePhaseString( NodePhase phase );
#ifdef NAMESERVER_PROCESS
#define MPI_Abort(a,b) abort()
#endif

const char *NodePhaseString( NodePhase phase )
{
    const char *str;

    switch( phase )
    {
        case Phase_Ready:
            str = "Phase_Ready";
            break;
        case Phase_Activating:
            str = "Phase_Activating";
            break;
        case Phase_SoftDown:
            str = "Phase_SoftDown";
            break;
        case Phase_SoftUp:
            str = "Phase_SoftUp";
            break;
        default:
            str = "NodePhase - Undefined";
            break;
    }

    return( str );
}

void CCluster::ActivateSpare( CNode *spareNode, CNode *downNode, bool checkHealth )
{
    const char method_name[] = "CCluster::ActivateSpare";
    TRACE_ENTRY;
    // if not checking health, assume the spare is healthy
    bool spareHealthy = checkHealth ? false : true;
    int tmCount = 0;
    CNode *node;
    CLNode *lnode;

    if (trace_settings & TRACE_INIT)
    {
        trace_printf( "%s@%d - pnid=%d, name=%s (%s) is taking over pnid=%d, name=%s (%s), check health=%d, isIntegrating=%d , integrating pnid=%d\n"
                    , method_name, __LINE__
                    , spareNode->GetPNid(), spareNode->GetName(), StateString(spareNode->GetState())
                    , downNode->GetPNid(), downNode->GetName(), StateString(downNode->GetState())
                    , checkHealth, IsIntegrating(), integratingPNid_ );
    }

    if ( checkHealth )
    {
        // TODO: Execute physical node health check script here
        spareHealthy = true;
        if ( !spareHealthy )
        {
            // and tell the cluster the node is down, since the spare can't takeover
            CReplNodeDown *repl = new CReplNodeDown(downNode->GetPNid());
            Replicator.addItem(repl);
        }
    }

    if ( spareHealthy )
    {
        if ( downNode->GetPNid() != spareNode->GetPNid() )
        {
            // Move down node's logical nodes to spare node
            downNode->MoveLNodes( spareNode );

            spareNode->SetPhase( Phase_Activating );

            Nodes->AddToSpareNodesList( downNode->GetPNid() );

            if ( !IsIntegrating() )
            {
                downNode->SetState( State_Down );

                // Send process death notices
#ifndef NAMESERVER_PROCESS
                spareNode->KillAllDown();
#endif

                // Send node down notice
                lnode = spareNode->GetFirstLNode();
                for ( ; lnode; lnode = lnode->GetNextP() )
                {
                    // Watchdog process clone was removed in KillAllDown
                    lnode->Down();
                }
            }
        }

        // Any DTMs running?
        for ( int i=0; !tmCount && i < Nodes->GetPNodesCount(); i++ )
        {
            node = Nodes->GetNodeByMap( i );
            lnode = node->GetFirstLNode();
            for ( ; lnode; lnode = lnode->GetNextP() )
            {
                CProcess *process = lnode->GetProcessLByType( ProcessType_DTM );
                if ( process  ) tmCount++;
            }
        }

        // Create Watchdog and PSD processes if this node is the activating spare
        if ( spareNode->GetPNid() == MyPNID )
        {
#ifndef NAMESERVER_PROCESS
            Monitor->StartPrimitiveProcesses();
#endif
        }
        else
        {
            // Check for end of joining phase on node re-integration
            if ( spareNode->GetState() == State_Joining )
            {
                spareNode->SetState( State_Up );
            }
#ifndef NAMESERVER_PROCESS
            if ( tmCount )
            {
                // Send node prepare notice to local DTM processes
                lnode = spareNode->GetFirstLNode();
                for ( ; lnode; lnode = lnode->GetNextP() )
                {
                    lnode->PrepareForTransactions( downNode->GetPNid() != spareNode->GetPNid() );
                }
            }
#else
            ResetIntegratingPNid();
#endif
        }

#ifndef NAMESERVER_PROCESS
        if ( downNode->GetPNid() != spareNode->GetPNid() )
        {
            // we need to abort any active TmSync
            if (( MyNode->GetTmSyncState() == SyncState_Start    ) ||
                ( MyNode->GetTmSyncState() == SyncState_Continue ) ||
                ( MyNode->GetTmSyncState() == SyncState_Commit   )   )
            {
                MyNode->SetTmSyncState( SyncState_Abort );
                Monitor->SetAbortPendingTmSync();
                if (trace_settings & (TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
                   trace_printf("%s@%d" " - Node "  "%d" " TmSyncState updated (" "%d" ")" "\n", method_name, __LINE__, MyPNID, MyNode->GetTmSyncState());
            }
        }
#endif

        if (trace_settings & TRACE_INIT)
        {
            trace_printf( "%s@%d - Spare node activating! pnid=%d, name=(%s)\n"
                        , method_name, __LINE__
                        , spareNode->GetPNid(), spareNode->GetName());
        }
    }

    if ( spareNode->GetPNid() == MyPNID && spareHealthy )
    {
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
           trace_printf( "%s@%d" " - Replicating activate spare node pnid=%d, name=%s (%s), spare=%d, down pnid=%d, name=%s (%s), DTM count=%d\n"
                       , method_name, __LINE__
                       , spareNode->GetPNid(), spareNode->GetName(), StateString(spareNode->GetState())
                       , spareNode->IsSpareNode()
                       , downNode->GetPNid(), downNode->GetName(), StateString(downNode->GetState())
                       , tmCount );
        // Let other monitors know is ok to activate this spare node
        CReplActivateSpare *repl = new CReplActivateSpare( MyPNID, downNode->GetPNid() );
        Replicator.addItem(repl);

#ifndef NAMESERVER_PROCESS
        if ( !tmCount )
        {
            // No DTMs in environment so implicitly make ready for transactions
            lnode = MyNode->GetFirstLNode();
            for ( ; lnode; lnode = lnode->GetNextP() )
            {
                ReqQueue.enqueueTmReadyReq( lnode->GetNid() );
            }
        }
#endif
    }

    TRACE_EXIT;
}

#ifndef NAMESERVER_PROCESS
void CCluster::NodeTmReady( int nid )
{
    const char method_name[] = "CCluster::NodeTmReady";
    TRACE_ENTRY;

    if (trace_settings & TRACE_INIT)
    {
        trace_printf( "%s@%d - nid=%d\n", method_name, __LINE__, nid );
    }

    tmReadyCount_++;

    if (trace_settings & (TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
    {
        trace_printf( "%s@%d - TmReady, nid=%d, tm count=%d, soft node down=%d, LNodesCount=%d\n"
                    , method_name, __LINE__
                    , nid
                    , tmReadyCount_
                    , MyNode->IsSoftNodeDown()
                    , MyNode->GetLNodesCount() );
    }

    MyNode->StartPStartDPersistentDTM( nid );

    if ( MyNode->GetLNodesCount() == tmReadyCount_ )
    {
        if ( MyNode->IsSoftNodeDown() )
        {
            MyNode->ResetSoftNodeDown();

            MyNode->SetPhase( Phase_Ready );

            char la_buf[MON_STRING_BUF_SIZE];
            sprintf( la_buf, "[%s], Soft Node up! pnid=%d, name=(%s)\n"
                   , method_name, MyNode->GetPNid(), MyNode->GetName());
            mon_log_write(MON_CLUSTER_NODE_TM_READY_1, SQ_LOG_INFO, la_buf);
        }
        else
        {
            char la_buf[MON_STRING_BUF_SIZE];
            sprintf(la_buf, "[%s], Node activated! pnid=%d, name=(%s) \n", method_name, MyNode->GetPNid(), MyNode->GetName());
            mon_log_write(MON_CLUSTER_NODE_TM_READY_2, SQ_LOG_INFO, la_buf);

            // Let other monitors know the node is up
            CReplActivateSpare *repl = new CReplActivateSpare( MyPNID, -1 );
            Replicator.addItem(repl);
        }
    }

    TRACE_EXIT;
}
#endif

void CCluster::NodeReady( CNode *spareNode )
{
    const char method_name[] = "CCluster::NodeReady";
    TRACE_ENTRY;

    if (trace_settings & TRACE_INIT)
    {
        trace_printf( "%s@%d - spare node %s pnid=%d\n"
                    , method_name, __LINE__, spareNode->GetName(), spareNode->GetPNid() );
    }

    assert( spareNode->GetState() == State_Up );

    // Send node up notice
    CLNode *lnode = spareNode->GetFirstLNode();
    for ( ; lnode; lnode = lnode->GetNextP() )
    {
        lnode->Up();
    }

    spareNode->SetActivatingSpare( false );
    ResetIntegratingPNid();

    TRACE_EXIT;
}

void CCluster::UpdateMonitorPort (const char* newMaster)
{
    const char method_name[] = "CCluster::UpdateMonitorPort";
    TRACE_ENTRY;
    
    char *monitorPort = getenv ("MONITOR_COMM_PORT");
    if ((monitorPort) && (newMaster))
    {
         strcpy( IntegratingMonitorPort, newMaster );
         strcat( IntegratingMonitorPort, ":");
         strcat( IntegratingMonitorPort, monitorPort);

          if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
          {
               trace_printf("%s@%d" " (MasterMonitor) UpdateMonitorPort Updating IntegratingMonitorPort to %s\n",
                             method_name, __LINE__,IntegratingMonitorPort );
          }
    } 
    TRACE_EXIT;
}

// Assign leaders as required
// Current leaders are TM Leader and Monitor Leader
void CCluster::AssignLeaders( int pnid, const char* failedMaster, bool checkProcess )
{
    const char method_name[] = "CCluster::AssignLeaders";
    TRACE_ENTRY;

#ifndef NAMESERVER_PROCESS
    AssignTmLeader ( pnid, checkProcess );
#else
    pnid = pnid;
    checkProcess = checkProcess;
#endif
    AssignMonitorLeader ( failedMaster );

    TRACE_EXIT;
}

// Assign monitor lead in the case of failure
void CCluster::AssignMonitorLeader( const char* failedMaster )
{
    const char method_name[] = "CCluster::AssignMonitorLeader";
    TRACE_ENTRY;

    int i = 0;
    int rc = 0;
    
    int monitorLeaderPNid = -1;
    CNode *node = NULL;
    
    if (failedMaster == NULL)
    {
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
        {
            trace_printf( "%s@%d" " - (MasterMonitor) failedMaster is NULL, returning\n" , method_name, __LINE__);
        }
        TRACE_EXIT;
        return;
    }

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
    {
        trace_printf( "%s@%d" " - (MasterMonitor) "  " MonitorLeader (%s) failed!\n"
                    , method_name, __LINE__, failedMaster );
    }

    if (!IsAgentMode || !ZClientEnabled)
    {
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
        {
               trace_printf( "%s@%d" " - (MasterMonitor) not AgentMode or zookeeper not enabled, returning\n"
                 , method_name, __LINE__);
        }
        TRACE_EXIT;
        return;
    }
    // delete old master if needed
    const char *masterMonitor = ZClient->WaitForAndReturnMaster (false);
    if (masterMonitor)
    {   
        // IFF it is the failed master, delete, do not delete anything else because we could delete a new master
        if (strcmp (masterMonitor, failedMaster) == 0)
        {
            ZClient->WatchNodeMasterDelete (failedMaster);
            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
            {
                 trace_printf( "%s@%d" " - (MasterMonitor) deleting master %s\n"
                              , method_name, __LINE__, masterMonitor );
             }
        }
        // no worries
        else
        {            
             rc = ZClient->WatchMasterNode( masterMonitor ); 
             UpdateMonitorPort ( masterMonitor );
             if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
             {
                   trace_printf( "%s@%d" " - (MasterMonitor) master did not match, set watch (rc = %d) and returning %s\n"
                     , method_name, __LINE__, rc, masterMonitor );
             }
             TRACE_EXIT;
             return;
         }
    }

    // choose a new master
    if (((MyNode) && ((MyNode->GetState() != State_Up) ||(!IAmIntegrated))) || (MyNode == NULL /* not set up yet*/))
    {
        // Do not let this monitor participate in choosing the master.  It can wait until an integrated
        // monitor makes a decision.
         if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
         {
              trace_printf( "%s@%d" " - (MasterMonitor) This Node is not set up yet and will not participate in master choice!\n"
                    , method_name, __LINE__);
         }
         
         // wait until another monitor choose a master
         const char *masterMonitor = ZClient->WaitForAndReturnMaster (true);
         if (masterMonitor)
         {
             rc = ZClient->WatchMasterNode( masterMonitor ); 
             if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
             {
                  trace_printf("%s@%d" " (MasterMonitor) AssignMonitorLeader WatchMasterNode with rc = %d\n", method_name, __LINE__, rc);
             }

          UpdateMonitorPort ( masterMonitor );
          }
          TRACE_EXIT;
          return;
    }
 
    // For all monitors who are up - choose the master using the same logic
    for (i=0; i<GetConfigPNodesMax(); i++)
    {
        monitorLeaderPNid++; // set to -1, so this will bump it to 0 on the first time through

        if (monitorLeaderPNid == GetConfigPNodesMax())
        {
             if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
             {
                 trace_printf("%s@%d" " (MasterMonitor) AssignMonitorLeader  Unable to create or set watch\n", method_name, __LINE__);
             }
             char    buf[MON_STRING_BUF_SIZE];
             snprintf( buf, sizeof(buf)
                           , "[%s], Unable to create or set watch on master, hit max\n"
                           , method_name );
            mon_log_write(MON_CLUSTER_ASSIGNMONITORLEADER_1, SQ_LOG_ERR, buf);
            break;
        }

        if (Node[monitorLeaderPNid] == NULL)
        {
            continue;
        }

        node = Node[monitorLeaderPNid];

        // skip this node
        if ( node == NULL )
        {
            continue; 
        }

        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
        {
            trace_printf( "%s@%d - Node pnid=%d (%s), phase=%s, isSoftNodeDown=%d\n"
                        , method_name, __LINE__
                        , node->GetPNid()
                        , node->GetName()
                        , NodePhaseString(node->GetPhase())
                        , node->IsSoftNodeDown());
        }

        if ( node->IsSpareNode() ||
             node->IsSoftNodeDown() ||
             node->GetState() != State_Up ||
             node->GetPhase() != Phase_Ready )
        {
            continue; // skip this node for any of the above reasons
        }

        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
        {
            trace_printf("%s@%d" " - Node "  "%d" " is the new monitorLeaderPNid." "\n", method_name, __LINE__, node->GetPNid());
        }

        const char *masterMonitor = ZClient->WaitForAndReturnMaster (false);
    
        //nobody has written it yet, we don't want to overwrite anything
        if (!masterMonitor)
        {
            rc = ZClient->CreateMasterZNode ( node->GetName() );
            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
            {
                trace_printf("%s@%d" " (MasterMonitor) AssignMonitorLeader CreateMasterZNode with rc = %d\n", method_name, __LINE__, rc);
            }
            char    buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf)
                              , "[%s], Master Monitor is %s on node %d\n"
                              , method_name, node->GetName(), node->GetPNid() );
            mon_log_write(MON_CLUSTER_ASSIGNMONITORLEADER_2, SQ_LOG_INFO, buf);

            if ( (rc == ZOK) || (rc == ZNODEEXISTS) )
            {
                 rc = ZClient->WatchMasterNode( node->GetName() ); 
                 if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
                 {
                     trace_printf("%s@%d" " (MasterMonitor) AssignMonitorLeader WatchMasterNode with rc = %d\n", method_name, __LINE__, rc);
                 }
            }
            else
            {
                 if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
                 {
                     trace_printf("%s@%d" " (MasterMonitor) AssignMonitorLeader  Unable to create or set watch\n", method_name, __LINE__);
                 }
                 char    buf[MON_STRING_BUF_SIZE];
                 snprintf( buf, sizeof(buf)
                           , "[%s], Unable to create or set watch on master node %s\n"
                           , method_name, node->GetName() );
                 mon_log_write(MON_CLUSTER_ASSIGNMONITORLEADER_3, SQ_LOG_ERR, buf);
            }
       }
       else
       {
           rc = ZClient->WatchMasterNode( masterMonitor ); 
           char    buf[MON_STRING_BUF_SIZE];
           snprintf( buf, sizeof(buf)
                          , "[%s], Master Monitor is %s\n"
                          , method_name, masterMonitor);
           mon_log_write(MON_CLUSTER_ASSIGNMONITORLEADER_4, SQ_LOG_INFO, buf);
           if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
           {
               trace_printf("%s@%d" " (MasterMonitor) AssignMonitorLeader WatchMasterNode with rc = %d\n", method_name, __LINE__, rc);
           }
        }

        break;
    }

    TRACE_EXIT;
}

#ifndef NAMESERVER_PROCESS
// Assigns a new TMLeader if given pnid is same as tmLeaderNid_
// TmLeader is a logical node num.
// pnid has gone down, so if that node was previously the TM leader, a new one needs to be chosen.
void CCluster::AssignTmLeader( int pnid, bool checkProcess )
{
    const char method_name[] = "CCluster::AssignTmLeader";
    TRACE_ENTRY;

    int i = 0;
    CNode *node = NULL;
    CProcess *process = NULL;

    int TmLeaderPNid = LNode[tmLeaderNid_]->GetNode()->GetPNid();

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
    {
        trace_printf( "%s@%d - pnid=%d, checkProcess=%d, tmLeaderNid_=%d, TmLeaderPNid=%d\n"
                    , method_name, __LINE__
                    , pnid, checkProcess, tmLeaderNid_, TmLeaderPNid );
    }

    if (TmLeaderPNid != pnid)
    {
        node = LNode[tmLeaderNid_]->GetNode();

        if (checkProcess)
        {
            process = LNode[tmLeaderNid_]->GetProcessLByType( ProcessType_DTM );
            if (process)
            {
                if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
                {
                    if (node)
                        trace_printf( "%s@%d - Node pnid=%d (%s), phase=%s, "
                                      "isSoftNodeDown=%d, checkProcess=%d\n"
                                    , method_name, __LINE__
                                    , node->GetPNid()
                                    , node->GetName()
                                    , NodePhaseString(node->GetPhase())
                                    , node->IsSoftNodeDown()
                                    , checkProcess );
                }
                return;
            }
            else
            {
                if (NameServerEnabled)
                {
                    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
                    {
                        trace_printf( "%s@%d - Getting process from Name Server, nid=%d, type=%s\n"
                                    , method_name, __LINE__
                                    , tmLeaderNid_, ProcessTypeString(ProcessType_DTM) );
                    }
                
                    process = Nodes->GetProcessLByTypeNs( tmLeaderNid_, ProcessType_DTM );
                    if (process)
                    {
                        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
                        {
                            if (node)
                                trace_printf( "%s@%d - Node pnid=%d (%s), phase=%s, "
                                              "isSoftNodeDown=%d, checkProcess=%d\n"
                                            , method_name, __LINE__
                                            , node->GetPNid()
                                            , node->GetName()
                                            , NodePhaseString(node->GetPhase())
                                            , node->IsSoftNodeDown()
                                            , checkProcess );
                        }
                        return;
                    }
                }
            }
        }
        else
        {
            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
            {
                if (node)
                    trace_printf( "%s@%d - Node pnid=%d (%s), phase=%s, "
                                  "isSoftNodeDown=%d, checkProcess=%d\n"
                                , method_name, __LINE__
                                , node->GetPNid()
                                , node->GetName()
                                , NodePhaseString(node->GetPhase())
                                , node->IsSoftNodeDown()
                                , checkProcess );
            }
            return;
        }
    }

    node = Node[TmLeaderPNid];

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
    {
        trace_printf( "%s@%d" " - Node "  "%d" " TmLeader failed! (checkProcess=%d)\n"
                    , method_name, __LINE__, tmLeaderNid_, checkProcess );
    }

    for (i=0; i<GetConfigPNodesMax(); i++)
    {
        TmLeaderPNid++;

        if (TmLeaderPNid == GetConfigPNodesMax())
        {
            TmLeaderPNid = 0; // restart with nid 0
        }

        if (TmLeaderPNid == pnid)
        {
            continue; // this is the node that is going down, skip it
        }

        if (Node[TmLeaderPNid] == NULL)
        {
            continue;
        }

        node = Node[TmLeaderPNid];

        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
        {
            trace_printf( "%s@%d - Node pnid=%d (%s), phase=%s, isSoftNodeDown=%d\n"
                        , method_name, __LINE__
                        , node->GetPNid()
                        , node->GetName()
                        , NodePhaseString(node->GetPhase())
                        , node->IsSoftNodeDown());
        }

        if ( node->IsSpareNode() ||
             node->IsSoftNodeDown() ||
             node->GetState() != State_Up ||
             node->GetPhase() != Phase_Ready )
        {
            continue; // skip this node for any of the above reasons
        }

        tmLeaderNid_ = node->GetFirstLNode()->GetNid();

        if (checkProcess)
        {
            process = LNode[tmLeaderNid_]->GetProcessLByType( ProcessType_DTM );
            if (!process)
            {
                continue; // skip this node no DTM process exists
            }
        }

        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
        {
            trace_printf("%s@%d" " - Node "  "%d" " is the new TmLeader." "\n", method_name, __LINE__, tmLeaderNid_);
        }

        break;
    }

    TRACE_EXIT;
}
#endif


CCluster::CCluster (void)
      :NumRanks (-1)
      ,socks_(NULL)
      ,sockPorts_(NULL)
      ,commSock_(-1)
      ,syncPort_(0)
      ,syncSock_(-1)
#ifdef NAMESERVER_PROCESS
      ,mon2nsSock_(-1)
#endif
      ,epollFD_(-1),
      Node (NULL),
      LNode (NULL),
      tmSyncPNid_ (-1),
      currentNodes_ (0),
      configPNodesCount_ (-1),
      configPNodesMax_ (-1),
      nodeMap_ (NULL),
#ifndef NAMESERVER_PROCESS
      tmLeaderNid_ (-1),
      tmReadyCount_(0),
#endif
      minRecvCount_(4096),
      recvBuffer_(NULL),
      recvBuffer2_(NULL),
      swpRecCount_(0),
      barrierCount_(0),
      allGatherCount_(0),
      commDupCount_(0),
      barrierCountSaved_(0),
      allGatherCountSaved_(0),
      commDupCountSaved_(0),
      inBarrier_(false),
      inAllGather_(false),
      inCommDup_(false),
      monInitComplete_(false),
      monSyncResponsive_(true),
      integratingPNid_(-1),
      joinComm_(MPI_COMM_NULL),
      joinSock_(-1),
      lastSeqNum_(0),
      lowSeqNum_(0),
      highSeqNum_(0),
      reconnectSeqNum_(0),
      seqNum_(1),
      waitForWatchdogExit_(false)
      ,waitForNameServerExit_(false)
      ,checkSeqNum_(false)
      ,validateNodeDown_(false)
      ,enqueuedDown_(false)
      ,nodeDownDeathNotices_(true)
      ,verifierNum_(0)
#ifdef NAMESERVER_PROCESS
      ,myMonConnCount_(0)
      ,minMonConnCount_(0)
      ,minMonConnPnid_(-1)
#else
      ,clusterProcCount_(0)
#endif
{
    int i;
    const char method_name[] = "CCluster::CCluster";
    TRACE_ENTRY;

    configMaster_ = -1;
    MPI_Comm_set_errhandler(MPI_COMM_WORLD,MPI_ERRORS_RETURN);

    char *env = getenv("SQ_MON_CHECK_SEQNUM");
    if ( env )
    {
        int val = atoi(env);
        if ( val > 0)
        {
            checkSeqNum_ = (val != 0);
        }
    }

    if (trace_settings & TRACE_INIT)
       trace_printf("%s@%d Checking sync sequence numbers is %s\n",
                    method_name, __LINE__,
                    (checkSeqNum_ ? "enabled" : "disabled"));

    CClusterConfig *clusterConfig = Nodes->GetClusterConfig();
    configPNodesMax_ = clusterConfig->GetPNodesConfigMax();

    // get master from CClusterConfig
    configMaster_ = clusterConfig->GetConfigMaster();

    // Compute minimum "sync cycles" per second.   The minimum is 1/10
    // the expected number, assuming "next_test_delay" cycles per second (where
    // next_test_delay is in microseconds).
    syncMinPerSec_ = 1000000 / next_test_delay / 10;

    agMaxElapsed_.tv_sec = 0;
    agMaxElapsed_.tv_nsec = 0;
    agMinElapsed_.tv_sec = 10000;
    agMinElapsed_.tv_nsec = 0;

    // Allocate structures for monitor point-to-point communications
    //
    //   The current approach is to allocate to a maximum number (MAX_NODES).
    //
    //   The actual number could be based on the number of nodes configured
    //   which is better from a memory allocation perspective. However,
    //   this requires changing to an index-to-pnid map structure to access
    //   physical node objects (CNode) in the array structures and managing
    //   the map as nodes are added and deleted. (an optimization task)
    //
    comms_        = new MPI_Comm[MAX_NODES];
    otherMonRank_ = new int[MAX_NODES];
    socks_        = new int[MAX_NODES];
    sockPorts_    = new int[MAX_NODES];

    for ( int i =0; i < MAX_NODE_MASKS ; i++ )
    {
        upNodes_.upNodes[i] = 0;
    }

    for (i=0; i < MAX_NODES; ++i)
    {
        comms_[i] = MPI_COMM_NULL;
        socks_[i] = -1;
        sockPorts_[i] = -1;
    }

    env = getenv("SQ_MON_NODE_DOWN_VALIDATION");
    if ( env )
    {
        int val = atoi(env);
        if ( val > 0)
        {
            validateNodeDown_ = (val != 0);
        }
    }

    char buf[MON_STRING_BUF_SIZE];
    snprintf(buf, sizeof(buf), "[%s] Validation of node down is %s\n",
             method_name, (validateNodeDown_ ? "enabled" : "disabled"));
    mon_log_write(MON_CLUSTER_CLUSTER_1, SQ_LOG_INFO, buf);

    InitializeConfigCluster();

    for (size_t j=0; j<(sizeof(agElapsed_)/sizeof(int)); ++j)
    {
        agElapsed_[j] = 0;
    }

    char *p = getenv("MON_MIN_RECV_COUNT");
    if ( p )
    {
        long int val = strtoul(p, NULL, 10);
        if (errno != ERANGE)
        {
            minRecvCount_ = val;
        }
    }

    p = getenv("SQ_MON_NODE_DOWN_DEATH_MESSAGES");
    if ( p && atoi(p) == 0)
    {
        nodeDownDeathNotices_ = false;
    }

    // build the node objects & Sync collision assignment arrays
    // these buffers will be used in ShareWithPeers in AllGather
    // operation to get TMSync data as well as Replication data.
    // Allocate the maximum allowed so that we pay the price only once.
    // This wastes a bit of memory but reduces complexity when
    // adding and deleting nodes. Usage is based on GetConfigPNodesMax()
    // the maximum number that can be configured.
    recvBuffer_ = new struct sync_buffer_def[GetConfigPNodesMax()];
    recvBuffer2_ = new struct sync_buffer_def[GetConfigPNodesMax()];

    TRACE_EXIT;
}

CCluster::~CCluster (void)
{
    const char method_name[] = "CCluster::~CCluster";
    TRACE_ENTRY;

    if (epollFD_ != -1)
    {
        close( epollFD_ );
    }

    if (commSock_ != -1)
    {
        close( commSock_ );
    }

    if (syncSock_ != -1)
    {
        close( syncSock_ );
    }

    delete [] comms_;
    delete [] otherMonRank_;
    delete [] socks_;
    delete [] sockPorts_;
    if (nodeMap_)
    {
        delete [] nodeMap_;
        nodeMap_ = NULL;
    }

    delete [] recvBuffer2_;
    delete [] recvBuffer_;

    TRACE_EXIT;
}

int CCluster::incrGetVerifierNum()
{
    verifierNum_++;
    if ( verifierNum_ < 0 )
    {
        verifierNum_ = 0;
    }

    return verifierNum_;
}

// For a reintegrated monitor node, following the first sync cycle, obtain the
// current sync cycle sequence number.   And verify that all nodes agree
// on the sequence number.
unsigned long long CCluster::EnsureAndGetSeqNum(cluster_state_def_t nodestate[])
{
    const char method_name[] = "CCluster::EnsureAndGetSeqNum";
    TRACE_ENTRY;

    unsigned long long seqNum = 0;

    for (int i = 0; i < GetConfigPNodesCount(); i++)
    {
        if (trace_settings & TRACE_RECOVERY)
        {
            trace_printf("%s@%d nodestate[%d].seq_num=%lld, seqNum=%lld\n", method_name, __LINE__, i, nodestate[indexToPnid_[i]].seq_num, seqNum );
        }
        if (nodestate[indexToPnid_[i]].seq_num > 1)
        {
            if (seqNum == 0)
            {
                seqNum = nodestate[indexToPnid_[i]].seq_num;
            }
            else
            {
                assert(nodestate[indexToPnid_[i]].seq_num == seqNum);
            }
        }
        if (trace_settings & TRACE_RECOVERY)
        {
            trace_printf("%s@%d nodestate[%d].seq_num=%lld, seqNum=%lld\n", method_name, __LINE__, i, nodestate[indexToPnid_[i]].seq_num, seqNum );
        }
    }

    TRACE_EXIT;
    return seqNum;
}


void CCluster::HardNodeDown (int pnid, bool communicate_state)
{
#ifndef NAMESERVER_PROCESS
    char port_fname[MAX_PROCESS_PATH];
    char temp_fname[MAX_PROCESS_PATH];
#endif
    CNode  *node;
    CLNode *lnode;
    char    buf[MON_STRING_BUF_SIZE];

    const char method_name[] = "CCluster::HardNodeDown";
    TRACE_ENTRY;

    node = Nodes->GetNode(pnid);

    if (trace_settings & (TRACE_REQUEST | TRACE_INIT | TRACE_RECOVERY))
       trace_printf( "%s@%d - pnid=%d, comm_state=%d, state=%s, isInQuiesceState=%d,"
                     " (local pnid=%d, state=%s, isInQuiesceState=%d, "
                     "shutdown level=%d)\n", method_name, __LINE__,
                     pnid, communicate_state, StateString(node->GetState()),
                     node->isInQuiesceState(),
                     MyPNID, StateString(MyNode->GetState()),
                     MyNode->isInQuiesceState(), MyNode->GetShutdownLevel() );

    if (( MyPNID == pnid              ) &&
        ( MyNode->GetState() == State_Down ||
          MyNode->IsKillingNode() ) )
    {
        // we are coming down ... don't process it
        if ( !IsRealCluster && MyNode->isInQuiesceState())
        {
          // in virtual env, this would be called after node quiescing,
          // so continue with mark down processing.
        }
        else
        {
          return;
        }
    }

    if ( (MyNode->GetShutdownLevel() != ShutdownLevel_Undefined) &&
         (pnid != MyPNID) ) // some other node went down while shutdown was in progress
    {
        snprintf(buf, sizeof(buf), "[%s], Node failure during shutdown, down nid = %d\n", method_name, pnid);
        mon_log_write(MON_CLUSTER_MARKDOWN_1, SQ_LOG_ERR, buf);

        if (!waitForWatchdogExit_) // if WDT is not exiting
        {
            // bring down this node because TSE backup processes may not exit
            // if the primary was on the node that went down.
            ReqQueue.enqueueDownReq(MyPNID);
        }
    }

    if ( communicate_state && pnid != MyPNID )
    {
        // just communicate the change and let the real node handle it.
        node->SetChangeState( true );
        return;
    }

#ifndef NAMESERVER_PROCESS
    if ( !Emulate_Down )
    {
        if( !IsRealCluster )
        {
            snprintf(port_fname, sizeof(port_fname), "%s/monitor.%d.port.%s",getenv("MPI_TMPDIR"),pnid,node->GetName());
        }
        else
        {
            // Remove the domain portion of the name if any
            char short_node_name[MPI_MAX_PROCESSOR_NAME];
            char str1[MPI_MAX_PROCESSOR_NAME];
            memset( short_node_name, 0, MPI_MAX_PROCESSOR_NAME );
            memset( str1, 0, MPI_MAX_PROCESSOR_NAME );
            strcpy (str1, node->GetName() );

            char *str1_dot = strchr( (char *) str1, '.' );
            if ( str1_dot )
            {
                memcpy( short_node_name, str1, str1_dot - str1 );
            }
            else
            {
                strcpy (short_node_name, str1 );
            }
            snprintf(port_fname, sizeof(port_fname), "%s/monitor.port.%s",getenv("MPI_TMPDIR"),short_node_name);
        }
        sprintf(temp_fname, "%s.bak", port_fname);
        remove(temp_fname);
        rename(port_fname, temp_fname);
    }
#endif

    if (node->GetState() != State_Down || !node->isInQuiesceState())
    {
        snprintf(buf, sizeof(buf),
                 "[CCluster::HardNodeDown], Node %s (%d) is going down.\n",
                 node->GetName(), node->GetPNid());
        mon_log_write(MON_CLUSTER_MARKDOWN_2, SQ_LOG_CRIT, buf);

        node->SetKillingNode( true );

        if ( MyPNID == pnid &&
             (MyNode->GetState() == State_Up || MyNode->GetState() == State_Shutdown) &&
            !MyNode->isInQuiesceState() )
        {
            STATE state = MyNode->GetState();
            switch ( state )
            {
            case State_Up:
            case State_Shutdown:
                // do node quiescing and let HealthCheck thread know that quiescing has started
                // setting internal state to 'quiesce' will prevent replicating process exits
                // and reject normal shutdown requests in all nodes while we are quiescing.
                if (!waitForWatchdogExit_) // if WDT is not exiting
                {
                    MyNode->setQuiesceState();
                    HealthCheck.setState(MON_NODE_QUIESCE);
                }
                break;
            default: // in all other states
                if ( ! Emulate_Down )
                {
                    // make sure no processes are alive if in the middle of re-integration
#ifndef NAMESERVER_PROCESS
                    node->KillAllDown();
#endif
                    snprintf(buf, sizeof(buf),
                             "[CCluster::HardNodeDown], Node %s (%d)is down.\n",
                             node->GetName(), node->GetPNid());
                    mon_log_write(MON_CLUSTER_MARKDOWN_3, SQ_LOG_ERR, buf);
                    // Don't generate a core file, abort is intentional
                    struct rlimit limit;
                    limit.rlim_cur = 0;
                    limit.rlim_max = 0;
                    setrlimit(RLIMIT_CORE, &limit);
                    MPI_Abort(MPI_COMM_SELF,99);
                }
            }
        }
        else
        {
            if ( node->GetPNid() == integratingPNid_ )
            {
                ResetIntegratingPNid();
            }
#ifndef NAMESERVER_PROCESS
            node->KillAllDown();
#endif
            node->SetState( State_Down );
            // Send node down message to local node's processes
            lnode = node->GetFirstLNode();
            for ( ; lnode; lnode = lnode->GetNextP() )
            {
                lnode->Down();
            }
            if ( ZClientEnabled )
            {
                ZClient->WatchNodeDelete( node->GetName() );
                ZClient->WatchNodeMasterDelete( node->GetName() );
            }
        }
    }

#ifndef NAMESERVER_PROCESS
    // we need to abort any active TmSync
    if (( MyNode->GetTmSyncState() == SyncState_Start    ) ||
        ( MyNode->GetTmSyncState() == SyncState_Continue ) ||
        ( MyNode->GetTmSyncState() == SyncState_Commit   )   )
    {
        MyNode->SetTmSyncState( SyncState_Abort );
        Monitor->SetAbortPendingTmSync();
        if (trace_settings & (TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
           trace_printf("%s@%d - Node %s (pnid=%d) TmSyncState updated (%d)(%s)\n", method_name, __LINE__, MyNode->GetName(), MyPNID, MyNode->GetTmSyncState(), SyncStateString( MyNode->GetTmSyncState() ));
    }
#endif

#ifndef NAMESERVER_PROCESS
    if ( Emulate_Down )
    {
        AssignTmLeader(pnid, false);
    }
    else
#endif
    {
        AssignLeaders(pnid, node->GetName(), false);
    }

    TRACE_EXIT;
}

void CCluster::SoftNodeDown( int pnid )
{
    CNode  *node;
    char    buf[MON_STRING_BUF_SIZE];

    const char method_name[] = "CCluster::SoftNodeDown";
    TRACE_ENTRY;

    node = Nodes->GetNode(pnid);

    if (trace_settings & (TRACE_REQUEST | TRACE_INIT | TRACE_RECOVERY))
    {
        trace_printf( "%s@%d - pnid=%d, state=%s, phase=%s, isInQuiesceState=%d, isSoftNodeDown=%d"
                      " (local pnid=%d, state=%s, phase=%s, isInQuiesceState=%d, isSoftNodeDown=%d "
                      "shutdown level=%d)\n"
                    , method_name, __LINE__
                    , pnid, StateString(node->GetState())
                    , NodePhaseString(node->GetPhase())
                    , node->isInQuiesceState()
                    , node->IsSoftNodeDown()
                    , MyPNID, StateString(MyNode->GetState())
                    , NodePhaseString(MyNode->GetPhase())
                    , MyNode->isInQuiesceState()
                    , MyNode->IsSoftNodeDown()
                    , MyNode->GetShutdownLevel() );
    }

    if (( MyPNID == pnid              ) &&
        ( MyNode->GetState() == State_Down ||
          MyNode->IsKillingNode() ) )
    {
        // we are coming down ... don't process it
        return;
    }

    snprintf( buf, sizeof(buf)
            , "[%s], Node %s (%d) is going soft down.\n"
            , method_name, node->GetName(), node->GetPNid());
    mon_log_write(MON_CLUSTER_SOFTNODEDOWN_1, SQ_LOG_ERR, buf);

    node->SetKillingNode( true );

    if ( node->GetState() == State_Up )
    {
        node->SetSoftNodeDown();            // Set soft down flag
        node->SetPhase( Phase_SoftDown );   // Suspend TMSync on node

        if ( node->GetPNid() == MyPNID )
        {
            // and tell remote monitor processes the node is soft down
            CReplSoftNodeDown *repl = new CReplSoftNodeDown( MyPNID );
            Replicator.addItem(repl);
        }

#ifndef NAMESERVER_PROCESS
        node->KillAllDownSoft();            // Kill all processes
#endif

        snprintf( buf, sizeof(buf)
                , "[%s], Node %s (%d) executed soft down.\n"
                , method_name, node->GetName(), node->GetPNid() );
        mon_log_write(MON_CLUSTER_SOFTNODEDOWN_2, SQ_LOG_ERR, buf);
    }
    else
    {
        snprintf( buf, sizeof(buf),
                  "[%s], Node %s (%d) soft node down not executed, state=%s\n"
                , method_name, node->GetName()
                , node->GetPNid()
                , StateString(MyNode->GetState()) );
        mon_log_write(MON_CLUSTER_SOFTNODEDOWN_3, SQ_LOG_ERR, buf);
        // Probably a programmer bonehead!
        abort();
    }

#ifndef NAMESERVER_PROCESS
    // we need to abort any active TmSync
    if (( MyNode->GetTmSyncState() == SyncState_Start    ) ||
        ( MyNode->GetTmSyncState() == SyncState_Continue ) ||
        ( MyNode->GetTmSyncState() == SyncState_Commit   )   )
    {
        MyNode->SetTmSyncState( SyncState_Abort );
        Monitor->SetAbortPendingTmSync();
        if (trace_settings & (TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
           trace_printf("%s@%d - Node %s (pnid=%d) TmSyncState updated (%d)(%s)\n", method_name, __LINE__, MyNode->GetName(), MyPNID, MyNode->GetTmSyncState(), SyncStateString( MyNode->GetTmSyncState() ));
    }
#endif

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST | TRACE_SYNC | TRACE_TMSYNC))
    {
        trace_printf( "%s@%d - Node pnid=%d (%s), phase=%s, isSoftNodeDown=%d\n"
                    , method_name, __LINE__
                    , node->GetPNid()
                    , node->GetName()
                    , NodePhaseString(node->GetPhase())
                    , node->IsSoftNodeDown());
    }

    AssignLeaders(pnid, node->GetName(), false);

    TRACE_EXIT;
}

bool CCluster::CheckSpareSet( int pnid )
{
    bool activatedSpare = false;
    bool done = false;
    unsigned int ii;
    unsigned int jj;
    CNode *newNode = Nodes->GetNode( pnid );

    const char method_name[] = "CCluster::CheckSpareSet";
    TRACE_ENTRY;

    // Build spare node set
    CNode *spareNode;
    NodesList spareSetList;
    NodesList *spareNodesConfigList = Nodes->GetSpareNodesConfigList();
    NodesList::iterator itSn;
    for ( itSn = spareNodesConfigList->begin();
          itSn != spareNodesConfigList->end() && !done ; itSn++ )
    {
        spareNode = *itSn;
        PNidVector sparePNids = spareNode->GetSparePNids();
        // if the new node is a spare node in the configuration
        if ( newNode->GetPNid() == spareNode->GetPNid() )
        {
            // Add the spare node and each node it is configured to spare to the set
            spareSetList.push_back( spareNode );

            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                trace_printf("%s@%d - pnid=%d, name=(%s) is a configured Spare\n", method_name, __LINE__, spareNode->GetPNid(), spareNode->GetName());

            for ( ii = 0; ii < sparePNids.size(); ii++ )
            {
                spareSetList.push_back( Nodes->GetNode(sparePNids[ii]) );

                if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                    trace_printf("%s@%d - pnid=%d, name=(%s) is in Spare set\n", method_name, __LINE__, Nodes->GetNode(sparePNids[ii])->GetPNid(), Nodes->GetNode(sparePNids[ii])->GetName());
            }
            done = true;
        }
        else
        {
            // Check each pnid it is configured to spare
            for ( jj = 0; jj < sparePNids.size(); jj++ )
            {
                // if the new node is in the spare set of a spare node
                if ( newNode->GetPNid() == sparePNids[jj] )
                {
                    // Add the spare node and each node it is configured to spare to the set
                    spareSetList.push_back( spareNode );

                    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                        trace_printf("%s@%d - pnid=%d, name=(%s) is a configured Spare\n", method_name, __LINE__, spareNode->GetPNid(), spareNode->GetName());

                    for ( ii = 0; ii < sparePNids.size(); ii++ )
                    {
                        spareSetList.push_back( Nodes->GetNode(sparePNids[ii]) );

                        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                            trace_printf("%s@%d - pnid=%d, name=(%s) is in Spare set\n", method_name, __LINE__, Nodes->GetNode(sparePNids[ii])->GetPNid(), Nodes->GetNode(sparePNids[ii])->GetName());
                    }
                    done = true;
                }
            }
        }
    }

    if (newNode && trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        trace_printf( "%s@%d - new node pnid=%d, name=(%s), zid=%d\n"
                    , method_name, __LINE__
                    , newNode->GetPNid(), newNode->GetName(), newNode->GetZone());
    }

    // if the newNode still owns the zone
    if ( newNode && newNode->GetZone() != -1 )
    {
        // assume implicit spare node activation
        // (no need to move logical nodes to physical node)
        // since HardNodeUp() already set State_Up,
        // just reset spare node flag and remove from available spare nodes
        newNode->ResetSpareNode();
        Nodes->RemoveFromSpareNodesList( newNode );
        ActivateSpare( newNode, newNode );
        activatedSpare = true;
        TRACE_EXIT;
        return( activatedSpare );
    }

    CLNode  *lnode;
    CNode   *node;
    CNode   *downNode = NULL;

    // Now check the state of each configured logical node in the set for down state
    spareNode = newNode;  // new node (pnid) is the spare to activate
    NodesList::iterator itSs;
    for ( itSs = spareSetList.begin(); itSs != spareSetList.end(); itSs++ )
    {
        node = *itSs;
        if ( node->GetPNid() != pnid )
        {
            // Find the first down node
            if ( !downNode )
            {
                lnode = node->GetFirstLNode();
                if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                    trace_printf( "%s@%d - node nid=%d, pnid=%d(%s), state=%s\n"
                                , method_name, __LINE__, lnode?lnode->GetNid():-1
                                , node->GetPNid(), node->GetName()
                                , StateString( node->GetState() ) );
                if ( lnode && lnode->GetState() == State_Down )
                {
                    downNode = node;
                }
            }
        }
        if ( spareNode && downNode )
        {
            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                trace_printf( "%s@%d - spare node pnid=%d (%s), down node pnid=%d (%s) \n"
                            , method_name, __LINE__
                            , spareNode->GetPNid(), spareNode->GetName()
                            , downNode->GetPNid(), downNode->GetName());
            break;
        }
    }

    if ( spareNode && downNode )
    {
        Nodes->RemoveFromSpareNodesList( spareNode );
        spareNode->ResetSpareNode();
        if ( downNode->GetPNid() != pnid )
        { // the spare node does not own the down logical nodes so activate it
            ActivateSpare( spareNode, downNode );
        }
        activatedSpare = true;
    }

    TRACE_EXIT;
    return( activatedSpare );
}

const char *JoiningPhaseString( JOINING_PHASE phase )
{
    const char *str;

    switch( phase )
    {
        case JoiningPhase_Unknown:
            str = "JoiningPhase_Unknown";
            break;
        case JoiningPhase_1:
            str = "JoiningPhase_1";
            break;
        case JoiningPhase_2:
            str = "JoiningPhase_2";
            break;
        case JoiningPhase_3:
            str = "JoiningPhase_3";
            break;
        default:
            str = "JoiningPhase - Undefined";
            break;
    }

    return( str );
}

struct message_def *CCluster::JoinMessage( const char *node_name, int pnid, JOINING_PHASE phase )
{
    struct message_def *msg;

    const char method_name[] = "CCluster::JoinMessage";
    TRACE_ENTRY;

    // Record statistics (sonar counters)
    if (sonar_verify_state(SONAR_ENABLED | SONAR_MONITOR_ENABLED))
       MonStats->notice_death_Incr();

    msg = new struct message_def;
    msg->type = MsgType_NodeJoining;
    msg->noreply = true;
    msg->u.request.type = ReqType_Notice;
    strcpy( msg->u.request.u.joining.node_name, node_name );
    msg->u.request.u.joining.pnid = pnid;
    msg->u.request.u.joining.phase = phase;

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST_DETAIL))
        trace_printf("%s@%d - Joining notice for node %s (pnid=%d, phase=%d)\n",
                     method_name, __LINE__, node_name, pnid, phase );
    TRACE_EXIT;

    return msg;
}

struct message_def *CCluster::SpareUpMessage( const char *node_name, int pnid )
{
    struct message_def *msg;

    const char method_name[] = "CCluster::SpareUpMessage";
    TRACE_ENTRY;

    // Record statistics (sonar counters)
    if (sonar_verify_state(SONAR_ENABLED | SONAR_MONITOR_ENABLED))
       MonStats->notice_death_Incr();

    msg = new struct message_def;
    msg->type = MsgType_SpareUp;
    msg->noreply = true;
    msg->u.request.type = ReqType_Notice;
    strcpy( msg->u.request.u.spare_up.node_name, node_name );
    msg->u.request.u.spare_up.pnid = pnid;

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST_DETAIL))
        trace_printf("%s@%d - Spare node up notice for node %s nid=%d\n",
                     method_name, __LINE__, node_name, pnid );
    TRACE_EXIT;

    return msg;
}

struct message_def *CCluster::ReIntegErrorMessage( const char *msgText )
{
    struct message_def *msg;

    const char method_name[] = "CCluster::ReIntegErrorMessage";
    TRACE_ENTRY;

    msg = new struct message_def;
    msg->type = MsgType_ReintegrationError;
    msg->noreply = true;
    msg->u.request.type = ReqType_Notice;
    strncpy( msg->u.request.u.reintegrate.msg, msgText,
             sizeof(msg->u.request.u.reintegrate.msg) );

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST_DETAIL))
        trace_printf("%s@%d - Reintegrate notice %s\n",
                     method_name, __LINE__, msgText );

    TRACE_EXIT;

    return msg;
}

int CCluster::HardNodeUp( int pnid, char *node_name )
{
    bool    spareNodeActivated = false;
    int     rc = MPI_SUCCESS;
    int     tmCount = 0;
    CNode  *node;
    CLNode *lnode;
    STATE   nodeState;

    const char method_name[] = "CCluster::HardNodeUp";
    TRACE_ENTRY;

    if (trace_settings & (TRACE_REQUEST | TRACE_INIT | TRACE_RECOVERY))
       trace_printf( "%s@%d - pnid=%d, name=%s (MyPNID = %d), currentNodes_=%d\n"
                   , method_name, __LINE__, pnid, node_name, MyPNID, currentNodes_ );

    if ( pnid == -1 )
    {
        node = Nodes->GetNode( node_name );
    }
    else
    {
        node = Nodes->GetNode( pnid );
    }

    if ( node == NULL )
    {
        if (trace_settings & (TRACE_REQUEST | TRACE_INIT | TRACE_RECOVERY))
           trace_printf( "%s@%d" " - Invalid node, pnid=%d, name=%s" "\n"
                       , method_name, __LINE__, pnid, node_name );

        return( MPI_ERR_NAME );
    }

    nodeState = node->GetState();

    if (trace_settings & (TRACE_REQUEST | TRACE_INIT | TRACE_RECOVERY))
       trace_printf( "%s@%d" " - Node state=%s" "\n"
                   , method_name, __LINE__, StateString( nodeState ) );
    if ( nodeState != State_Up )
    {
        if ( nodeState == State_Down )
        {
            node->SetKillingNode( false );
            if ( Emulate_Down )
            {
                // Any DTMs running?
                for ( int i=0; !tmCount && i < Nodes->GetPNodesCount(); i++ )
                {
                    CNode  *tempNode = Nodes->GetNodeByMap( i );
                    lnode = tempNode->GetFirstLNode();
                    for ( ; lnode; lnode = lnode->GetNextP() )
                    {
                        CProcess *process = lnode->GetProcessLByType( ProcessType_DTM );
                        if ( process  ) tmCount++;
                    }
                }
                if ( tmCount )
                {
                    IAmIntegrated = true;
                }
                // We need to remove any old process objects before we restart the node.
                node->CleanUpProcesses();
                node->SetState( State_Up );
                if ( MyPNID == pnid )
                {
                    MyNode->clearQuiesceState();
                    HealthCheck.initializeVars();
                    SMSIntegrating = true;
#ifndef NAMESERVER_PROCESS
                    Monitor->StartPrimitiveProcesses();
#endif
                    // Let other monitors know this node is up
                    CReplNodeUp *repl = new CReplNodeUp(MyPNID);
                    Replicator.addItem(repl);
                }
                else
                {
                    if ( tmCount )
                    {
#ifndef NAMESERVER_PROCESS
                        // Send node prepare notice to local DTM processes
                        lnode = node->GetFirstLNode();
                        for ( ; lnode; lnode = lnode->GetNextP() )
                        {
                            lnode->PrepareForTransactions( true );
                        }
#endif
                    }
                    else
                    {
                        // Process logical node up
                        lnode = node->GetFirstLNode();
                        for ( ; lnode; lnode = lnode->GetNextP() )
                        {
                            lnode->Up();
                        }
                    }
                }
            }
            else
            {
                if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                    trace_printf( "%s@%d - Unexpectedly executing HardNodeUp.  Expecting to do accept in commAccept thread\n",
                                  method_name, __LINE__ );

            }
        }
        else if ( nodeState == State_Merged )
        {
            node->SetKillingNode( false );
            node->SetState( State_Joining );

            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                trace_printf( "%s@%d" " - New monitor %s, pnid=%d, state=%s" "\n"
                            , method_name, __LINE__, node->GetName(), node->GetPNid(), StateString( node->GetState() ) );
                for ( int i =0; i < Nodes->GetPNodesCount(); i++ )
                {
                    trace_printf( "%s@%d socks_[indexToPnid_[%d]=%d]=%d, sockPorts_[indexToPnid_[%d]=%d]=%d\n"
                                , method_name, __LINE__
                                , i, indexToPnid_[i], socks_[indexToPnid_[i]]
                                , i, indexToPnid_[i], sockPorts_[indexToPnid_[i]] );
                }
            }
            if ( MyNode->IsCreator() )
            {
#ifndef NAMESERVER_PROCESS
                SQ_theLocalIOToClient->putOnNoticeQueue( MyNode->GetCreatorPid()
                                                       , MyNode->GetCreatorVerifier()
                                                       , JoinMessage( node->GetName()
                                                                    , node->GetPNid()
                                                                    , JoiningPhase_1 )
                                                       , NULL);
#endif

                // save the current seq num in the snapshot request.
                // this sequence number will match the state of the cluster
                // when this request is processed.
                ReqQueue.enqueueSnapshotReq(seqNum_);
            }
            if ( MyPNID == pnid )
            {
                // request and process revive packet from the creator.
                // when complete, this will call HardNodeUp again.
                ReqQueue.enqueueReviveReq( );
            }
            else
            {
                if ( ZClientEnabled )
                {
                    rc = ZClient->WatchNode( node->GetName() );
                    if ( rc != ZOK )
                    {
                        char    buf[MON_STRING_BUF_SIZE];
                        snprintf( buf, sizeof(buf)
                                , "[%s], Unable to set node watch on %s, pnid%d\n"
                                , method_name, node->GetName(), node->GetPNid() );
                        mon_log_write(MON_CLUSTER_HARDNODEUP_1, SQ_LOG_ERR, buf);
                    }
                }
            }
        }
        else if ( nodeState == State_Joining )
        {
            // The new monitor comes in here first and schedules a node up request on all nodes.
            // All other monitors come here next, including the creator.
            // The new monitor will not come here again because
            // CReplNodeUp is a noop for the one who schedules it.
            node->SetState( State_Up );

            if ( Nodes->GetSNodesCount() == 0 )
            { // Spare nodes not configured so bring up my logical nodes
                if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                   trace_printf( "%s@%d" " - No spare nodes configured node=%s, pnid=%d, state=%s\n"
                               , method_name, __LINE__, node->GetName(), node->GetPNid()
                               , StateString(node->GetState()) );
                if ( MyPNID == pnid )
                {
                    ActivateSpare( node, node );
                }
            }
            else
            {
                node->SetSpareNode();
                Nodes->AddToSpareNodesList( node->GetPNid() );
                if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                   trace_printf( "%s@%d" " - Adding to available spares node=%s, pnid=%d\n"
                               , method_name, __LINE__, node->GetName(), node->GetPNid() );
                // Check for a node down in spare set and activate down node if found
                spareNodeActivated = CheckSpareSet( node->GetPNid() );
                if ( spareNodeActivated )
                {
                    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                       trace_printf( "%s@%d" " - Activated spare node=%s, pnid=%d\n"
                                   , method_name, __LINE__, node->GetName(), node->GetPNid() );
                }
                else
                {
                    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                       trace_printf( "%s@%d" " - Available spare node=%s, pnid=%d\n"
                                   , method_name, __LINE__, node->GetName(), node->GetPNid() );

                    // Spare node not activated
                    if ( MyNode->IsCreator() )
                    {
                        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                           trace_printf( "%s@%d" " - Sending spare up notice to creator shell(%d) spare node=%s, pnid=%d\n"
                                       , method_name, __LINE__, MyNode->GetCreatorPid(), node->GetName(), node->GetPNid() );
#ifndef NAMESERVER_PROCESS
                        // Tell creator spare node is up
                        SQ_theLocalIOToClient->putOnNoticeQueue( MyNode->GetCreatorPid()
                                                               , MyNode->GetCreatorVerifier()
                                                               , SpareUpMessage( node->GetName()
                                                                               , node->GetPNid() )
                                                               , NULL);
#endif
                    }
                }
            }

            if ( MyPNID == pnid )
            {
                // Any DTMs running?
                for ( int i=0; !tmCount && i < Nodes->GetPNodesCount(); i++ )
                {
                    CNode  *tempNode = Nodes->GetNodeByMap( i );
                    lnode = tempNode->GetFirstLNode();
                    for ( ; lnode; lnode = lnode->GetNextP() )
                    {
                        CProcess *process = lnode->GetProcessLByType( ProcessType_DTM );
                        if ( process  ) tmCount++;
                    }
                }
                if ( !tmCount && !spareNodeActivated )
                {
                    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                       trace_printf( "%s@%d" " - Replicating node up %s, pnid=%d, state=%s, spare=%d, DTM count=%d\n"
                                   , method_name, __LINE__, node->GetName(), node->GetPNid()
                                   , StateString(node->GetState()), node->IsSpareNode(), tmCount );
                    // Let other monitors know this node is up
                    CReplNodeUp *repl = new CReplNodeUp(MyPNID);
                    Replicator.addItem(repl);
                }
            }

            ResetIntegratingPNid();

            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
               trace_printf( "%s@%d" " - New monitor %s, pnid=%d, state=%s, spare=%d\n"
                           , method_name, __LINE__, node->GetName(), node->GetPNid()
                           , StateString(node->GetState()), node->IsSpareNode() );
        }
    }

    TRACE_EXIT;
    return( rc );
}

int CCluster::SoftNodeUpPrepare( int pnid )
{
    char    buf[MON_STRING_BUF_SIZE];
    int     rc = MPI_SUCCESS;
    int     tmCount = 0;
    CNode  *node;
    CLNode *lnode;
    STATE   nodeState;

    const char method_name[] = "CCluster::SoftNodeUpPrepare";
    TRACE_ENTRY;

    node = Nodes->GetNode( pnid );
    if ( node == NULL )
    {
        if (trace_settings & (TRACE_REQUEST | TRACE_INIT | TRACE_RECOVERY))
           trace_printf( "%s@%d - Invalid node, pnid=%d\n"
                       , method_name, __LINE__, pnid );

        return( MPI_ERR_NAME );
    }

    nodeState = node->GetState();

    if (trace_settings & (TRACE_REQUEST | TRACE_INIT | TRACE_RECOVERY))
       trace_printf( "%s@%d - Node name=%s, pnid=%d, state=%s, soft down=%d\n"
                   , method_name, __LINE__
                   , node->GetName()
                   , node->GetPNid()
                   , StateString( nodeState )
                   , node->IsSoftNodeDown() );

    if ( nodeState != State_Up )
    {
        if (trace_settings & (TRACE_REQUEST | TRACE_INIT | TRACE_RECOVERY))
            trace_printf( "%s@%d - Unexpectedly executing SoftNodeUp\n",
                          method_name, __LINE__ );
        // Programmer bonehead!
        abort();
    }

    node->SetKillingNode( false );

    node->ResetSoftNodeDown( );

    node->SetPhase( Phase_Ready );

    if ( MyPNID == pnid )
    {
        SMSIntegrating = true;
#ifndef NAMESERVER_PROCESS
        node->SetSoftNodeUp( );
        Monitor->StartPrimitiveProcesses();
#endif
        // Let other monitors know this node is preparing to soft up
        CReplSoftNodeUp *repl = new CReplSoftNodeUp(MyPNID);
        Replicator.addItem(repl);
    }
    else
    {
        // Any DTMs running?
        for ( int i=0; !tmCount && i < Nodes->GetPNodesCount(); i++ )
        {
            CNode  *tempNode = Nodes->GetNodeByMap( i );
            lnode = tempNode->GetFirstLNode();
            for ( ; lnode; lnode = lnode->GetNextP() )
            {
                CProcess *process = lnode->GetProcessLByType( ProcessType_DTM );
                if ( process  ) tmCount++;
            }
        }
        if ( tmCount )
        {
#ifndef NAMESERVER_PROCESS
            // Send DTM restarted notice to local DTM processes
            lnode = node->GetFirstLNode();
            for ( ; lnode; lnode = lnode->GetNextP() )
            {
                lnode->SendDTMRestarted();
            }
#endif
        }
        else
        {
            snprintf( buf, sizeof(buf),
                      "[%s], Node %s (%d) soft node up prepare not executed, state=%s, tmCount=%d\n"
                    , method_name, node->GetName()
                    , node->GetPNid()
                    , StateString(MyNode->GetState())
                    , tmCount );
            mon_log_write(MON_CLUSTER_SOFTNODEUP_1, SQ_LOG_WARNING, buf);
        }
    }

    TRACE_EXIT;
    return( rc );
}


const char *StateString( STATE state)
{
    const char *str;

    switch( state )
    {
        case State_Unknown:
            str = "State_Unknown";
            break;
        case State_Up:
            str = "State_Up";
            break;
        case State_Down:
            str = "State_Down";
            break;
        case State_Stopped:
            str = "State_Stopped";
            break;
        case State_Shutdown:
            str = "State_Shutdown";
            break;
        case State_Unlinked:
            str = "State_Unlinked";
            break;
        case State_Merging:
            str = "State_Merging";
            break;
        case State_Merged:
            str = "State_Merged";
            break;
        case State_Joining:
            str = "State_Joining";
            break;
        case State_Initializing:
            str = "State_Initializing";
            break;
        default:
            str = "State - Undefined";
            break;
    }

    return( str );
}

const char *SyncStateString( SyncState state)
{
    const char *str;

    switch( state )
    {
        case SyncState_Null:
            str = "SyncState_Null";
            break;
        case SyncState_Start:
            str = "SyncState_Start";
            break;
        case SyncState_Continue:
            str = "SyncState_Continue";
            break;
        case SyncState_Abort:
            str = "SyncState_Abort";
            break;
        case SyncState_Commit:
            str = "SyncState_Commit";
            break;
        case SyncState_Suspended:
            str = "SyncState_Suspended";
            break;
        default:
            str = "SyncState - Undefined";
            break;
    }

    return( str );
}


#ifndef NAMESERVER_PROCESS
void CCluster::AddTmsyncMsg( struct sync_buffer_def *tmSyncBuffer
                           , struct sync_def *sync
                           , struct internal_msg_def *msg)
{
    const char method_name[] = "CCluster::AddTmsyncMsg";
    TRACE_ENTRY;

    if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
        trace_printf("%s@%d - Requesting SyncType=%d\n", method_name,
                     __LINE__, sync->type);

    msg->type = InternalType_Sync;
    msg->u.sync.type = sync->type;
    msg->u.sync.pnid = sync->pnid;
    msg->u.sync.syncnid = sync->syncnid;
    msg->u.sync.tmleader = sync->tmleader;
    msg->u.sync.state = sync->state;
    msg->u.sync.count = sync->count;
    if ( sync->type == SyncType_TmData )
    {
        memmove (msg->u.sync.data, sync->data, sync->length);
    }
    msg->u.sync.length = sync->length;

    // We can have only have a single "InternalType_Sync" msg in our
    // SyncBuffer, else we cause a collision.

    int msgSize = (MSG_HDR_SIZE + sizeof(sync_def) - MAX_SYNC_DATA
                   + sync->length );

    // Insert the message size into the message header
    msg->replSize = msgSize;
    tmSyncBuffer->msgInfo.msg_count = 1;
    tmSyncBuffer->msgInfo.msg_offset += msgSize;

    // Set end-of-buffer marker
    msg = (struct internal_msg_def *)
        &tmSyncBuffer->msg[tmSyncBuffer->msgInfo.msg_offset];
    msg->type = InternalType_Null;

    TRACE_EXIT;
}
#endif

#ifndef NAMESERVER_PROCESS
void CCluster::DoDeviceReq(char * ldevName)
{
    const char method_name[] = "CCluster::DoDeviceReq";
    TRACE_ENTRY;

    CProcess *process;

    if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
        trace_printf("%s@%d - Internal device request for ldev %s\n",
                     method_name, __LINE__, ldevName);
    Nodes->GetLNode(ldevName, &process);
    if (!process)
    {
        if (trace_settings & TRACE_SYNC)
            trace_printf("%s@%d - Device processing but can't find device %s\n",
                         method_name, __LINE__, ldevName);
    }
    else
    {
        CLogicalDevice *ldev;
        ldev = Devices->GetLogicalDevice( ldevName );
        if ( !ldev )
        {   // The device name is not known on this node
            // we need to clone the device
            ldev = Devices->CloneDevice( process );
        }
        if ( ldev )
        {
            bool rstate = false;
            if ( ldev->Mounted() )
            {
                rstate = ldev->UnMount( false );
                if (!rstate)
                {
                    if (trace_settings & TRACE_REQUEST)
                        trace_printf("%s@%d - Can't unmount device %s for "
                                     "process %s (%d, %d)\n", method_name,
                                     __LINE__, ldev->name(), process->GetName(),
                                     process->GetNid(), process->GetPid());
                }
            }
            if ( rstate )
            {
                rstate = ldev->Mount( process, false );
                if (!rstate)
                {
                    if (trace_settings & TRACE_REQUEST)
                        trace_printf("%s@%d - Can't mount device %s for "
                                     "process %s (%d, %d)\n", method_name,
                                     __LINE__, ldev->name(), process->GetName(),
                                     process->GetNid(), process->GetPid());
                }
                else
                {
                    if (trace_settings & TRACE_REQUEST)
                        trace_printf("%s@%d - Mounted device %s for process "
                                     "%s (%d, %d)\n", method_name, __LINE__,
                                     ldev->name(), process->GetName(),
                                     process->GetNid(), process->GetPid());
                }
            }
        }
        else
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf(buf, sizeof(buf), "[%s], Can't find ldev %s.\n", method_name,
                    ldevName);
            mon_log_write(MON_CLUSTER_DODEVICEREQ_1, SQ_LOG_ERR, buf);
        }
    }

    TRACE_EXIT;
}
#endif

#ifdef EXCHANGE_CPU_SCHEDULING_DATA
void CCluster::SaveSchedData( struct internal_msg_def *recv_msg )
{
    const char method_name[] = "CCluster::SaveSchedData";
    TRACE_ENTRY;

    int nid = recv_msg->u.scheddata.PNid;
    Node[nid]->SetNumCores( recv_msg->u.scheddata.processors );
    Node[nid]->SetFreeMemory( recv_msg->u.scheddata.memory_free );
    Node[nid]->SetFreeSwap( recv_msg->u.scheddata.swap_free );
    Node[nid]->SetFreeCache( recv_msg->u.scheddata.cache_free );
    Node[nid]->SetMemTotal( recv_msg->u.scheddata.memory_total );
    Node[nid]->SetMemActive( recv_msg->u.scheddata.memory_active );
    Node[nid]->SetMemInactive( recv_msg->u.scheddata.memory_inactive );
    Node[nid]->SetMemDirty( recv_msg->u.scheddata.memory_dirty );
    Node[nid]->SetMemWriteback( recv_msg->u.scheddata.memory_writeback );
    Node[nid]->SetMemVMallocUsed( recv_msg->u.scheddata.memory_VMallocUsed );
    Node[nid]->SetBTime( recv_msg->u.scheddata.btime );

    CLNode *lnode;
    lnode = Node[nid]->GetFirstLNode();
    int i = 0;

    for ( ; lnode; lnode = lnode->GetNextP() )
    {
        lnode->SetCpuUser(recv_msg->u.scheddata.proc_stats[i].cpu_user);
        lnode->SetCpuNice(recv_msg->u.scheddata.proc_stats[i].cpu_nice);
        lnode->SetCpuSystem(recv_msg->u.scheddata.proc_stats[i].cpu_system);
        lnode->SetCpuIdle(recv_msg->u.scheddata.proc_stats[i].cpu_idle);
        lnode->SetCpuIowait(recv_msg->u.scheddata.proc_stats[i].cpu_iowait);
        lnode->SetCpuIrq(recv_msg->u.scheddata.proc_stats[i].cpu_irq);
        lnode->SetCpuSoftIrq(recv_msg->u.scheddata.proc_stats[i].cpu_soft_irq);

        ++i;
    }

    TRACE_EXIT;
}
#endif

void CCluster::HandleOtherNodeMsg (struct internal_msg_def *recv_msg,
                                   int pnid)
{
    const char method_name[] = "CCluster::HandleOtherNodeMsg";
    TRACE_ENTRY;

    CNode *downNode;
    CNode *spareNode;
#ifndef NAMESERVER_PROCESS
    CProcess *process;
    CLNode  *lnode;
#endif

    switch (recv_msg->type)
    {
    case InternalType_Null:
        if (trace_settings & TRACE_SYNC_DETAIL)
            trace_printf("%s@%d - Node n%d has nothing to "
                         "update. \n", method_name, __LINE__, pnid);
        break;

    case InternalType_ActivateSpare:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal activate spare request, spare pnid=%d, down pnid=%d\n"
                        , method_name, __LINE__
                        , recv_msg->u.activate_spare.spare_pnid
                        , recv_msg->u.activate_spare.down_pnid);

        downNode = NULL;
        if ( recv_msg->u.activate_spare.down_pnid != -1 )
        {
            downNode = Nodes->GetNode( recv_msg->u.activate_spare.down_pnid );
        }
        spareNode = Nodes->GetNode( recv_msg->u.activate_spare.spare_pnid );
        ReqQueue.enqueueActivateSpareReq( spareNode, downNode );
        break;

    case InternalType_NameServerAdd:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf( "%s@%d - Internal NameServer add request for node_name=%s\n"
                        , method_name, __LINE__
                        , recv_msg->u.nameserver_add.node_name );

        // Queue the nameserver add request for processing by a worker thread.
        ReqQueue.enqueueNameServerAddReq( recv_msg->u.nameserver_add.req_nid
                                        , recv_msg->u.nameserver_add.req_pid
                                        , recv_msg->u.nameserver_add.req_verifier
                                        , recv_msg->u.nameserver_add.node_name );
        break;

    case InternalType_NameServerDelete:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf( "%s@%d - Internal NameServer delete request for node=%s\n"
                        , method_name, __LINE__, recv_msg->u.nameserver_delete.node_name);

        // Queue the nameserver delete request for processing by a worker thread.
        ReqQueue.enqueueNameServerDeleteReq( recv_msg->u.nameserver_delete.req_nid
                                           , recv_msg->u.nameserver_delete.req_pid
                                           , recv_msg->u.nameserver_delete.req_verifier
                                           , recv_msg->u.nameserver_delete.node_name );
        break;

    case InternalType_NodeAdd:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf( "%s@%d - Internal node add request for node_name=%s, "
                          "first_core=%d, last_core=%d, "
                          "processors=%d, roles=%d\n"
                        , method_name, __LINE__
                        , recv_msg->u.node_add.node_name
                        , recv_msg->u.node_add.first_core
                        , recv_msg->u.node_add.last_core
                        , recv_msg->u.node_add.processors
                        , recv_msg->u.node_add.roles );

        // Queue the node add request for processing by a worker thread.
        ReqQueue.enqueueNodeAddReq( recv_msg->u.node_add.req_nid
                                  , recv_msg->u.node_add.req_pid
                                  , recv_msg->u.node_add.req_verifier
                                  , recv_msg->u.node_add.node_name
                                  , recv_msg->u.node_add.first_core
                                  , recv_msg->u.node_add.last_core
                                  , recv_msg->u.node_add.processors
                                  , recv_msg->u.node_add.roles );
        break;

    case InternalType_Clone:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal clone request, process (%d, %d)"
                         " %s\n", method_name, __LINE__,
                         recv_msg->u.clone.nid, recv_msg->u.clone.os_pid,
                         (recv_msg->u.clone.backup?" Backup":""));

        ReqQueue.enqueueCloneReq( &recv_msg->u.clone );
        break;

#ifndef NAMESERVER_PROCESS
    case InternalType_Device:
        ReqQueue.enqueueDeviceReq(recv_msg->u.device.ldev_name);
        break;
#endif

    case InternalType_Shutdown:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal shutdown request for level=%d\n", method_name, __LINE__, recv_msg->u.shutdown.level);

        // Queue the shutdown request for processing by a worker thread.
        ReqQueue.enqueueShutdownReq( recv_msg->u.shutdown.level );
        break;

    case InternalType_NodeDelete:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf( "%s@%d - Internal node delete request for pnid=%d\n"
                        , method_name, __LINE__, recv_msg->u.node_delete.pnid);

        // Queue the node delete request for processing by a worker thread.
        ReqQueue.enqueueNodeDeleteReq( recv_msg->u.node_delete.req_nid
                                     , recv_msg->u.node_delete.req_pid
                                     , recv_msg->u.node_delete.req_verifier
                                     , recv_msg->u.node_delete.pnid );
        break;

    case InternalType_Down:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal down node request for pnid=%d\n", method_name, __LINE__, recv_msg->u.down.pnid);

        // Queue the node down request for processing by a worker thread.
        ReqQueue.enqueueDownReq( recv_msg->u.down.pnid );
        break;

    case InternalType_NodeName:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal node name request (%s to %s)\n", method_name, __LINE__, recv_msg->u.node_name.current_name, recv_msg->u.node_name.new_name);

        // Queue the node name request for processing by a worker thread.
        ReqQueue.enqueueNodeNameReq( recv_msg->u.node_name.req_nid
                                   , recv_msg->u.node_name.req_pid
                                   , recv_msg->u.node_name.req_verifier
                                   , recv_msg->u.node_name.current_name
                                   , recv_msg->u.node_name.new_name );
        break;

    case InternalType_SoftNodeDown:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal soft node down request for pnid=%d\n", method_name, __LINE__, recv_msg->u.down.pnid);

        // Queue the node down request for processing by a worker thread.
        ReqQueue.enqueueSoftNodeDownReq( recv_msg->u.down.pnid );
        break;

    case InternalType_SoftNodeUp:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal soft node up request for pnid=%d\n", method_name, __LINE__, recv_msg->u.up.pnid);

        // Queue the node up request for processing by a worker thread.
        ReqQueue.enqueueSoftNodeUpReq( recv_msg->u.up.pnid );
        break;

    case InternalType_Up:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal up node request for pnid=%d\n", method_name, __LINE__, recv_msg->u.up.pnid);

        // Queue the node up request for processing by a worker thread.
        ReqQueue.enqueueUpReq( recv_msg->u.up.pnid, NULL, -1 );
        break;

#ifndef NAMESERVER_PROCESS
    case InternalType_Dump:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal dump request for nid=%d, pid=%d\n",
                         method_name, __LINE__,
                         recv_msg->u.dump.nid, recv_msg->u.dump.pid);
        lnode = Nodes->GetLNode( recv_msg->u.dump.nid );
        if ( lnode )
        {
            process = lnode->GetProcessL(recv_msg->u.dump.pid);

            if (process)
            {
                int verifier = recv_msg->u.dump.verifier;
                if ( (verifier == -1) || (verifier == process->GetVerifier()) )
                {
                    process->DumpBegin(recv_msg->u.dump.dumper_nid,
                                       recv_msg->u.dump.dumper_pid,
                                       recv_msg->u.dump.dumper_verifier,
                                       recv_msg->u.dump.core_file);
                }
                else
                {
                    char buf[MON_STRING_BUF_SIZE];
                    snprintf(buf, sizeof(buf), "[%s], Can't find process nid=%d, "
                             "pid=%d, verifier=%d for dump target.\n", method_name,
                             recv_msg->u.dump.nid, recv_msg->u.dump.pid,
                             recv_msg->u.dump.verifier);
                    mon_log_write(MON_CLUSTER_HANDLEOTHERNODE_1, SQ_LOG_ERR, buf);
                }
            }
            else
            {
                char buf[MON_STRING_BUF_SIZE];
                snprintf(buf, sizeof(buf), "[%s], Can't find process nid=%d, "
                         "pid=%d for dump target.\n", method_name,
                         recv_msg->u.dump.nid, recv_msg->u.dump.pid);
                mon_log_write(MON_CLUSTER_HANDLEOTHERNODE_2, SQ_LOG_ERR, buf);
            }
        }

        break;

    case InternalType_DumpComplete:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal dump-complete request for nid=%d, pid=%d\n",
                         method_name, __LINE__,
                         recv_msg->u.dump.nid, recv_msg->u.dump.pid);
        lnode = Nodes->GetLNode( recv_msg->u.dump.nid );
        if ( lnode )
        {
            process = lnode->GetProcessL(recv_msg->u.dump.pid);

            if (process)
            {
                int verifier = recv_msg->u.dump.verifier;
                if ( (verifier == -1) || (verifier == process->GetVerifier()) )
                {
                    process->DumpEnd(recv_msg->u.dump.status, recv_msg->u.dump.core_file);
                }
                else
                {
                    char buf[MON_STRING_BUF_SIZE];
                    snprintf(buf, sizeof(buf), "[%s], Can't find process nid=%d, "
                             "pid=%d, verifier=%d for dump target.\n", method_name,
                             recv_msg->u.dump.nid, recv_msg->u.dump.pid,
                             recv_msg->u.dump.verifier);
                    mon_log_write(MON_CLUSTER_HANDLEOTHERNODE_3, SQ_LOG_ERR, buf);
                }
            }
            else
            {
                // Dump completion handled in CProcess::Exit()
                char buf[MON_STRING_BUF_SIZE];
                snprintf(buf, sizeof(buf), "[%s], Can't find process nid=%d, "
                         "pid=%d for dump complete target.\n", method_name,
                         recv_msg->u.dump.nid, recv_msg->u.dump.pid);
                mon_log_write(MON_CLUSTER_HANDLEOTHERNODE_4, SQ_LOG_ERR, buf);
            }
        }
        break;
#endif

    case InternalType_Exit:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal exit request for %s (%d, %d)\n", method_name, __LINE__, recv_msg->u.exit.name, recv_msg->u.exit.nid, recv_msg->u.exit.pid);
#ifndef NAMESERVER_PROCESS
        ReqQueue.enqueueExitReq( &recv_msg->u.exit );
#else
        ReqQueue.enqueueExitNsReq( &recv_msg->u.exit_ns );
#endif
        break;

#ifndef NAMESERVER_PROCESS
    case InternalType_Event:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal event request\n", method_name, __LINE__);
        if ( MyNode->IsMyNode(recv_msg->u.event.nid) )
        {
            if (trace_settings & TRACE_SYNC)
                trace_printf("%s@%d - processing event for (%d, %d)\n", method_name, __LINE__, recv_msg->u.event.nid, recv_msg->u.event.pid);

            lnode = Nodes->GetLNode( recv_msg->u.event.nid );
            if ( lnode )
            {
                process = lnode->GetProcessL(recv_msg->u.event.pid);

                if (process)
                {
                    int verifier = recv_msg->u.dump.verifier;
                    if ( (verifier == -1) || (verifier == process->GetVerifier()) )
                    {
                        process->GenerateEvent (recv_msg->u.event.event_id,
                                                recv_msg->u.event.length,
                                                &recv_msg->u.event.data);
                    }
                    else
                    {
                        char buf[MON_STRING_BUF_SIZE];
                        snprintf(buf, sizeof(buf), "[%s], Can't find process nid=%d, "
                                 "pid=%d, verifier=%d for event=%d\n", method_name,
                                 recv_msg->u.event.nid, recv_msg->u.event.pid,
                                 recv_msg->u.event.verifier, recv_msg->u.event.event_id);
                        mon_log_write(MON_CLUSTER_HANDLEOTHERNODE_5, SQ_LOG_ERR, buf);
                    }
                }
                else
                {
                    char buf[MON_STRING_BUF_SIZE];
                    snprintf(buf, sizeof(buf), "[%s], Can't find process nid"
                             "=%d, pid=%d for processing event.\n",
                             method_name,
                             recv_msg->u.event.nid, recv_msg->u.event.pid);
                    mon_log_write(MON_CLUSTER_HANDLEOTHERNODE_6, SQ_LOG_ERR,
                                  buf);
                }
            }
        }
        break;
#endif

#ifndef NAMESERVER_PROCESS
    case InternalType_IoData:
        if (trace_settings & (TRACE_SYNC_DETAIL | TRACE_REQUEST_DETAIL | TRACE_REDIRECTION))
            trace_printf("%s@%d - Internal IO data request\n", method_name, __LINE__);
        if ( MyNode->IsMyNode(recv_msg->u.iodata.nid) )
        {
            if (trace_settings & (TRACE_SYNC | TRACE_REDIRECTION))
                trace_printf("%s@%d - processing IO Data for (%d, %d)\n", method_name, __LINE__, recv_msg->u.iodata.nid, recv_msg->u.iodata.pid);

            lnode = Nodes->GetLNode( recv_msg->u.iodata.nid );
            if ( lnode )
            {
                process = lnode->GetProcessL(recv_msg->u.iodata.pid);

                if (process)
                {
                    int fd;
                    if (recv_msg->u.iodata.ioType == STDIN_DATA)
                    {
                        fd = process->FdStdin();
                    }
                    else
                    {
                        fd = process->FdStdout();
                    }
                    Redirector.disposeIoData(fd,
                                             recv_msg->u.iodata.length,
                                             recv_msg->u.iodata.data);
                }
                else
                {
                    char buf[MON_STRING_BUF_SIZE];
                    snprintf(buf, sizeof(buf), "[%s], Can't find process nid"
                             "=%d, pid=%d for processing IO Data.\n",
                             method_name,
                             recv_msg->u.iodata.nid, recv_msg->u.iodata.pid);
                    mon_log_write(MON_CLUSTER_HANDLEOTHERNODE_7, SQ_LOG_ERR,
                                  buf);
                }
            }
        }
        break;
#endif

#ifndef NAMESERVER_PROCESS
    case InternalType_StdinReq:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal STDIN request\n", method_name, __LINE__);
        if ( !MyNode->IsMyNode(recv_msg->u.stdin_req.supplier_nid) )
        {
            break;

        }

        if (trace_settings & (TRACE_SYNC | TRACE_REDIRECTION))
            trace_printf("%s@%d - stdin request from (%d,%d)"
                         ", type=%d, for supplier (%d, %d)\n",
                         method_name, __LINE__,
                         recv_msg->u.stdin_req.nid,
                         recv_msg->u.stdin_req.pid,
                         recv_msg->u.stdin_req.reqType,
                         recv_msg->u.stdin_req.supplier_nid,
                         recv_msg->u.stdin_req.supplier_pid);

        lnode = Nodes->GetLNode( recv_msg->u.stdin_req.nid );
        if ( lnode == NULL )
        {
            break;
        }
        process = lnode->GetProcessL(recv_msg->u.stdin_req.pid);

        if (process)
        {
            if (recv_msg->u.stdin_req.reqType == STDIN_REQ_DATA)
            {
                // Set up to forward stdin data to requester.
                // Save file descriptor associated with stdin
                // so can find the redirector object later.
                CProcess *supProcess;
                lnode = Nodes->GetLNode( recv_msg->u.stdin_req.supplier_nid );
                if ( lnode )
                {
                    supProcess = lnode->GetProcessL ( recv_msg->u.stdin_req.supplier_pid );
                    if (supProcess)
                    {
                        int fd;
                        fd = Redirector.stdinRemote(supProcess->infile(),
                                                    recv_msg->u.stdin_req.nid,
                                                    recv_msg->u.stdin_req.pid);
                        process->FdStdin(fd);
                    }
                    else
                    {
                        char buf[MON_STRING_BUF_SIZE];
                        snprintf(buf, sizeof(buf), "[%s], Can't find process "
                                 "nid=%d, pid=%d for stdin data request.\n",
                                 method_name,
                                 recv_msg->u.stdin_req.supplier_nid,
                                 recv_msg->u.stdin_req.supplier_pid);
                        mon_log_write(MON_CLUSTER_HANDLEOTHERNODE_8,
                                      SQ_LOG_ERR, buf);
                    }
                }
            }
            else if (recv_msg->u.stdin_req.reqType == STDIN_FLOW_OFF)
            {
                Redirector.stdinOff(process->FdStdin());
            }
            else if (recv_msg->u.stdin_req.reqType == STDIN_FLOW_ON)
            {
                Redirector.stdinOn(process->FdStdin());
            }
        }
        else
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf(buf, sizeof(buf), "[%s], Can't find process nid=%d, "
                     "pid=%d for stdin data request.\n", method_name,
                     recv_msg->u.stdin_req.nid,
                     recv_msg->u.stdin_req.pid);
            mon_log_write(MON_CLUSTER_HANDLEOTHERNODE_9, SQ_LOG_INFO, buf);
        }
        break;
#endif

#ifndef NAMESERVER_PROCESS
    case InternalType_Kill:
        // Queue the kill request for processing by a worker thread.
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal kill request for (%d, %d), abort =%d\n", method_name, __LINE__, recv_msg->u.kill.nid, recv_msg->u.kill.pid, recv_msg->u.kill.persistent_abort);

        ReqQueue.enqueueKillReq( &recv_msg->u.kill );
        break;
#endif

    case InternalType_Process:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal process request\n", method_name, __LINE__);
        if ( MyNode->IsMyNode(recv_msg->u.process.nid) )
        {   // Need to create process on this node.
            // Queue process creation request for handling by worker thread
#ifdef NAMESERVER_PROCESS
            ReqQueue.enqueueNewProcNsReq( &recv_msg->u.process );
#endif
#ifndef NAMESERVER_PROCESS
            ReqQueue.enqueueNewProcReq( &recv_msg->u.process );
#endif
        }
        break;

    case InternalType_ProcessInit:
        if ( MyNode->IsMyNode(recv_msg->u.processInit.origNid) )
        {  // New process request originated on this node
            ReqQueue.enqueueProcInitReq( &recv_msg->u.processInit );
        }
        break;

#ifndef NAMESERVER_PROCESS
    case InternalType_Open:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal open request for (%d, %d), opened (%d, %d)\n", method_name, __LINE__, recv_msg->u.open.nid, recv_msg->u.open.pid, recv_msg->u.open.opened_nid, recv_msg->u.open.opened_pid);

        ReqQueue.enqueueOpenReq( &recv_msg->u.open );
        break;
#endif

    case InternalType_SchedData:
#ifdef EXCHANGE_CPU_SCHEDULING_DATA
        SaveSchedData( recv_msg );
#endif
        break;

    case InternalType_Set:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal set request\n", method_name, __LINE__);
        ReqQueue.enqueueSetReq( &recv_msg->u.set );
        break;

    case InternalType_UniqStr:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal unique string request\n", method_name, __LINE__);
        ReqQueue.enqueueUniqStrReq( &recv_msg->u.uniqstr );
        break;

#ifndef NAMESERVER_PROCESS
    case InternalType_Sync:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_TMSYNC))
            trace_printf("%s@%d - Internal sync request for"
                         " Node %s, pnid=%d, SyncType=%d\n",
                         method_name, __LINE__, Node[pnid]->GetName(), pnid,
                         recv_msg->u.sync.type);
        switch (recv_msg->u.sync.type )
        {
        case SyncType_TmData:
            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                trace_printf("%s@%d - TMSYNC(TmData) on Node %s (pnid=%d), (phase=%d)\n", method_name, __LINE__, Node[pnid]->GetName(), pnid, MyNode->GetPhase());
            if ( ! MyNode->IsSpareNode() && MyNode->GetPhase() != Phase_Ready )
            {
                MyNode->CheckActivationPhase();
            }
            if ( ! MyNode->IsSpareNode() && MyNode->GetPhase() == Phase_Ready )
            {
                if ( MyNode->GetTmSyncState() == SyncState_Null )
                {
                    // Begin a Slave Sync Start
                    if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                        trace_printf("%s@%d - Slave Sync Start on Node %s (pnid=%d)\n", method_name, __LINE__, Node[pnid]->GetName(), pnid);
                    tmSyncPNid_ = pnid;
                    Node[pnid]->SetTmSyncState( recv_msg->u.sync.state );
                    if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                    {
                        trace_printf("%s@%d - Node %s (pnid=%d) TmSyncState updated (%d)(%s)\n", method_name, __LINE__, Node[pnid]->GetName(), pnid, Node[pnid]->GetTmSyncState(), SyncStateString( Node[pnid]->GetTmSyncState() ));
                    }
                    Monitor->CoordinateTmDataBlock( &recv_msg->u.sync );
                }
                else
                {
                    if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                        trace_printf("%s@%d - Sync State Collision! Node %s (pnid=%d) TmSyncState=(%d)(%s)\n", method_name, __LINE__, MyNode->GetName(), MyPNID, MyNode->GetTmSyncState(), SyncStateString( MyNode->GetTmSyncState()) );
                    if ( MyNode->GetTmSyncState() == SyncState_Continue )
                    {
                        if ( pnid > tmSyncPNid_ )
                            // highest node id will continue
                        {
                            // They take priority ... we abort
                            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                                trace_printf("%s@%d - Aborting Slave Sync Start on node %s (pnid=%d)\n", method_name, __LINE__, Node[Monitor->tmSyncPNid_]->GetName(), Monitor->tmSyncPNid_);
                            MyNode->SetTmSyncState( SyncState_Null );
                            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                                trace_printf("%s@%d - Node %s (pnid=%d) TmSyncState updated (%d)(%s)\n", method_name, __LINE__, MyNode->GetName(), MyPNID, MyNode->GetTmSyncState(), SyncStateString( MyNode->GetTmSyncState() ) );
                            Monitor->ReQueue_TmSync (false);
                            // Continue with other node's Slave TmSync Start request
                            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                                trace_printf("%s@%d - Slave Sync Start on node %s (pnid=%d)\n", method_name, __LINE__, Node[pnid]->GetName(), pnid);
                            tmSyncPNid_ = pnid;
                            Node[pnid]->SetTmSyncState( recv_msg->u.sync.state );
                            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                            {
                                trace_printf("%s@%d - Node %s (pnid=%d) TmSyncState updated (%d)(%s)\n", method_name, __LINE__, Node[pnid]->GetName(), pnid, Node[pnid]->GetTmSyncState(), SyncStateString( Node[pnid]->GetTmSyncState() ));
                            }
                            Monitor->CoordinateTmDataBlock (&recv_msg->u.sync);
                        }
                    }
                    else if ( MyNode->GetTmSyncState() == SyncState_Start )
                    {
                        // Check if they continue with Master Sync Start
                        if ( pnid > MyPNID )
                            // highest node id will continue
                        {
                            // They take priority ... we abort
                            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                                trace_printf("%s@%d - Aborted Master Sync Start\n", method_name, __LINE__);
                            MyNode->SetTmSyncState( SyncState_Null );
                            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                                trace_printf("%s@%d - Node %s (pnid=%d) TmSyncState updated (%d)(%s)\n", method_name, __LINE__, MyNode->GetName(), MyPNID, MyNode->GetTmSyncState(), SyncStateString( MyNode->GetTmSyncState() ) );
                            // Continue with other node's Slave TmSync Start request
                            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                                trace_printf("%s@%d - Slave Sync Start on node %s (pnid=%d)\n", method_name, __LINE__, Node[pnid]->GetName(), pnid);
                            tmSyncPNid_ = pnid;
                            Node[pnid]->SetTmSyncState( recv_msg->u.sync.state );
                            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                            {
                                trace_printf("%s@%d - Node %s (pnid=%d) TmSyncState updated (%d)(%s)\n", method_name, __LINE__, Node[pnid]->GetName(), pnid, Node[pnid]->GetTmSyncState(), SyncStateString( Node[pnid]->GetTmSyncState() ));
                            }
                            Monitor->CoordinateTmDataBlock (&recv_msg->u.sync);
                        }
                        else
                        {
                            // We continue and assume they abort
                            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                                trace_printf("%s@%d - Continuing with Master Sync Start\n", method_name, __LINE__);
                        }
                    }
                    else
                    {
                        if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                            trace_printf("%s@%d - Invalid TmSync_State\n", method_name, __LINE__);
                    }
                }
            }
            break;

        case SyncType_TmSyncState:
            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                trace_printf("%s@%d - TMSYNC(TmSyncState) on Node %s (pnid=%d)\n", method_name, __LINE__, Node[pnid]->GetName(), pnid);
            break;

        default:
            {
            char buf[MON_STRING_BUF_SIZE];
            snprintf(buf, sizeof(buf), "[%s], Unknown SyncType from pnid=%d.\n", method_name, pnid);
            mon_log_write(MON_CLUSTER_HANDLEOTHERNODE_10, SQ_LOG_ERR, buf);
            }
        }
        break;
#endif

    default:
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf(buf, sizeof(buf), "[%s], Unknown Internal message received, Physical Node=%d.\n", method_name, pnid);
            mon_log_write(MON_CLUSTER_HANDLEOTHERNODE_11, SQ_LOG_ERR, buf);
        }
    }

    TRACE_EXIT;
}

void CCluster::HandleMyNodeMsg (struct internal_msg_def *recv_msg,
                                int pnid)
{
    const char method_name[] = "CCluster::HandleMyNodeMsg";
    TRACE_ENTRY;

#ifndef NAMESERVER_PROCESS
    CProcess *process;
    CLNode  *lnode;
#endif

    if (trace_settings & TRACE_SYNC_DETAIL)
        trace_printf("%s@%d - Marking object as replicated, msg type=%d\n",
                     method_name, __LINE__, recv_msg->type);
    switch (recv_msg->type)
    {

    case InternalType_Null:
        if (trace_settings & TRACE_SYNC_DETAIL)
            trace_printf("%s@%d - Physical Node pnid=n%d has nothing to "
                         "update. \n", method_name, __LINE__, pnid);
        break;

    case InternalType_ActivateSpare:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal activate spare request, spare pnid=%d, down pnid=%d\n"
                        , method_name, __LINE__
                        , recv_msg->u.activate_spare.spare_pnid
                        , recv_msg->u.activate_spare.down_pnid);
        break;

    case InternalType_NameServerAdd:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf( "%s@%d - Internal NameServer add request for node_name=%s\n"
                        , method_name, __LINE__
                        , recv_msg->u.nameserver_add.node_name );

        // Queue the nameserver add request for processing by a worker thread.
        ReqQueue.enqueueNameServerAddReq( recv_msg->u.nameserver_add.req_nid
                                        , recv_msg->u.nameserver_add.req_pid
                                        , recv_msg->u.nameserver_add.req_verifier
                                        , recv_msg->u.nameserver_add.node_name );
        break;

    case InternalType_NameServerDelete:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf( "%s@%d - Internal NameServer delete request for node=%s\n"
                        , method_name, __LINE__, recv_msg->u.nameserver_delete.node_name);

        // Queue the nameserver delete request for processing by a worker thread.
        ReqQueue.enqueueNameServerDeleteReq( recv_msg->u.nameserver_delete.req_nid
                                           , recv_msg->u.nameserver_delete.req_pid
                                           , recv_msg->u.nameserver_delete.req_verifier
                                           , recv_msg->u.nameserver_delete.node_name );
        break;

    case InternalType_NodeAdd:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf( "%s@%d - Internal node add request for node_name=%s, "
                          "first_core=%d, last_core=%d, "
                          "processors=%d, roles=%d\n"
                        , method_name, __LINE__
                        , recv_msg->u.node_add.node_name
                        , recv_msg->u.node_add.first_core
                        , recv_msg->u.node_add.last_core
                        , recv_msg->u.node_add.processors
                        , recv_msg->u.node_add.roles );

        // Queue the node add request for processing by a worker thread.
        ReqQueue.enqueueNodeAddReq( recv_msg->u.node_add.req_nid
                                  , recv_msg->u.node_add.req_pid
                                  , recv_msg->u.node_add.req_verifier
                                  , recv_msg->u.node_add.node_name
                                  , recv_msg->u.node_add.first_core
                                  , recv_msg->u.node_add.last_core
                                  , recv_msg->u.node_add.processors
                                  , recv_msg->u.node_add.roles );
        break;

    case InternalType_Clone:
#ifndef NAMESERVER_PROCESS
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal clone request, completed replicating process (%d, %d) %s\n", method_name, __LINE__, recv_msg->u.clone.nid, recv_msg->u.clone.os_pid, (recv_msg->u.clone.backup?" Backup":""));
#else        
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal clone request, process (%d, %d)"
                         " %s\n", method_name, __LINE__,
                         recv_msg->u.clone.nid, recv_msg->u.clone.os_pid,
                         (recv_msg->u.clone.backup?" Backup":""));

        ReqQueue.enqueueCloneReq( &recv_msg->u.clone );
#endif
        break;

#ifndef NAMESERVER_PROCESS
    case InternalType_Device:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal device request, completed device processing for ldev %s\n", method_name, __LINE__, recv_msg->u.device.ldev_name);
        break;
#endif

    case InternalType_Shutdown:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal shutdown request for level=%d\n", method_name, __LINE__, recv_msg->u.shutdown.level);

        // Queue the shutdown request for processing by a worker thread.
        ReqQueue.enqueueShutdownReq( recv_msg->u.shutdown.level );
        break;

    case InternalType_NodeDelete:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf( "%s@%d - Internal node delete request for pnid=%d\n"
                        , method_name, __LINE__, recv_msg->u.node_delete.pnid);

        // Queue the node delete request for processing by a worker thread.
        ReqQueue.enqueueNodeDeleteReq( recv_msg->u.node_delete.req_nid
                                     , recv_msg->u.node_delete.req_pid
                                     , recv_msg->u.node_delete.req_verifier
                                     , recv_msg->u.node_delete.pnid );
        break;

    case InternalType_Down:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal down node request for pnid=%d\n", method_name, __LINE__, recv_msg->u.down.pnid);
        break;

    case InternalType_NodeName:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal node name request (%s to %s)\n", method_name, __LINE__, recv_msg->u.node_name.current_name, recv_msg->u.node_name.new_name);

        // Queue the node name request for processing by a worker thread.
        ReqQueue.enqueueNodeNameReq( recv_msg->u.node_name.req_nid
                                   , recv_msg->u.node_name.req_pid
                                   , recv_msg->u.node_name.req_verifier
                                   , recv_msg->u.node_name.current_name
                                   , recv_msg->u.node_name.new_name );
        break;

    case InternalType_SoftNodeDown:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal soft down node request for pnid=%d\n", method_name, __LINE__, recv_msg->u.down.pnid);
        break;

    case InternalType_SoftNodeUp:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal soft up node request for pnid=%d\n", method_name, __LINE__, recv_msg->u.up.pnid);
        break;

    case InternalType_Up:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal up node request for pnid=%d\n", method_name, __LINE__, recv_msg->u.up.pnid);
        break;

#ifndef NAMESERVER_PROCESS
    case InternalType_Dump:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal dump request for nid=%d, pid=%d\n",
                         method_name, __LINE__,
                         recv_msg->u.dump.nid, recv_msg->u.dump.pid);

        lnode = Nodes->GetLNode( recv_msg->u.dump.nid );
        if ( lnode )
        {
            process = lnode->GetProcessL(recv_msg->u.dump.pid);

            if (process)
            {
                int verifier = recv_msg->u.dump.verifier;
                if ( (verifier == -1) || (verifier == process->GetVerifier()) )
                {
                    process->DumpBegin(recv_msg->u.dump.dumper_nid,
                                       recv_msg->u.dump.dumper_pid,
                                       recv_msg->u.dump.dumper_verifier,
                                       recv_msg->u.dump.core_file);
                }
                else
                {
                    char buf[MON_STRING_BUF_SIZE];
                    snprintf(buf, sizeof(buf), "[%s], Can't find process nid=%d, "
                             "pid=%d, verifier=%d for dump target.\n", method_name,
                             recv_msg->u.dump.nid, recv_msg->u.dump.pid,
                             recv_msg->u.dump.verifier);
                    mon_log_write(MON_CLUSTER_HANDLEMYNODE_1, SQ_LOG_ERR, buf);
                }
            }
            else
            {
                char buf[MON_STRING_BUF_SIZE];
                snprintf(buf, sizeof(buf), "[%s], Can't find process nid=%d, "
                         "pid=%d for dump target.\n", method_name,
                         recv_msg->u.dump.nid, recv_msg->u.dump.pid);
                mon_log_write(MON_CLUSTER_HANDLEMYNODE_2, SQ_LOG_ERR, buf);
            }
        }
        break;

    case InternalType_DumpComplete:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal dump-complete request for nid=%d, pid=%d\n",
                         method_name, __LINE__,
                         recv_msg->u.dump.nid, recv_msg->u.dump.pid);
        lnode = Nodes->GetLNode( recv_msg->u.dump.nid );
        if ( lnode )
        {
            process = lnode->GetProcessL(recv_msg->u.dump.pid);

            if (process)
            {
                int verifier = recv_msg->u.dump.verifier;
                if ( (verifier == -1) || (verifier == process->GetVerifier()) )
                {
                    process->DumpEnd(recv_msg->u.dump.status, recv_msg->u.dump.core_file);
                }
                else
                {
                    char buf[MON_STRING_BUF_SIZE];
                    snprintf(buf, sizeof(buf), "[%s], Can't find process nid=%d, "
                             "pid=%d, verifier=%d for dump target.\n", method_name,
                             recv_msg->u.dump.nid, recv_msg->u.dump.pid,
                             recv_msg->u.dump.verifier);
                    mon_log_write(MON_CLUSTER_HANDLEMYNODE_3, SQ_LOG_ERR, buf);
                }
            }
            else
            {
                // Dump completion handled in CProcess::Exit()
                char buf[MON_STRING_BUF_SIZE];
                snprintf(buf, sizeof(buf), "[%s], Can't find process nid=%d, "
                         "pid=%d for dump complete target.\n", method_name,
                         recv_msg->u.dump.nid, recv_msg->u.dump.pid);
                mon_log_write(MON_CLUSTER_HANDLEMYNODE_4, SQ_LOG_ERR, buf);
            }
        }
        break;
#endif

    case InternalType_Exit:
        // Final process exit logic is done in Process_Exit, not here
        // as in the past.
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal exit request for %s (%d, %d)\n", method_name, __LINE__, recv_msg->u.exit_ns.name, recv_msg->u.exit_ns.nid, recv_msg->u.exit_ns.pid);
#ifdef NAMESERVER_PROCESS
        ReqQueue.enqueueExitNsReq( &recv_msg->u.exit_ns );
#endif
        break;

#ifndef NAMESERVER_PROCESS
    case InternalType_Event:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal event request\n", method_name, __LINE__);
        break;
#endif

#ifndef NAMESERVER_PROCESS
    case InternalType_IoData:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal IO data request\n", method_name, __LINE__);
        break;
#endif

#ifndef NAMESERVER_PROCESS
    case InternalType_StdinReq:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal STDIN request\n", method_name, __LINE__);
        break;
#endif

#ifndef NAMESERVER_PROCESS
    case InternalType_Kill:
        // Queue the kill request for processing by a worker thread.
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal kill request for (%d, %d), abort =%d\n", method_name, __LINE__, recv_msg->u.kill.nid, recv_msg->u.kill.pid, recv_msg->u.kill.persistent_abort);

        ReqQueue.enqueueKillReq( &recv_msg->u.kill );
        break;
#endif

    case InternalType_Process:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal process request, completed process replication for (%d, %d) %s\n", method_name,  __LINE__, recv_msg->u.process.pid, recv_msg->u.process.nid, (recv_msg->u.process.backup?" Backup":""));
        break;

    case InternalType_ProcessInit:
        // No action needed
        break;

#ifndef NAMESERVER_PROCESS
    case InternalType_Open:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal open request, completed open replication, "
                         "(%d, %d:%d)  opened (%d, %d:%d)\n",
                         method_name, __LINE__,
                         recv_msg->u.open.nid,
                         recv_msg->u.open.pid,
                         recv_msg->u.open.verifier,
                         recv_msg->u.open.opened_nid,
                         recv_msg->u.open.opened_pid,
                         recv_msg->u.open.opened_verifier);
        break;
#endif

    case InternalType_SchedData:
        // No action needed
        break;

    case InternalType_Set:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal set request, completed replicating key %s::%s\n", method_name, __LINE__, recv_msg->u.set.group, recv_msg->u.set.key);
        break;

    case InternalType_UniqStr:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS))
            trace_printf("%s@%d - Internal unique string request, completed replicating (%d, %d)\n", method_name, __LINE__, recv_msg->u.uniqstr.nid, recv_msg->u.uniqstr.id);
        break;

#ifndef NAMESERVER_PROCESS
    case InternalType_Sync:
        if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_TMSYNC))
            trace_printf("%s@%d - Internal sync request for node %s, pnid=%d, SyncType=%d\n"
                         , method_name, __LINE__, Node[pnid]->GetName(), pnid, recv_msg->u.sync.type);
        switch (recv_msg->u.sync.type )
        {
        case SyncType_TmData:
            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                trace_printf("%s@%d    - TMSYNC(TmData) on Node %s (pnid=%d)\n", method_name, __LINE__, Node[MyPNID]->GetName(), MyPNID);
            tmSyncPNid_ = MyPNID;
            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                trace_printf("%s@%d    - Sync communicated, tmSyncPNid_=%d\n", method_name, __LINE__, tmSyncPNid_);
            if ( ! MyNode->IsSpareNode() && MyNode->GetPhase() != Phase_Ready )
            {
                MyNode->CheckActivationPhase();
            }
            if ( MyNode->GetTmSyncState() == SyncState_Start &&
                 MyNode->GetPhase() == Phase_Ready &&
                 MyNode->GetLNodesCount() > 1 )
            {
                // Begin a Slave Sync Start to other
                // logical nodes in my physical node
                if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                    trace_printf("%s@%d - Slave Sync Start on local node %s, pnid=%d\n", method_name, __LINE__, Node[pnid]->GetName(), pnid);
                Monitor->CoordinateTmDataBlock( &recv_msg->u.sync );
            }
            break;

        case SyncType_TmSyncState:
            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                trace_printf("%s@%d    - TMSYNC(TmSyncState) on Node %s (pnid=%d)\n", method_name, __LINE__, Node[MyPNID]->GetName(), MyPNID);
            break;

        default:
            {
                char buf[MON_STRING_BUF_SIZE];
                snprintf(buf, sizeof(buf), "[%s], Unknown SyncType from node %s, pnid=%d during processing local SyncType.\n", method_name, Node[pnid]->GetName(), pnid);
                mon_log_write(MON_CLUSTER_HANDLEMYNODE_5, SQ_LOG_ERR, buf);
            }
        }
        break;
#endif

    default:
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf(buf, sizeof(buf), "[%s], Unknown Internal message received during processing local SyncType for pnid=%d.\n", method_name, pnid);
            mon_log_write(MON_CLUSTER_HANDLEMYNODE_6, SQ_LOG_ERR, buf);
        }

    }

    TRACE_EXIT;
}


bool CCluster::responsive()
{
    const char method_name[] = "CCluster::responsive";
    TRACE_ENTRY;

    int barrierDiff = barrierCount_ - barrierCountSaved_;

    // if no difference in barrier count, sync thread is not responsive
    if  ( !barrierDiff && isMonInitComplete() )
    {
        // this proc is called every SYNC_MAX_RESPONSIVE+1 secs
        cumulativeDelaySec_ += CCluster::SYNC_MAX_RESPONSIVE + 1;

        monSyncResponsive_ = false; // sync thread is no longer responsive

        if ( CommType == CommType_InfiniBand )
        {
            // if sync thread is stuck in mpi call, one of the following checks will be true
            if ( inBarrier_ || inAllGather_ || inCommDup_ )
            {
                mem_log_write(MON_CLUSTER_RESPONSIVE_1, cumulativeDelaySec_,
                              ( ( (inBarrier_ << 1) | inAllGather_ ) << 1 ) | inCommDup_);
            }
            else // non-mpi took quite long
            {
                mem_log_write(MON_CLUSTER_RESPONSIVE_2, cumulativeDelaySec_);
            }
        }
        else
        {
            // if sync thread is stuck in mpi call
            if ( inBarrier_ )
            {
                mem_log_write(MON_CLUSTER_RESPONSIVE_1, cumulativeDelaySec_,
                              inBarrier_);
            }
            else // non-mpi took quite long
            {
                mem_log_write(MON_CLUSTER_RESPONSIVE_2, cumulativeDelaySec_);
            }
        }
    }
    else if (barrierDiff < syncMinPerSec_)
    {
        mem_log_write(MON_CLUSTER_RESPONSIVE_3, barrierDiff, syncMinPerSec_);
        cumulativeDelaySec_ = 0;
        monSyncResponsive_ = true; // slow but responsive
    }
    else
    {
        cumulativeDelaySec_ = 0;
        monSyncResponsive_ = true; // truely responsive
    }

    barrierCountSaved_ = barrierCount_;
    if ( CommType == CommType_InfiniBand )
    {
        allGatherCountSaved_ = allGatherCount_;
        commDupCountSaved_ = commDupCount_;
    }

    TRACE_EXIT;

    return monSyncResponsive_;
}


int CCluster::MPIAllgather(void *sendbuf, int sendcount, MPI_Datatype sendtype,
                 void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm Comm)
{
    const char method_name[] = "CCluster::MPIAllGather";
    TRACE_ENTRY;

    inAllGather_ = true;

    int rc = MPI_Allgather (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, Comm);

    inAllGather_ = false;

    allGatherCount_++;

    TRACE_EXIT;
    return rc;
}

bool CCluster::ReinitializeConfigCluster( bool nodeAdded, int pnid )
{
    const char method_name[] = "CCluster::ReinitializeConfigCluster";
    TRACE_ENTRY;

    int     rs = true;
    CNode  *pnode;

    // Update node membership in the cluster

    if (trace_settings & (TRACE_INIT | TRACE_REQUEST))
        trace_printf( "%s@%d - Configured physical nodes count=%d\n"
                    , method_name, __LINE__
                    , GetConfigPNodesCount() );

    if (nodeAdded)
    {
        // Add node to monitor's view of the cluster
        pnode = Nodes->AddNode( pnid );
        if ( !pnode )
        {
            rs = false;
        }
    }
    else
    {
        // Delete node from monitor's view of the cluster
        if ( !Nodes->DeleteNode( pnid ) )
        {
            rs = false;
        }
    }

    if ( rs )
    {
        CClusterConfig *clusterConfig = Nodes->GetClusterConfig();
        configPNodesCount_ = clusterConfig->GetPNodesCount();
        Nodes->UpdateCluster();
    }

    if (trace_settings & (TRACE_INIT | TRACE_REQUEST))
        trace_printf( "%s@%d - Configured physical nodes count=%d\n"
                    , method_name, __LINE__
                    , GetConfigPNodesCount() );

    TRACE_EXIT;
    return( rs );
}

void CCluster::InitializeConfigCluster( void )
{
#ifndef NAMESERVER_PROCESS // nameserver is running in agent mode
    int rc;
#endif

    const char method_name[] = "CCluster::InitializeConfigCluster";
    TRACE_ENTRY;

    int worldSize = 0;
    MPI_Comm_size (MPI_COMM_WORLD, &worldSize);
#ifdef NAMESERVER_PROCESS
    if ( !IsRealCluster )
    {
        char *nodes = getenv( "SQ_VIRTUAL_NODES" );
        worldSize = atoi(nodes);
        if ( worldSize <= 0 )
        {
            worldSize = 1;
        }
    }
#endif
    CClusterConfig *clusterConfig = Nodes->GetClusterConfig();
    configPNodesCount_ = clusterConfig->GetPNodesCount();
    int rankToPnid[configPNodesCount_];

#ifdef NAMESERVER_PROCESS
    currentNodes_ = 1;  // non-master Name Servers join set through master Name Server
#else
    if (IAmIntegrating || IsAgentMode)
    {
        currentNodes_ = 1;  // non-master monitors join cluster through master monitor
    }
    else
    {
        currentNodes_ = worldSize;
    }
#endif

    if ( !IsRealCluster )
    {
        // Set virtual cluster size to collective size
        MPI_Comm_size (MPI_COMM_WORLD, &configPNodesCount_);
#ifdef NAMESERVER_PROCESS
        configPNodesCount_ = worldSize;
#endif

        // For virtual cluster set physical node id equal to rank
        for (int i=0; i<worldSize; ++i)
        {
            rankToPnid[i] = i;

            // Set bit indicating node is up
            upNodes_.upNodes[i/MAX_NODE_BITMASK] |= (1ull << (i%MAX_NODE_BITMASK));
        }
    }
    else
    {
        for (int i=0; i<configPNodesCount_; ++i)
        {
            rankToPnid[i] = i;
            rankToPnid[i] = rankToPnid[i];  // make compiler happy
        }
    }

    // Build the monitor's configured view of the cluster
    if ( IsRealCluster )
    {   // Map node name to physical node id
        // (for virtual nodes physical node equals "rank" (previously set))
        if (MyPNID == -1)
        {
            MyPNID = clusterConfig->GetPNid( Node_name );
            if (MyPNID == -1)
            {
                char buf[MON_STRING_BUF_SIZE];
                snprintf(buf, sizeof(buf), "[%s@%d] Can't find node name=%s in cluster configuration\n",
                         method_name, __LINE__, Node_name );
                mon_log_write(MON_CLUSTER_INITCONFIGCLUSTER_1, SQ_LOG_CRIT, buf);

                MPI_Abort(MPI_COMM_SELF,99);
            }
        }
    }

    Nodes->AddNodes( );
    MyNode = Nodes->GetNode(MyPNID);
    Nodes->SetupCluster( &Node, &LNode, &indexToPnid_ );

    if ( CommType == CommType_Sockets )
    {
        InitServerSock();
    }

    if (trace_settings & TRACE_INIT)
    {
        trace_printf( "%s@%d (MasterMonitor) IAmIntegrating=%d,"
                      " IsAgentMode=%d, IsMaster=%d,"
                      " MasterMonitorName=%s, Node_name=%s\n"
                    , method_name, __LINE__
                    , IAmIntegrating
                    , IsAgentMode, IsMaster, MasterMonitorName, Node_name );
    }

    if (IAmIntegrating || IsAgentMode)
    {
#ifndef NAMESERVER_PROCESS
        int TmLeaderPNid = -1;
        if (IsMaster)
        {
            tmLeaderNid_ = Nodes->GetFirstNid();
            TmLeaderPNid = LNode[tmLeaderNid_]->GetNode()->GetPNid();
        }
#endif
        // Monitors processes in AGENT mode in a real cluster initialize all
        // remote nodes to a down state. The master monitor and the joining
        // monitors will set the joining node state to up as part of the node
        // re-integration processing as monitor processes join the cluster
        // through the master.
        for (int i=0; i < clusterConfig->GetPNodesCount(); i++)
        {
            if (Node[indexToPnid_[i]])
            {
                if (Node[indexToPnid_[i]]->GetPNid() == MyPNID)
                { // Set bit indicating node is up
                    upNodes_.upNodes[indexToPnid_[i]/MAX_NODE_BITMASK] |=
                        (1ull << (indexToPnid_[i]%MAX_NODE_BITMASK));
                }
                else
                { // Set node state to down
                    Node[indexToPnid_[i]]->SetState( State_Down );
#ifndef NAMESERVER_PROCESS
                    if (IsMaster)
                    {
                        if (TmLeaderPNid == indexToPnid_[i])
                        {
                            AssignTmLeader(indexToPnid_[i], false);
                        }
                    }
#endif
                }
            }
        }
    }
#ifndef NAMESERVER_PROCESS // nameserver is running in agent mode
    else
    {
        char *nodeNames = 0;
        if ( IsRealCluster )
        {
            if (trace_settings & TRACE_INIT)
                trace_printf( "%s@%d Collecting port numbers and node names, "
                              "configPNodesCount_=%d, worldSize=%d, pnid=%d (%s:%s)\n"
                              "MyCommPort=%s\nMySyncPort=%s\n"
                             , method_name, __LINE__
                             , GetConfigPNodesCount(), worldSize
                             , MyPNID, MyNode->GetName(), MyNode->GetCommPort()
                             , MyCommPort, MySyncPort );

            bool nodeStatus[GetConfigPNodesCount()];
            for (int i=0; i<GetConfigPNodesCount(); ++i)
            {
                nodeStatus[i] = false;

                if (trace_settings & (TRACE_INIT | TRACE_REQUEST))
                    trace_printf( "%s@%d - nodeStatus[%d]=%d\n"
                                , method_name, __LINE__, i, nodeStatus[i] ) ;
            }

            // Collect comm port info from other monitors
            char *commPortNums = new char[worldSize * MPI_MAX_PORT_NAME];
            rc = MPI_Allgather (MyCommPort, MPI_MAX_PORT_NAME, MPI_CHAR, commPortNums,
                                MPI_MAX_PORT_NAME, MPI_CHAR, MPI_COMM_WORLD);
            if (rc != MPI_SUCCESS)
            {
                char buf[MON_STRING_BUF_SIZE];
                snprintf(buf, sizeof(buf), "[%s@%d] MPI_Allgather error=%s\n",
                         method_name, __LINE__, ErrorMsg(rc));
                mon_log_write(MON_CLUSTER_INITCONFIGCLUSTER_2, SQ_LOG_CRIT, buf);

                MPI_Abort(MPI_COMM_SELF,99);
            }

            // Collect sync port info from other monitors
            char *syncPortNums = new char[worldSize * MPI_MAX_PORT_NAME];
            rc = MPI_Allgather (MySyncPort, MPI_MAX_PORT_NAME, MPI_CHAR, syncPortNums,
                                MPI_MAX_PORT_NAME, MPI_CHAR, MPI_COMM_WORLD);
            if (rc != MPI_SUCCESS)
            {
                char buf[MON_STRING_BUF_SIZE];
                snprintf(buf, sizeof(buf), "[%s@%d] MPI_Allgather error=%s\n",
                         method_name, __LINE__, ErrorMsg(rc));
                mon_log_write(MON_CLUSTER_INITCONFIGCLUSTER_2, SQ_LOG_CRIT, buf);

                MPI_Abort(MPI_COMM_SELF,99);
            }

            // Exchange Node Names with collective
            nodeNames = new char[worldSize * MPI_MAX_PROCESSOR_NAME];
            rc = MPI_Allgather (Node_name, MPI_MAX_PROCESSOR_NAME, MPI_CHAR,
                                nodeNames, MPI_MAX_PROCESSOR_NAME, MPI_CHAR,
                                MPI_COMM_WORLD);
            if (rc != MPI_SUCCESS)
            {
                char buf[MON_STRING_BUF_SIZE];
                snprintf(buf, sizeof(buf), "[%s@%d] MPI_Allgather error=%s\n",
                         method_name, __LINE__, ErrorMsg(rc));
                mon_log_write(MON_CLUSTER_INITCONFIGCLUSTER_3, SQ_LOG_CRIT, buf);

                MPI_Abort(MPI_COMM_SELF,99);
            }

            // For each node name received get corresponding CNode object and
            // store port number in it.
            char * nodeName;
            CNode * node;
            for (int i = 0; i < worldSize; i++)
            {
                nodeName = &nodeNames[ i * MPI_MAX_PROCESSOR_NAME ];
                node = Nodes->GetNode( nodeName );
                if ( node )
                {
                    node->SetCommPort( &commPortNums[ i * MPI_MAX_PORT_NAME] );
                    node->SetSyncPort( &syncPortNums[ i * MPI_MAX_PORT_NAME] );
                    rankToPnid[i] = node->GetPNid();
                    nodeStatus[rankToPnid[i]] = true;

                    if (trace_settings & TRACE_INIT)
                    {
                        trace_printf( "%s@%d rankToPnid[%d]=%d (%s:%s:%s)"
                                      "(node=%s,commPort=%s,syncPort=%s)\n"
                                    , method_name, __LINE__, i, rankToPnid[i]
                                    , node->GetName()
                                    , node->GetCommPort()
                                    , node->GetSyncPort()
                                    , &nodeNames[ i * MPI_MAX_PROCESSOR_NAME]
                                    , &commPortNums[ i * MPI_MAX_PORT_NAME]
                                    , &syncPortNums[ i * MPI_MAX_PORT_NAME]);
                    }
                }
                else
                {
                    rankToPnid[i] = -1;

                    // Unexpectedly could not map node name to CNode object
                    char buf[MON_STRING_BUF_SIZE];
                    snprintf(buf, sizeof(buf), "[%s@%d] Unable to find node "
                             "object for node %s\n", method_name, __LINE__,
                             nodeName );
                    mon_log_write(MON_CLUSTER_INITCONFIGCLUSTER_4, SQ_LOG_CRIT, buf);
                }
            }
            delete [] commPortNums;
            delete [] syncPortNums;

            tmLeaderNid_ = Nodes->GetFirstNid();
            int TmLeaderPNid = LNode[tmLeaderNid_]->GetNode()->GetPNid();

            // Any nodes not in the initial MPI_COMM_WORLD are down.
            for (int i=0; i<GetConfigPNodesCount(); ++i)
            {
                if ( nodeStatus[indexToPnid_[i]] == false )
                {
                    if (trace_settings & (TRACE_INIT | TRACE_REQUEST))
                        trace_printf( "%s@%d - nodeStatus[%d]=%d"
                                      ", indexToPnid_[%d]=%d\n"
                                    , method_name, __LINE__
                                    , i, nodeStatus[i]
                                    , i, indexToPnid_[i] ) ;

                    node = Nodes->GetNode(indexToPnid_[i]);
                    if ( node ) node->SetState( State_Down );
                    // assign new TmLeader if TMLeader node is dead.
                    if (TmLeaderPNid == indexToPnid_[i])
                    {
                        AssignTmLeader(indexToPnid_[i], false);
                    }
                }
                else
                {   // Set bit indicating node is up

                    if (trace_settings & (TRACE_INIT | TRACE_REQUEST))
                        trace_printf( "%s@%d - nodeStatus[%d]=%d"
                                      ", indexToPnid_[%d]=%d\n"
                                    , method_name, __LINE__
                                    , i, nodeStatus[i]
                                    , i, indexToPnid_[i] ) ;

                    upNodes_.upNodes[indexToPnid_[i]/MAX_NODE_BITMASK] |=
                        (1ull << (indexToPnid_[i]%MAX_NODE_BITMASK));
                }
            }
        }
        else
        {
            tmLeaderNid_ = 0;
        }

        // Initialize communicators for point-to-point communications
        int myRank;
        MPI_Comm_rank( MPI_COMM_WORLD, &myRank );
        if ( !IsRealCluster )
            myRank = MyPNID;

        InitClusterComm(worldSize, myRank, rankToPnid);
        if ( CommType == CommType_Sockets )
        {
            InitClusterSocks(worldSize, myRank, nodeNames, rankToPnid);
            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                for ( int i =0; i < worldSize; i++ )
                {
                    trace_printf( "%s@%d socks_[%d]=%d\n"
                                , method_name, __LINE__
                                , rankToPnid[i], socks_[rankToPnid[i]]);
                }
            }
        }

        if (nodeNames) delete [] nodeNames;
    }
#endif

    if ( CommType == CommType_Sockets )
    {
        // Allgather() cluster sockets are established as remote
        // monitor processes join the cluster
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
        {
            for ( int i =0; i < clusterConfig->GetPNodesCount() ; i++ )
            {
                trace_printf( "%s@%d %s (%d), state=%s, socks_[%d]=%d\n"
                            , method_name, __LINE__
                            , Node[indexToPnid_[i]]->GetName()
                            , Node[indexToPnid_[i]]->GetPNid()
                            , StateString(Node[indexToPnid_[i]]->GetState())
                            , indexToPnid_[i], socks_[indexToPnid_[i]]);
            }
        }
    }
    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        for ( int i =0; i < MAX_NODE_MASKS ; i++ )
        {
            trace_printf( "%s@%d upNodes set[%d]: %llx\n"
                        , method_name, __LINE__
                        , i, upNodes_.upNodes[i]);
        }
    }

#ifndef NAMESERVER_PROCESS
    // Kill the MPICH hydra_pmi_proxy to prevent it from killing all
    // processes in cluster when mpirun or monitor processes are killed
    if (!IsAgentMode  || (IsAgentMode && IsMPIChild))
    {
        kill( getppid(), SIGKILL );
    }
#endif

    TRACE_EXIT;
}

void CCluster::InitClusterComm(int worldSize, int myRank, int * rankToPnid)
{
    const char method_name[] = "CCluster::InitClusterComm";
    TRACE_ENTRY;

    // Compute an array of "colors" for use with  MPI_Comm_split.
    int *splitColors;
    splitColors = new int[worldSize*worldSize*2];
    int *splitOtherNode;
    splitOtherNode = new int[worldSize*worldSize*2];
    int splitRows = 0;
    for ( int i=0; i<(worldSize*worldSize*2); ++i)
    {
        splitColors[i] = MPI_UNDEFINED;
        splitOtherNode[i] = -1;
    }

    int color = 1;
    bool placed;
    for (int i = 0; i < worldSize; i++)
    {
        for (int j = i+1; j < worldSize; j++)
        {
            // Find a free slot for rank "i" to rank "j"

            placed = false;
            for (int k=0; k<splitRows; ++k)
            {
                if (    splitColors[k*worldSize+i] == MPI_UNDEFINED
                     && splitColors[k*worldSize+j] == MPI_UNDEFINED )
                {
                    splitColors[k*worldSize+i] = color;
                    splitColors[k*worldSize+j] = color;
                    placed = true;

                    if (myRank == i)
                        splitOtherNode[k] = j;
                    else if (myRank == j)
                        splitOtherNode[k] = i;
                    break;
                }
            }
            if (!placed)
            {   // Need to use a new row
                splitColors[splitRows*worldSize+i] = color;
                splitColors[splitRows*worldSize+j] = color;

                if (myRank == i)
                    splitOtherNode[splitRows] = j;
                else if (myRank == j)
                    splitOtherNode[splitRows] = i;

                ++splitRows;
            }

            ++color;
        }
    }

    if (trace_settings & TRACE_INIT)
    {
        trace_printf("%s@%d Created %d splitRows for worldSize=%d, myRank=%d\n",
                     method_name, __LINE__, splitRows, worldSize, myRank);
        string line;
        char fragment[50];
        for (int i=0; i<splitRows; ++i)
        {
            sprintf(fragment, "%s@%d splitColors[%d]=", method_name, __LINE__,
                    i);
            line = fragment;
            for (int j=0; j<worldSize; ++j)
            {
                sprintf(fragment, " %d,", splitColors[i*worldSize+j]);
                line += fragment;
            }
            line += "\n";
            trace_printf(line.c_str());

            trace_printf("%s@%d splitOtherNode[%d]=%d\n", method_name,
                         __LINE__, i, splitOtherNode[i]);
        }
    }

    // Create one communicator for each other rank in MPI_COMM_WORLD
    // This permits point-to-point communication with each rank.
    int myRankInComm;
    MPI_Comm ncomm;
    int nid;

    for (int nSplit=0; nSplit < splitRows; ++nSplit)
    {
        color = splitColors[nSplit*worldSize+myRank];
        MPI_Comm_split(MPI_COMM_WORLD, color, myRank, &ncomm);
        if (ncomm == MPI_COMM_NULL)
        {
            if (splitColors[nSplit*worldSize+myRank] != MPI_UNDEFINED)
            {
                if (trace_settings & TRACE_INIT)
                {
                    trace_printf("%s@%d Rank %d: Unexpected MPI_COMM_NULL from "
                                 "MPI_Comm_split, nSplit=%d\n",
                                 method_name, __LINE__,myRank, nSplit);
                }
            }
        }
        else
        {
            // Set comms_ (communicators) array element for the
            // physical node.
            nid = rankToPnid[splitOtherNode[nSplit]];
            comms_[nid] = ncomm;

            MPI_Comm_rank(ncomm, &myRankInComm);
            otherMonRank_[nid] = (myRankInComm == 0)? 1: 0;

            if (trace_settings & TRACE_INIT)
            {
                trace_printf("%s@%d Rank %d: MPI_Comm_split %d, color=%d, "
                             "comms_[%d] is orig rank #%d, "
                             "otherMonRank_=%d\n",
                             method_name, __LINE__,
                             myRank, nSplit, color,
                             nid, splitOtherNode[nSplit],
                             otherMonRank_[nid]);
            }
        }
    }

    delete [] splitColors;
    delete [] splitOtherNode;

    TRACE_EXIT;
}

void CCluster::HandleReintegrateError( int rc, int err,
                                       int pnid, nodeId_t *nodeInfo,
                                       bool abortIn )
{
    const char method_name[] = "CCluster::HandleReintegrateError";
    TRACE_ENTRY;

    char buf[MON_STRING_BUF_SIZE];

    switch ( err )
    {
    case Reintegrate_Err1:
        snprintf(buf, sizeof(buf), "[%s], can't to connect to creator monitor"
                 " port: %s - Error: %s.\n",
                 method_name, IntegratingMonitorPort, ErrorMsg(rc));
        break;

    case Reintegrate_Err2:
        snprintf(buf, sizeof(buf), "[%s], can't merge intercomm to existing "
                 "MPI collective - Error: %s.\n",
                 method_name, ErrorMsg(rc));

        break;

    case Reintegrate_Err3:
        snprintf(buf, sizeof(buf), "[%s], unable to obtain cluster info "
                 "from creator monitor: %s.\n", method_name, ErrorMsg(rc));
        break;

    case Reintegrate_Err4:
        snprintf(buf, sizeof(buf), "[%s], Failed to send name/port "
                 "to node %d (%s): %s.\n", method_name, pnid,
                 nodeInfo->nodeName, ErrorMsg(rc));
        break;

    case Reintegrate_Err5:
        snprintf(buf, sizeof(buf), "[%s], can't to connect to "
                 " node %d monitor, commPort=%s, syncPort=%s: %s.\n",
                 method_name, pnid, nodeInfo->commPort,
                 nodeInfo->syncPort, ErrorMsg(rc));
        break;

    case Reintegrate_Err6:
        snprintf(buf, sizeof(buf), "[%s], can't merge intercomm "
                 "for node %d: %s.\n", method_name, pnid,
                 ErrorMsg(rc));
        break;

    case Reintegrate_Err7:
        snprintf(buf, sizeof(buf), "[%s], can't disconnect "
                 "intercomm for node %d: %s.\n", method_name, pnid,
                 ErrorMsg(rc));
        break;

    case Reintegrate_Err8:
        snprintf(buf, sizeof(buf), "[%s], Failed to send status to creator "
                 "monitor: %s\n", method_name, ErrorMsg(rc));
        break;

    case Reintegrate_Err9:
        snprintf(buf, sizeof(buf), "[%s], Failed to send name/port "
                 "to creator monitor: %s.\n", method_name, ErrorMsg(rc));
        break;

    case Reintegrate_Err10:
        snprintf(buf, sizeof(buf), "[%s], Monitor initialization failed (could"
                 " not write to port file).  Aborting.\n", method_name);
        break;

    case Reintegrate_Err11:
        snprintf(buf, sizeof(buf), "[%s], Monitor initialization failed (could"
                 " not open port file).  Aborting.\n", method_name);
        break;

    case Reintegrate_Err12:
        snprintf(buf, sizeof(buf), "[%s], Monitor initialization failed (could"
                 " not initialize local io).  Aborting.\n", method_name);
        break;

    case Reintegrate_Err13:
        snprintf(buf, sizeof(buf), "[%s], Monitor initialization failed (could"
                 " not initialize devices).  Aborting.\n", method_name);
        break;

    case Reintegrate_Err14:
        snprintf(buf, sizeof(buf), "[%s] Aborting.\n", method_name);
        break;

    case Reintegrate_Err15:
        snprintf(buf, sizeof(buf), "[%s], no connect acknowledgement "
                 "for node %d: %s.\n", method_name, pnid,
                 ErrorMsg(rc));
        break;

    default:
        snprintf(buf, sizeof(buf), "[%s], Reintegration error: %s\n",
                 method_name, ErrorMsg(rc));
    }

    mon_log_write(MON_CLUSTER_REINTEGRATE_1, SQ_LOG_ERR, buf);

    if ( abortIn )
        MPI_Abort(MPI_COMM_SELF,99);

    TRACE_EXIT;
}

void CCluster::SendReIntegrateStatus( STATE nodeState, int initErr )
{
    const char method_name[] = "CCluster::SendReIntegrateStatus";
    int rc;
    nodeStatus_t nodeStatus;
    nodeStatus.state = nodeState;
    nodeStatus.status = initErr;

    switch( CommType )
    {
        case CommType_InfiniBand:
            rc = Monitor->SendMPI( (char *) &nodeStatus
                                 , sizeof(nodeStatus_t)
                                 , 0
                                 , MON_XCHNG_DATA
                                 , joinComm_ );
            if ( rc )
            {
                HandleReintegrateError( rc, Reintegrate_Err8, -1, NULL, true );
            }
            break;
        case CommType_Sockets:
            rc = Monitor->SendSock( (char *) &nodeStatus
                                  , sizeof(nodeStatus_t)
                                  , joinSock_
                                  , method_name );
            if ( rc )
            {
                HandleReintegrateError( rc, Reintegrate_Err8, -1, NULL, true );
            }
            break;
        default:
            // Programmer bonehead!
            abort();
    }

    if ( nodeState != State_Up )
    {  // Initialization error, abort.

        mem_log_write(CMonLog::MON_REINTEGRATE_9, MyPNID, initErr);
        HandleReintegrateError( rc, initErr, -1, NULL, true );
    }
}

bool CCluster::PingSockPeer(CNode *node)
{
    const char method_name[] = "CCluster::PingSockPeer";
    TRACE_ENTRY;

    static int sv_connect_wait_timeout = -2;
    static int sv_connect_retry_count = 1;
    if ( sv_connect_wait_timeout == -2 )
    {
        // Use the EPOLL timeout and retry values
        char *lv_connect_wait_timeout_env = getenv( "SQ_MON_EPOLL_WAIT_TIMEOUT" );
        if ( lv_connect_wait_timeout_env )
        {
            // Timeout in seconds
            sv_connect_wait_timeout = atoi( lv_connect_wait_timeout_env );
            char *lv_connect_retry_count_env = getenv( "SQ_MON_EPOLL_RETRY_COUNT" );
            if ( lv_connect_retry_count_env )
            {
                sv_connect_retry_count = atoi( lv_connect_retry_count_env );
            }
            if ( sv_connect_retry_count > 180 )
            {
                sv_connect_retry_count = 180;
            }
        }
        else
        {
            // default to 64 seconds
            sv_connect_wait_timeout = 16;
            sv_connect_retry_count = 4;
        }

        char buf[MON_STRING_BUF_SIZE];
        snprintf( buf, sizeof(buf)
                , "[%s@%d] Ping connect timeout wait_timeout=1 second, retry_count=%d\n"
                , method_name
                ,  __LINE__
                , (sv_connect_retry_count * sv_connect_wait_timeout) );

        mon_log_write( MON_PINGSOCKPEER_3, SQ_LOG_INFO, buf );
    }

    bool rs = true;
    int  rc;
    int  pingSock = -1;

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        trace_printf( "%s@%d - Pinging remote monitor %s, pnid=%d\n"
                    , method_name, __LINE__
                    , node->GetName(), node->GetPNid() );
    }

    // Attempt to connect with remote monitor in one seconds increments
    // to recover as quickly as possible or give up trying
    for (int i = 0; i < (sv_connect_retry_count*sv_connect_wait_timeout); i++ )
    {
        // Disable internal retries
        pingSock = Monitor->Connect( node->GetCommPort(), false );
        if ( pingSock < 0 )
        {
            if (node->GetState() != State_Up)
            {
                if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                {
                    trace_printf( "%s@%d - Node %s (%d) is not up, "
                                  "socks_[%d]=%d\n"
                                , method_name, __LINE__
                                , node->GetName(), node->GetPNid()
                                , node->GetPNid(), socks_[node->GetPNid()] );
                }
                break;
            }
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf)
                    , "[%s@%d] Retrying connect to remote monitor %s, pnid=%d, retry=%d\n"
                    , method_name
                    ,  __LINE__
                    , node->GetName(), node->GetPNid(), i );
            mon_log_write( MON_PINGSOCKPEER_4, SQ_LOG_INFO, buf );
            sleep( 1 );
        }
        else
        {
            break;
        }
    }
    if ( pingSock < 0 )
    {
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
        {
            trace_printf( "%s@%d - Can't connect to remote monitor %s, pnid=%d\n"
                        , method_name, __LINE__
                        , node->GetName(), node->GetPNid() );
        }
        return(false);
    }

    nodeId_t nodeInfo;

    nodeInfo.pnid = MyPNID;
    strcpy(nodeInfo.nodeName, MyNode->GetName());
    strcpy(nodeInfo.commPort, MyNode->GetCommPort());
    strcpy(nodeInfo.syncPort, MyNode->GetSyncPort());
    nodeInfo.ping = true;
    nodeInfo.creatorPNid = -1;
    nodeInfo.creator = false;
    nodeInfo.creatorShellPid = -1;
    nodeInfo.creatorShellVerifier = -1;

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        trace_printf( "Sending my nodeInfo.pnid=%d\n"
                      "        nodeInfo.nodeName=%s\n"
                      "        nodeInfo.commPort=%s\n"
                      "        nodeInfo.syncPort=%s\n"
                      "        nodeInfo.creatorPNid=%d\n"
                      "        nodeInfo.creator=%d\n"
                      "        nodeInfo.creatorShellPid=%d\n"
                      "        nodeInfo.creatorShellVerifier=%d\n"
                      "        nodeInfo.ping=%d\n"
                    , nodeInfo.pnid
                    , nodeInfo.nodeName
                    , nodeInfo.commPort
                    , nodeInfo.syncPort
                    , nodeInfo.creatorPNid
                    , nodeInfo.creator
                    , nodeInfo.creatorShellPid
                    , nodeInfo.creatorShellVerifier
                    , nodeInfo.ping );
    }

    rc = Monitor->SendSock( (char *) &nodeInfo
                          , sizeof(nodeId_t)
                          , pingSock
                          , method_name );

    if ( rc )
    {
        rs = false;
        char buf[MON_STRING_BUF_SIZE];
        snprintf( buf, sizeof(buf)
                , "[%s], Cannot send ping node info to node %s: (%s)\n"
                , method_name, node->GetName(), ErrorMsg(rc));
        mon_log_write(MON_PINGSOCKPEER_1, SQ_LOG_ERR, buf);
    }
    else
    {
        // Get info about connecting monitor
        rc = Monitor->ReceiveSock( (char *) &nodeInfo
                                 , sizeof(nodeId_t)
                                 , pingSock
                                 , method_name );
        if ( rc )
        {   // Handle error
            rs = false;
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf)
                    , "[%s], Cannot receive ping node info from node %s: (%s)\n"
                    , method_name, node->GetName(), ErrorMsg(rc));
            mon_log_write(MON_PINGSOCKPEER_2, SQ_LOG_ERR, buf);
        }
        else
        {
            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                trace_printf( "Received from nodeInfo.pnid=%d\n"
                              "        nodeInfo.nodeName=%s\n"
                              "        nodeInfo.commPort=%s\n"
                              "        nodeInfo.syncPort=%s\n"
                              "        nodeInfo.ping=%d\n"
                            , nodeInfo.pnid
                            , nodeInfo.nodeName
                            , nodeInfo.commPort
                            , nodeInfo.syncPort
                            , nodeInfo.ping );
            }
        }
    }

    close( pingSock );

    TRACE_EXIT;
    return( rs );
}

void CCluster::ReIntegrate( int initProblem )
{
    const char method_name[] = "CCluster::ReIntegrate";
    TRACE_ENTRY;

    switch( CommType )
    {
        case CommType_InfiniBand:
            ReIntegrateMPI( initProblem );
            break;
        case CommType_Sockets:
            ReIntegrateSock( initProblem );
            break;
        default:
            // Programmer bonehead!
            abort();
    }

    TRACE_EXIT;
}

void CCluster::ReIntegrateMPI( int initProblem )
{
    const char method_name[] = "CCluster::ReIntegrateMPI";
    TRACE_ENTRY;

    int rc;
    bool haveCreatorComm = false;
    MPI_Comm interComm;
    MPI_Comm intraComm = MPI_COMM_NULL;
    MPI_Comm intraCommCreatorMon = MPI_COMM_NULL;

    nodeId_t myNodeInfo;
    strcpy(myNodeInfo.nodeName, MyNode->GetName());
    strcpy(myNodeInfo.commPort, MyNode->GetCommPort());
    // Set bit indicating my node is up
    upNodes_.upNodes[MyPNID/MAX_NODE_BITMASK] |= (1ull << (MyPNID%MAX_NODE_BITMASK));

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
       trace_printf("%s@%d - Connect to creator monitor (port %s)\n",
                    method_name, __LINE__, IntegratingMonitorPort);

    mem_log_write(CMonLog::MON_REINTEGRATE_1, MyPNID);

    if (trace_settings & (TRACE_SYNC | TRACE_RECOVERY | TRACE_INIT))
    {
        for ( int i =0; i < MAX_NODE_MASKS ; i++ )
        {
            trace_printf( "%s@%d Integrating node %s (pnid=%d) "
                          "sees set[%d]: %llx\n"
                        , method_name, __LINE__
                        , MyNode->GetName(), MyPNID
                        , i, upNodes_.upNodes[i] );
        }
    }

    TEST_POINT( TP010_NODE_UP );
    // Connect with my creator monitor
    rc = MPI_Comm_connect( IntegratingMonitorPort,
                           MPI_INFO_NULL, 0, MPI_COMM_SELF, &joinComm_ );
    if ( rc )
    {
        HandleReintegrateError( rc, Reintegrate_Err1, -1, NULL, true );
    }

    MPI_Comm_set_errhandler( joinComm_, MPI_ERRORS_RETURN );

    mem_log_write(CMonLog::MON_REINTEGRATE_4, MyPNID);

    TEST_POINT( TP011_NODE_UP );

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        trace_printf("%s@%d Connected to creator monitor, sending id\n",
                     method_name, __LINE__);
    }

    // Send this node's name and port number so creator monitor
    // knows who we are, and set flag to let creator monitor it is the CREATOR.
    myNodeInfo.creator = true;
    myNodeInfo.creatorShellPid = CreatorShellPid;
    myNodeInfo.creatorShellVerifier = CreatorShellVerifier;
    if ((rc = Monitor->SendMPI((char *) &myNodeInfo, sizeof(nodeId_t), 0,
                            MON_XCHNG_DATA, joinComm_)))
        HandleReintegrateError( rc, Reintegrate_Err9, -1, NULL,
                                true );

    TEST_POINT( TP012_NODE_UP );

    // Merge the inter-communicators obtained from the connect/accept
    // between this new monitor and the creator monitor.
    if ((rc = MPI_Intercomm_merge( joinComm_, 1, &intraCommCreatorMon )))
        HandleReintegrateError( rc, Reintegrate_Err2, -1, NULL, true );

    MPI_Comm_set_errhandler( intraCommCreatorMon, MPI_ERRORS_RETURN );

    nodeId_t *nodeInfo = new nodeId_t[GetConfigPNodesCount()];

    mem_log_write(CMonLog::MON_REINTEGRATE_3, MyPNID);

    // Obtain node names & port numbers of existing monitors from
    // the creator monitor.
    if ((rc = Monitor->ReceiveMPI((char *)nodeInfo, sizeof(nodeId_t)*GetConfigPNodesCount(),
                               MPI_ANY_SOURCE, MON_XCHNG_DATA, joinComm_)))
        HandleReintegrateError( rc, Reintegrate_Err3, -1, NULL, true );

    if ( initProblem )
    {
        // The monitor encountered an initialization error.  Inform
        // the creator monitor that the node is down.  Then abort.
        SendReIntegrateStatus( State_Down, initProblem );
    }

    // Connect to each of the other existing monitors and let them know
    // we are the NEW monitor and reset the creator flag so they know they are
    // not the creator monitor.
    myNodeInfo.creator = false;
    myNodeInfo.creatorShellPid = -1;
    myNodeInfo.creatorShellVerifier = -1;
    for (int i = 0; i < GetConfigPNodesCount(); i++)
    {
        if (strcmp(nodeInfo[i].commPort, IntegratingMonitorPort) == 0)
        {   // Already connected to creator monitor
            comms_[i] = intraCommCreatorMon;
            otherMonRank_[i] = 0;
            ++currentNodes_;

            // Set bit indicating node is up
            upNodes_.upNodes[i/MAX_NODE_BITMASK] |= (1ull << (i%MAX_NODE_BITMASK));

            Node[i]->SetCommPort( IntegratingMonitorPort );
            Node[i]->SetState( State_Up );
            haveCreatorComm = true;
        }
        else if (nodeInfo[i].nodeName[0] != 0
                 && nodeInfo[i].commPort[0] != 0)
        {
            if ( haveCreatorComm && i >= GetConfigPNodesCount()/2)
                // Reintegration failure after connecting to half
                // of existing monitors.
                TEST_POINT( TP016_NODE_UP );

            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                trace_printf("%s@%d Attempting connection to node %d (%s), "
                             "port %s\n", method_name, __LINE__, i,
                             nodeInfo[i].nodeName, nodeInfo[i].commPort);
            }

            mem_log_write(CMonLog::MON_REINTEGRATE_5, MyPNID, i);

            TEST_POINT( TP013_NODE_UP );

            // Connect to existing monitor
            if ((rc = MPI_Comm_connect( nodeInfo[i].commPort,
                                        MPI_INFO_NULL, 0, MPI_COMM_SELF,
                                        &interComm )))
            {
                HandleReintegrateError( rc, Reintegrate_Err5, i, &nodeInfo[i],
                                        false );
                SendReIntegrateStatus( State_Down, Reintegrate_Err14 );
            }

            MPI_Comm_set_errhandler( interComm, MPI_ERRORS_RETURN );

            TEST_POINT( TP014_NODE_UP );

            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                trace_printf("%s@%d Connected to node %d (%s), sending id\n",
                             method_name, __LINE__,i,nodeInfo[i].nodeName);
            }

            // Send this nodes name and port number so other monitor
            // knows who we are.
            if ((rc = Monitor->SendMPI((char *) &myNodeInfo, sizeof(nodeId_t), 0,
                                    MON_XCHNG_DATA, interComm)))
            {
                HandleReintegrateError( rc, Reintegrate_Err4, i, &nodeInfo[i],
                                        false );
                SendReIntegrateStatus( State_Down, Reintegrate_Err14 );
            }

            if ((rc = MPI_Intercomm_merge(interComm, 1, &intraComm)))
            {
                HandleReintegrateError( rc, Reintegrate_Err6, i, NULL, false );
                SendReIntegrateStatus( State_Down, Reintegrate_Err14 );
            }

            // Get acknowledgement that other monitor is ready to
            // integrate this node.  This is an interlock to avoid a
            // race condition where the creator monitor could signal
            // the monitors in the cluster to integrate the new node
            // before one or more was ready to do the integration.
            int readyFlag;
            if ((rc = Monitor->ReceiveMPI((char *) &readyFlag, sizeof(readyFlag),
                                       MPI_ANY_SOURCE, MON_XCHNG_DATA,
                                       interComm)))
            {
                HandleReintegrateError( rc, Reintegrate_Err15, i, NULL,
                                        false );
                SendReIntegrateStatus( State_Down, Reintegrate_Err14 );
            }


            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                trace_printf( "%s@%d - Received ready-flag from node %d (%s)\n",
                              method_name, __LINE__, i,
                             nodeInfo[i].nodeName);
            }

            if ((rc = MPI_Comm_disconnect(&interComm)))
                HandleReintegrateError( rc, Reintegrate_Err7, i, NULL, false );

            MPI_Comm_set_errhandler(intraComm, MPI_ERRORS_RETURN);

            comms_[i] = intraComm;
            otherMonRank_[i] = 0;
            ++currentNodes_;
            Node[i]->SetSyncPort( nodeInfo[i].syncPort );
            Node[i]->SetState( State_Up );

            // Set bit indicating node is up
            upNodes_.upNodes[i/MAX_NODE_BITMASK] |= (1ull << (i%MAX_NODE_BITMASK));

            mem_log_write(CMonLog::MON_REINTEGRATE_6, MyPNID, i);
        }
        else if ( i != MyPNID)
        {
            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                trace_printf("%s@%d Connection to node %d not attempted, "
                             "no port information.  nodeInfo[%d].port=%s, "
                             "IntegratingMonitorPort=%s\n", method_name,
                             __LINE__, i, i, nodeInfo[i].commPort,
                             IntegratingMonitorPort);
            }
        }
    }

    if (trace_settings & (TRACE_SYNC | TRACE_RECOVERY | TRACE_INIT))
    {
        for ( int i =0; i < MAX_NODE_MASKS ; i++ )
        {
            trace_printf( "%s@%d Integrating node %s (pnid=%d) "
                          "sees set[%d]: %llx\n"
                        , method_name, __LINE__
                        , MyNode->GetName(), MyPNID
                        , i, upNodes_.upNodes[i] );
        }
    }

    mem_log_write(CMonLog::MON_REINTEGRATE_7, MyPNID);

    TEST_POINT( TP015_NODE_UP );

    // Inform creator monitor that connections are complete and
    // this monitor is ready to participate in "allgather"
    // communications with the other monitors.
    SendReIntegrateStatus( State_Up, 0 );

    mem_log_write(CMonLog::MON_REINTEGRATE_8, MyPNID);

    MyNode->SetState( State_Merged );

    delete[] nodeInfo;

    TRACE_EXIT;
}

void CCluster::ReIntegrateSock( int initProblem )
{
    const char method_name[] = "CCluster::ReIntegrateSock";
    TRACE_ENTRY;

    bool haveCreatorSocket = false;
    int rc;
    int existingCommFd;
    int existingSyncFd;
    char commPort[MPI_MAX_PORT_NAME];
    char syncPort[MPI_MAX_PORT_NAME];
    char *pch1;
    char *pch2;

    // Set bit indicating my node is up
    upNodes_.upNodes[MyPNID/MAX_NODE_BITMASK] |= (1ull << (MyPNID%MAX_NODE_BITMASK));

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
       trace_printf("%s@%d - Connect to creator monitor (port %s)\n",
                    method_name, __LINE__, IntegratingMonitorPort);

    mem_log_write(CMonLog::MON_REINTEGRATE_1, MyPNID);

    if (trace_settings & (TRACE_SYNC | TRACE_RECOVERY | TRACE_INIT))
    {
        for ( int i =0; i < MAX_NODE_MASKS ; i++ )
        {
            trace_printf( "%s@%d Integrating node %s (pnid=%d) "
                          "sees set[%d]: %llx\n"
                        , method_name, __LINE__
                        , MyNode->GetName(), MyPNID
                        , i, upNodes_.upNodes[i] );
        }
    }

    TEST_POINT( TP010_NODE_UP );

    // Connect with my creator monitor
    bool lv_done = false;
    bool lv_did_not_connect_in_first_attempt = false;
    while ( ! lv_done )
    {
        joinSock_ = Monitor->Connect( IntegratingMonitorPort );
        if ( joinSock_ < 0 )
        {
            if ( IsAgentMode )
            {
                lv_did_not_connect_in_first_attempt = true;
                sleep( 15 );
            }
            else
            {
                HandleReintegrateError( joinSock_, Reintegrate_Err1, -1, NULL, true );
            }
        }
        else
        {
            if ( lv_did_not_connect_in_first_attempt )
            {
                sleep( 10 );
            }
            lv_done = true;
        }
    }

    mem_log_write(CMonLog::MON_REINTEGRATE_4, MyPNID);

    TEST_POINT( TP011_NODE_UP );

    // Send this node's name and port number so creator monitor
    // knows who we are, and set flag to let creator monitor it is the CREATOR.
    nodeId_t myNodeInfo;
    strcpy(myNodeInfo.nodeName, MyNode->GetName());
    strcpy(myNodeInfo.commPort, MyNode->GetCommPort());
    strcpy(myNodeInfo.syncPort, MyNode->GetSyncPort());
    myNodeInfo.pnid = MyNode->GetPNid();
    myNodeInfo.creatorPNid = -1;
    myNodeInfo.creator = true;
    myNodeInfo.creatorShellPid = CreatorShellPid;
    myNodeInfo.creatorShellVerifier = CreatorShellVerifier;
    myNodeInfo.ping = false;

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        trace_printf( "%s@%d - Connected to creator monitor, sending my info:\n"
                      "        myNodeInfo.pnid=%d (%s)\n"
                      "        myNodeInfo.commPort=%s\n"
                      "        myNodeInfo.syncPort=%s\n"
                      "        myNodeInfo.creator=%d\n"
                      "        myNodeInfo.creatorShellPid=%d:%d\n"
                      "        myNodeInfo.ping=%d\n"
                    , method_name, __LINE__
                    , myNodeInfo.pnid
                    , myNodeInfo.nodeName
                    , myNodeInfo.commPort
                    , myNodeInfo.syncPort
                    , myNodeInfo.creator
                    , myNodeInfo.creatorShellPid
                    , myNodeInfo.creatorShellVerifier
                    , myNodeInfo.ping );
    }

    rc = Monitor->SendSock( (char *) &myNodeInfo
                          , sizeof(nodeId_t)
                          , joinSock_
                          , method_name );
    if ( rc )
    {
        HandleReintegrateError( rc, Reintegrate_Err9, -1, NULL, true );
    }

    TEST_POINT( TP012_NODE_UP );

    mem_log_write(CMonLog::MON_REINTEGRATE_3, MyPNID);

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        trace_printf("%s@%d Getting all node info from creator monitor\n",
                     method_name, __LINE__);
    }

    // Obtain node names & port numbers of existing monitors from
    // the creator monitor.
    int pnodeCount = Nodes->GetPNodesCount();
    nodeId_t *nodeInfo;
    size_t nodeInfoSize = (sizeof(nodeId_t) * pnodeCount);
    nodeInfo = (nodeId_t *) new char[nodeInfoSize];
    rc = Monitor->ReceiveSock( (char *)nodeInfo
                             , nodeInfoSize
                             , joinSock_
                             , method_name );
    if ( rc )
    {
        HandleReintegrateError( rc, Reintegrate_Err3, -1, NULL, true );
    }

    if ( initProblem )
    {
        // The monitor encountered an initialization error.  Inform
        // the creator monitor that the node is down.  Then abort.
        SendReIntegrateStatus( State_Down, initProblem );
    }

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        trace_printf( "%s@%d - Received port info from creator monitor\n"
                    , method_name, __LINE__);
        for (int i=0; i<pnodeCount; i++)
        {
            trace_printf( "Port info for pnid=%d\n"
                          "        nodeInfo[%d].nodeName=%s\n"
                          "        nodeInfo[%d].commPort=%s\n"
                          "        nodeInfo[%d].syncPort=%s\n"
                          "        nodeInfo[%d].creatorPNid=%d\n"
                        , nodeInfo[i].pnid
                        , i, nodeInfo[i].nodeName
                        , i, nodeInfo[i].commPort
                        , i, nodeInfo[i].syncPort
                        , i, nodeInfo[i].creatorPNid );
        }
    }
    // Connect to each of the other existing monitors and let them know
    // we are the NEW monitor and reset the creator flag so they know they are
    // not the creator monitor.
    myNodeInfo.creator = false;
    myNodeInfo.creatorShellPid = -1;
    myNodeInfo.creatorShellVerifier = -1;
    myNodeInfo.ping = false;
    for (int i=0; i<pnodeCount; i++)
    {
        if ( nodeInfo[i].creatorPNid != -1 &&
             nodeInfo[i].creatorPNid == nodeInfo[i].pnid )
        {
            // Get acknowledgement that creator monitor is ready to
            // integrate this node.
            int creatorpnid = -1;
            rc = Monitor->ReceiveSock( (char *) &creatorpnid
                                     , sizeof(creatorpnid)
                                     , joinSock_
                                     , method_name );
            if ( rc || creatorpnid != nodeInfo[i].creatorPNid )
            {
                HandleReintegrateError( rc, Reintegrate_Err15, i, NULL,
                                        false );
                SendReIntegrateStatus( State_Down, Reintegrate_Err14 );
            }

            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                trace_printf( "%s@%d - Received ready indication from creator "
                              "node %d nodeInfo[%d].nodeName=%s\n"
                            , method_name, __LINE__
                            , creatorpnid, i , nodeInfo[i].nodeName);
            }

            otherMonRank_[nodeInfo[i].pnid] = 0;
            ++currentNodes_;

            // Store port numbers for the node
            strncpy(commPort, nodeInfo[i].commPort, MPI_MAX_PORT_NAME);
            strncpy(syncPort, nodeInfo[i].syncPort, MPI_MAX_PORT_NAME);

            Node[nodeInfo[i].pnid]->SetCommPort( commPort );
            pch1 = strtok (commPort,":");
            pch1 = strtok (NULL,":");
            Node[nodeInfo[i].pnid]->SetCommSocketPort( atoi(pch1) );
            Node[nodeInfo[i].pnid]->SetSyncPort( syncPort );
            pch2 = strtok (syncPort,":");
            pch2 = strtok (NULL,":");
            Node[nodeInfo[i].pnid]->SetSyncSocketPort( atoi(pch2) );
            sockPorts_[nodeInfo[i].pnid] = Node[nodeInfo[i].pnid]->GetSyncSocketPort();

            Node[nodeInfo[i].pnid]->SetState( State_Up );

            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                trace_printf( "%s@%d - Setting node %d (%s), commPort=%s(%d), syncPort=%s(%d)\n"
                            , method_name, __LINE__
                            , Node[nodeInfo[i].pnid]->GetPNid()
                            , Node[nodeInfo[i].pnid]->GetName()
                            , pch1, atoi(pch1)
                            , pch2, atoi(pch2) );
            }

            // Tell creator we are ready to accept its connection
            int mypnid = MyPNID;
            rc = Monitor->SendSock( (char *) &mypnid
                                  , sizeof(mypnid)
                                  , joinSock_
                                  , method_name );
            if ( rc )
            {
                HandleReintegrateError( rc, Reintegrate_Err4, i, &nodeInfo[i],
                                        false );
                SendReIntegrateStatus( State_Down, Reintegrate_Err14 );
            }

            // Connect to creator monitor
            existingSyncFd = AcceptSyncSock();
            if ( existingSyncFd < 0 )
            {
                HandleReintegrateError( rc, Reintegrate_Err5, i, &nodeInfo[i],
                                        false );
                SendReIntegrateStatus( State_Down, Reintegrate_Err14 );
            }
            socks_[nodeInfo[i].pnid] = existingSyncFd; // ReIntegrateSock
            // Set bit indicating node is up
            upNodes_.upNodes[nodeInfo[i].pnid/MAX_NODE_BITMASK] |=
                (1ull << (nodeInfo[i].pnid%MAX_NODE_BITMASK));

            if (trace_settings & (TRACE_RECOVERY | TRACE_INIT))
            {
                trace_printf( "%s@%d Connected to creator node %d (%s)\n"
                            , method_name, __LINE__
                            , nodeInfo[i].creatorPNid
                            , nodeInfo[i].nodeName );
                trace_printf( "%s@%d socks_[%d]=%d\n"
                            , method_name, __LINE__
                            , nodeInfo[i].pnid, socks_[nodeInfo[i].pnid]);
                for ( int i =0; i < MAX_NODE_MASKS ; i++ )
                {
                    trace_printf( "%s@%d Integrating node %s (pnid=%d) "
                                  "sees set[%d]: %llx\n"
                                , method_name, __LINE__
                                , MyNode->GetName(), MyPNID
                                , i, upNodes_.upNodes[i] );
                }
            }

            haveCreatorSocket = true;
        }
        else if ( nodeInfo[i].nodeName[0] != 0 && nodeInfo[i].commPort[0]  != 0 )
        {
            if ( haveCreatorSocket && i >= pnodeCount/2)
                // Reintegration failure after connecting to half
                // of existing monitors.
                TEST_POINT( TP016_NODE_UP );

            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                trace_printf("%s@%d Attempting connection to node %d (%s), "
                             "port %s\n", method_name, __LINE__, nodeInfo[i].pnid,
                             nodeInfo[i].nodeName, nodeInfo[i].commPort);
            }

            mem_log_write(CMonLog::MON_REINTEGRATE_5, MyPNID, i);

            TEST_POINT( TP013_NODE_UP );

            // Connect to existing monitor
            existingCommFd = Monitor->Connect( nodeInfo[i].commPort );
            if ( existingCommFd < 0 )
            {
                HandleReintegrateError( rc, Reintegrate_Err5, i, &nodeInfo[i],
                                        false );
                SendReIntegrateStatus( State_Down, Reintegrate_Err14 );
            }

            TEST_POINT( TP014_NODE_UP );

            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                trace_printf("%s@%d Connected to node %d (%s), sending my node name\n",
                             method_name, __LINE__,i,nodeInfo[i].nodeName);
            }

            // Send this nodes name and port number so other monitor
            // knows who we are.
            rc = Monitor->SendSock( (char *) &myNodeInfo
                                  , sizeof(nodeId_t)
                                  , existingCommFd
                                  , method_name );
            if ( rc )
            {
                HandleReintegrateError( rc, Reintegrate_Err4, i, &nodeInfo[i],
                                        false );
                SendReIntegrateStatus( State_Down, Reintegrate_Err14 );
            }

            // Get acknowledgement that other monitor is ready to
            // integrate this node.  This is an interlock to avoid a
            // race condition where the creator monitor could signal
            // the monitors in the cluster to integrate the new node
            // before one or more was ready to do the integration.
            int remotepnid = -1;
            rc = Monitor->ReceiveSock( (char *) &remotepnid
                                     , sizeof(remotepnid)
                                     , existingCommFd
                                     , method_name );
            if ( rc || remotepnid != nodeInfo[i].pnid )
            {
                HandleReintegrateError( rc, Reintegrate_Err15, i, NULL,
                                        false );
                SendReIntegrateStatus( State_Down, Reintegrate_Err14 );
            }

            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                trace_printf( "%s@%d - Received ready indication from "
                              "node %d nodeInfo[%d].nodeName=%s\n"
                            , method_name, __LINE__
                            , remotepnid, i , nodeInfo[i].nodeName);
            }

            otherMonRank_[nodeInfo[i].pnid] = 0;
            ++currentNodes_;

            // Store port numbers for the node
            strncpy(commPort, nodeInfo[i].commPort, MPI_MAX_PORT_NAME);
            strncpy(syncPort, nodeInfo[i].syncPort, MPI_MAX_PORT_NAME);

            Node[nodeInfo[i].pnid]->SetCommPort( commPort );
            pch1 = strtok (commPort,":");
            pch1 = strtok (NULL,":");
            Node[nodeInfo[i].pnid]->SetCommSocketPort( atoi(pch1) );

            Node[nodeInfo[i].pnid]->SetSyncPort( syncPort );
            pch2 = strtok (syncPort,":");
            pch2 = strtok (NULL,":");
            Node[nodeInfo[i].pnid]->SetSyncSocketPort( atoi(pch2) );
            sockPorts_[nodeInfo[i].pnid] = Node[nodeInfo[i].pnid]->GetSyncSocketPort();

            Node[nodeInfo[i].pnid]->SetState( State_Up );

            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                trace_printf( "%s@%d - Setting node %d (%s), commPort=%s(%d), syncPort=%s(%d)\n"
                            , method_name, __LINE__
                            , Node[nodeInfo[i].pnid]->GetPNid()
                            , Node[nodeInfo[i].pnid]->GetName()
                            , pch1, atoi(pch1)
                            , pch2, atoi(pch2) );
            }

            // Connect to existing monitor
            existingSyncFd = AcceptSyncSock();
            if ( existingSyncFd < 0 )
            {
                HandleReintegrateError( rc, Reintegrate_Err5, i, &nodeInfo[i],
                                        false );
                SendReIntegrateStatus( State_Down, Reintegrate_Err14 );
            }
            socks_[nodeInfo[i].pnid] = existingSyncFd; // ReIntegrateSock

            // Set bit indicating node is up
            upNodes_.upNodes[nodeInfo[i].pnid/MAX_NODE_BITMASK] |=
            (1ull << (nodeInfo[i].pnid%MAX_NODE_BITMASK));

            if (trace_settings & (TRACE_RECOVERY | TRACE_INIT))
            {
                trace_printf( "%s@%d socks_[%d]=%d\n"
                            , method_name, __LINE__
                            , nodeInfo[i].pnid, socks_[nodeInfo[i].pnid]);
                for ( int i =0; i < MAX_NODE_MASKS ; i++ )
                {
                    trace_printf( "%s@%d Integrating node %s (pnid=%d) "
                                  "sees set[%d]: %llx\n"
                                , method_name, __LINE__
                                , MyNode->GetName(), MyPNID
                                , i, upNodes_.upNodes[i] );
                }
            }

            mem_log_write(CMonLog::MON_REINTEGRATE_6, MyPNID, i);
        }
        else if ( nodeInfo[i].pnid != MyPNID)
        {
            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                trace_printf( "%s@%d Connection to node %d not attempted, "
                              "since it's unavailable port information.\n"
                              "nodeInfo[%d].commPort=%s\n"
                              "nodeInfo[%d].syncPort=%s\n"
                              "IntegratingMonitorPort=%s\n"
                            , method_name, __LINE__
                            , nodeInfo[i].pnid
                            , i, nodeInfo[i].commPort
                            , i, nodeInfo[i].syncPort
                            , IntegratingMonitorPort);
            }
        }
    }

    if (trace_settings & (TRACE_RECOVERY | TRACE_INIT))
    {
        for (int i=0; i<pnodeCount; i++)
        {
            if (nodeInfo[i].pnid == -1) continue;
            if (Node[nodeInfo[i].pnid] == NULL) continue;
            trace_printf( "%s@%d - Node info for pnid=%d (%s)\n"
                          "        Node[%d] commPort=%s\n"
                          "        Node[%d] syncPort=%s\n"
                          "        Node[%d] creatorPNid=%d\n"
                        , method_name, __LINE__
                        , Node[nodeInfo[i].pnid]->GetPNid()
                        , Node[nodeInfo[i].pnid]->GetName()
                        , nodeInfo[i].pnid, Node[nodeInfo[i].pnid]->GetCommPort()
                        , nodeInfo[i].pnid, Node[nodeInfo[i].pnid]->GetSyncPort()
                        , nodeInfo[i].pnid, nodeInfo[i].creatorPNid);
        }
        for ( int i =0; i < pnodeCount; i++ )
        {
            if (nodeInfo[i].pnid == -1) continue;
            trace_printf( "%s@%d socks_[%d]=%d, sockPorts_[%d]=%d\n"
                        , method_name, __LINE__
                        , nodeInfo[i].pnid, socks_[nodeInfo[i].pnid]
                        , nodeInfo[i].pnid, sockPorts_[nodeInfo[i].pnid]);
        }
        for ( int i =0; i < MAX_NODE_MASKS ; i++ )
        {
            trace_printf( "%s@%d Integrating node %s (pnid=%d) "
                          "sees set[%d]: %llx\n"
                        , method_name, __LINE__
                        , MyNode->GetName(), MyPNID
                        , i, upNodes_.upNodes[i] );
        }
    }

    mem_log_write(CMonLog::MON_REINTEGRATE_7, MyPNID);

    TEST_POINT( TP015_NODE_UP );

    // Inform creator monitor that connections are complete and
    // this monitor is ready to participate in "allgather"
    // communications with the other monitors.
    SendReIntegrateStatus( State_Up, 0 );

    mem_log_write(CMonLog::MON_REINTEGRATE_8, MyPNID);

    MyNode->SetState( State_Merged );

    delete[] nodeInfo;

    TRACE_EXIT;
}

void CCluster::ResetIntegratingPNid( void )
{
    const char method_name[] = "CCluster::ResetIntegratingPNid";
    TRACE_ENTRY;

    switch( CommType )
    {
        case CommType_InfiniBand:
            if ( joinComm_ != MPI_COMM_NULL )
            {
                MPI_Comm_free( &joinComm_ );
                joinComm_ = MPI_COMM_NULL;
            }
            break;
        case CommType_Sockets:
            if ( joinSock_ != -1 )
            {
                close(joinSock_);
                joinSock_ = -1;
            }
            break;
        default:
            // Programmer bonehead!
            abort();
    }

    if ( MyNode->IsCreator() )
    {
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
        {
            trace_printf( "%s@%d - Resetting creator pnid=%d\n",
                          method_name, __LINE__, MyPNID );
        }

        MyNode->SetCreator( false, -1, -1 );
    }

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        trace_printf( "%s@%d - Resetting integratingPNid_=%d\n",
                      method_name, __LINE__, integratingPNid_ );
    }

    integratingPNid_ = -1;

#ifdef NAMESERVER_PROCESS
    if (!CommAcceptMon.isAccepting())
    {
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
        {
            trace_printf( "%s@%d - Triggering commAcceptorMon thread to begin accepting connections\n",
                          method_name, __LINE__ );
        }

        // Indicate to the commAcceptor thread to begin accepting connections
        CommAcceptMon.startAccepting();
    }
#endif

    if (!CommAccept.isAccepting())
    {
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
        {
            trace_printf( "%s@%d - Triggering commAcceptor thread to begin accepting connections\n",
                          method_name, __LINE__ );
        }

        // Indicate to the commAcceptor thread to begin accepting connections
        CommAccept.startAccepting();
    }

    TRACE_EXIT;
}

void CCluster::SetIntegratingPNid( int pnid )
{
    const char method_name[] = "CCluster::SetIntegratingPNid";
    TRACE_ENTRY;

    integratingPNid_ = pnid;

    TRACE_EXIT;
}

// Save information about a new communicator for a node that is reintegrating
void CCluster::addNewComm(int pnid, int otherRank,  MPI_Comm comm)
{
    const char method_name[] = "CCluster::addNewComm";
    TRACE_ENTRY;

    if (trace_settings & TRACE_RECOVERY)
    {
        trace_printf("%s@%d - saving communicator for pnid %d\n",
                     method_name, __LINE__, pnid);
    }

    // Insert info for new comm into list
    commInfo_t commInfo = {pnid, otherRank, comm, -1, {0, 0}};
    clock_gettime(CLOCK_REALTIME, &commInfo.ts);

    newCommsLock_.lock();
    newComms_.push_back( commInfo );
    newCommsLock_.unlock();

    TRACE_EXIT;
}

// A node is reintegrating.   Add the communicator for the node to the set of
// communicators used by "Allgather".
void CCluster::setNewComm( int pnid )
{
    const char method_name[] = "CCluster::setNewComm";
    TRACE_ENTRY;

    newComms_t::iterator it;
    bool foundComm = false;

    if ( comms_[pnid] != MPI_COMM_NULL )
    {   // Unexpectedly already have a communicator for this node
        char buf[MON_STRING_BUF_SIZE];
        snprintf(buf, sizeof(buf), "[%s]  Unexpectedly already have a "
                 "communicator for node %d\n", method_name, pnid);
        mon_log_write(MON_CLUSTER_SETNEWCOMM_1, SQ_LOG_ERR, buf);

        MPI_Comm_free( &comms_[pnid] );
        if ( CommType == CommType_Sockets )
        {
            shutdown( socks_[pnid], SHUT_RDWR);
            close( socks_[pnid] );
            socks_[pnid] = -1;
        }
    }

    newCommsLock_.lock();
    for ( it = newComms_.begin(); it != newComms_.end(); )
    {
        if ( it->pnid == pnid )
        {
            if ( comms_[pnid] != MPI_COMM_NULL )
            {   // Found another communicator for the specified node.
                // Disconnect from the previous one.  It must be a
                // stale leftover from a previous reintegration
                // attempt for the node.
                if (trace_settings & TRACE_RECOVERY)
                {
                    trace_printf("%s@%d - discarding stale communicator for "
                                 "pnid %d\n", method_name, __LINE__, pnid);
                }

                MPI_Comm_free( &comms_[pnid] );
                if ( CommType == CommType_Sockets )
                {
                    shutdown( socks_[pnid], SHUT_RDWR);
                    close( socks_[pnid] );
                    socks_[pnid] = -1;
                }
                --currentNodes_;
            }

            if (trace_settings & TRACE_RECOVERY)
            {
                trace_printf("%s@%d - setting new communicator for pnid %d, "
                             "otherRank=%d\n",
                             method_name, __LINE__, it->pnid, it->otherRank);
            }

            comms_[it->pnid] = it->comm;
            otherMonRank_[it->pnid] = it->otherRank;
            ++currentNodes_;
            // Set bit indicating node is up
            upNodes_.upNodes[it->pnid/MAX_NODE_BITMASK] |= (1ull << (it->pnid%MAX_NODE_BITMASK));

            // Delete current list element and advance to next one
            it = newComms_.erase ( it );

            foundComm = true;
        }
        else
        {   // Advance to next list element
            ++it;
        }
    }
    newCommsLock_.unlock();

    if ( !foundComm )
    {  // We have no communicator for the specified node.
        char buf[MON_STRING_BUF_SIZE];
        snprintf(buf, sizeof(buf), "[%s] Could not find a communicator for "
                 "node %d\n", method_name, pnid);
        mon_log_write(MON_CLUSTER_SETNEWCOMM_2, SQ_LOG_ERR, buf);
    }

    TRACE_EXIT;
}

// Save information about a new socket for a node that is reintegrating
void CCluster::addNewSock(int pnid, int otherRank, int sockFd)
{
    const char method_name[] = "CCluster::addNewSock";
    TRACE_ENTRY;

    if (trace_settings & TRACE_RECOVERY)
    {
        trace_printf("%s@%d - saving socket for pnid %d\n",
                     method_name, __LINE__, pnid);
    }

    // Insert info for new comm into list
    commInfo_t commInfo = {pnid, otherRank, MPI_COMM_NULL, sockFd, {0, 0}};
    clock_gettime(CLOCK_REALTIME, &commInfo.ts);

    newCommsLock_.lock();
    newComms_.push_back( commInfo );
    newCommsLock_.unlock();

    TRACE_EXIT;
}

// A node is reintegrating.   Add the socket for the node to the set of
// communicators used by "Allgather".
void CCluster::setNewSock( int pnid )
{
    const char method_name[] = "CCluster::setNewSock";
    TRACE_ENTRY;

    newComms_t::iterator it;
    bool foundSocket = false;

    if ( socks_[pnid] != -1 )
    {   // Unexpectedly already have a communicator for this node
        char buf[MON_STRING_BUF_SIZE];
        snprintf(buf, sizeof(buf), "[%s]  Unexpectedly already have a "
                 "socket for node %d\n", method_name, pnid);
        mon_log_write(MON_CLUSTER_SETNEWSOCK_1, SQ_LOG_ERR, buf);

        shutdown( socks_[pnid], SHUT_RDWR);
        close( socks_[pnid] );
        socks_[pnid] = -1;
    }

    newCommsLock_.lock();
    for ( it = newComms_.begin(); it != newComms_.end(); )
    {
        if ( it->pnid == pnid )
        {
            if ( socks_[pnid] != -1 )
            {   // Found another socket for the specified node.
                // Disconnect from the previous one.  It must be a
                // stale leftover from a previous reintegration
                // attempt for the node.
                if (trace_settings & TRACE_RECOVERY)
                {
                    trace_printf("%s@%d - discarding stale communicator for "
                                 "pnid %d\n", method_name, __LINE__, pnid);
                }

                shutdown( socks_[pnid], SHUT_RDWR);
                close( socks_[pnid] );
                socks_[pnid] = -1;
                --currentNodes_;
            }

            CNode *node = Nodes->GetNode( it->pnid );
            socks_[it->pnid] = it->socket; // setNewSock
            sockPorts_[it->pnid] = node->GetSyncSocketPort();
            otherMonRank_[it->pnid] = it->otherRank;
            ++currentNodes_;

            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                trace_printf( "%s@%d - Setting new communicator for %d (%s), "
                              "socks_[%d]=%d, sockPorts_[%d]=%d, otherMonRank_[%d]=%d\n"
                            , method_name, __LINE__
                            , node->GetPNid()
                            , node->GetName()
                            , it->pnid, socks_[it->pnid]
                            , it->pnid, sockPorts_[it->pnid]
                            , it->pnid, otherMonRank_[it->pnid] );
            }

            // Set bit indicating node is up
            upNodes_.upNodes[it->pnid/MAX_NODE_BITMASK] |= (1ull << (it->pnid%MAX_NODE_BITMASK));

            // Delete current list element and advance to next one
            it = newComms_.erase ( it );

            foundSocket = true;
        }
        else
        {   // Advance to next list element
            ++it;
        }
    }
    newCommsLock_.unlock();

    if ( !foundSocket )
    {  // We have no communicator for the specified node.
        char buf[MON_STRING_BUF_SIZE];
        snprintf(buf, sizeof(buf), "[%s] Could not find a socket for "
                 "node %d\n", method_name, pnid);
        mon_log_write(MON_CLUSTER_SETNEWSOCK_2, SQ_LOG_ERR, buf);
    }

    TRACE_EXIT;
}

int CCluster::Allgather( int nbytes, void *sbuf, char *rbuf, int tag, MPI_Status *stats )
{
    const char method_name[] = "CCluster::Allgather";
    TRACE_ENTRY;

    int err = 0;

    switch( CommType )
    {
        case CommType_InfiniBand:
            err = AllgatherIB( nbytes, sbuf, rbuf, tag, stats );
            break;
        case CommType_Sockets:
            err = AllgatherSock( nbytes, sbuf, rbuf, tag, stats );
            break;
        default:
            // Programmer bonehead!
            MPI_Abort(MPI_COMM_SELF,99);
    }

    TRACE_EXIT;
    return err;
}

int CCluster::AllgatherIB( int nbytes, void *sbuf, char *rbuf, int tag, MPI_Status *stats )
{
    const char method_name[] = "CCluster::AllgatherIB";
    TRACE_ENTRY;

    int e;
    int err = 0;

    MPI_Request r[2*GetConfigPNodesCount()];
    MPI_Status s[2*GetConfigPNodesCount()];
    for ( int i = 0; i < 2*GetConfigPNodesCount(); i++ )
    {
        s[i].MPI_ERROR = MPI_SUCCESS;
        r[i] = MPI_REQUEST_NULL;
    }

    char *cp = rbuf;
    for ( int i = 0; i < GetConfigPNodesCount(); i++ )
    {
        if ( comms_[i] != MPI_COMM_NULL && otherMonRank_[i] != -1 )
        {
            e = MPI_Send_init( sbuf, nbytes, MPI_CHAR, otherMonRank_[i], tag,
                comms_[i], &r[i] );
            if ( e != MPI_SUCCESS )
            {
                MPI_Error_class( e, &err );
                char buf[MON_STRING_BUF_SIZE];
                snprintf( buf, sizeof(buf)
                        , "[%s], Comunication error with pnid=%d (%s), "
                          "MPI_Send_init() error=%s (%d)\n"
                        , method_name, i, Node[i]->GetName()
                        , ErrorMsg(e), e );
                mon_log_write(MON_CLUSTER_ALLGATHERIB_1, SQ_LOG_ERR, buf);
                goto early_exit;
            }

            e = MPI_Recv_init( cp, CommBufSize, MPI_CHAR, otherMonRank_[i], tag,
                comms_[i], &r[i+GetConfigPNodesCount()] );
            if ( e != MPI_SUCCESS )
            {
                MPI_Error_class( e, &err );
                char buf[MON_STRING_BUF_SIZE];
                snprintf( buf, sizeof(buf)
                        , "[%s], Comunication error with pnid=%d (%s), "
                          "MPI_Recv_init() error=%s (%d)\n"
                        , method_name, i, Node[i]->GetName()
                        , ErrorMsg(e), e );
                mon_log_write(MON_CLUSTER_ALLGATHERIB_2, SQ_LOG_ERR, buf);
                goto early_exit;
            }
        }
        cp += CommBufSize;
    }
    for ( int i = 0; i < 2*GetConfigPNodesCount(); i++ )
    {
        if ( r[i] == MPI_REQUEST_NULL ) continue;
        e = MPI_Start( &r[i] );
        if ( e != MPI_SUCCESS )
        {
            MPI_Error_class( e, &err );
            char buf[MON_STRING_BUF_SIZE];
            int pnid = (i < GetConfigPNodesCount()) ? i : (i - GetConfigPNodesCount());
            snprintf( buf, sizeof(buf)
                    , "[%s], Comunication error with pnid=%d (%s), "
                      "MPI_Start() error=%s (%d)\n"
                    , method_name, pnid, Node[pnid]->GetName()
                    , ErrorMsg(e), e );
            mon_log_write(MON_CLUSTER_ALLGATHERIB_3, SQ_LOG_ERR, buf);
            goto early_exit;
        }
    }

    inBarrier_ = true;
    if (sonar_verify_state(SONAR_ENABLED | SONAR_MONITOR_ENABLED))
       MonStats->BarrierWaitIncr();

    e = MPI_Waitall( GetConfigPNodesCount()*2, r, s );
    if ( e != MPI_SUCCESS )
    {
        MPI_Error_class( e, &err );
        if ( err != MPI_ERR_IN_STATUS )
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf), "[%s], MPI_Waitall() error=%s (%d)\n"
                    , method_name, ErrorMsg(e), e );
            mon_log_write(MON_CLUSTER_ALLGATHERIB_4, SQ_LOG_ERR, buf);
            inBarrier_ = false;
            goto early_exit;
        }
    }

    if (sonar_verify_state(SONAR_ENABLED | SONAR_MONITOR_ENABLED))
       MonStats->BarrierWaitDecr();
    inBarrier_ = false;

    for ( int i = 0; i < GetConfigPNodesCount(); i++ )
    {
        stats[i] = s[i+GetConfigPNodesCount()];
    }
    if ( e == MPI_SUCCESS )
    {
        err = MPI_SUCCESS;
        goto early_exit;
    }

    for ( int i = 0; i < GetConfigPNodesCount(); i++ )
    {
        if ( s[i].MPI_ERROR != MPI_SUCCESS &&             // send
             s[i+GetConfigPNodesCount()].MPI_ERROR == MPI_SUCCESS )   // receive
        {
            stats[i].MPI_ERROR = s[i].MPI_ERROR;
        }
    }

early_exit:

    for ( int i = 0; i < 2*GetConfigPNodesCount(); i++ )
    {
        if ( r[i] != MPI_REQUEST_NULL )
        {
            MPI_Request_free( &r[i] );
        }
    }

    barrierCount_++;

    TRACE_EXIT;
    return err;
}

int CCluster::AllgatherSock( int nbytes, void *sbuf, char *rbuf, int tag, MPI_Status *stats )
{
    const char method_name[] = "CCluster::AllgatherSock";
    TRACE_ENTRY;

    bool reconnecting = false;
    static int hdrSize = Nodes->GetSyncHdrSize( );
    int err = MPI_SUCCESS;
    peer_t p[GetConfigPNodesMax()];
    memset( p, 0, sizeof(p) );
    tag = tag; // make compiler happy
    // Set to twice the ZClient session timeout
    static int sessionTimeout = ZClientEnabled
                                ? (ZClient->GetSessionTimeout() * 2) : 120;

    int nsent = 0, nrecv = 0;
    for ( int iPeer = 0; iPeer < GetConfigPNodesCount(); iPeer++ )
    {
        peer_t *peer = &p[indexToPnid_[iPeer]];
        stats[indexToPnid_[iPeer]].MPI_ERROR = MPI_SUCCESS;
        stats[indexToPnid_[iPeer]].count = 0;
        if ( indexToPnid_[iPeer] == MyPNID || socks_[indexToPnid_[iPeer]] == -1 )
        {
            peer->p_sending = peer->p_receiving = false;
            nsent++;
            nrecv++;
        }
        else
        {
            peer->p_sending = peer->p_receiving = true;
            peer->p_sent = peer->p_received = 0;
            peer->p_timeout_count = 0;
            peer->p_initial_check = true;
            peer->p_n2recv = -1;
            peer->p_buff = ((char *) rbuf) + (indexToPnid_[iPeer] * CommBufSize);

            struct epoll_event event;
            event.data.fd = socks_[indexToPnid_[iPeer]];
            event.events = EPOLLIN | EPOLLOUT | EPOLLET | EPOLLRDHUP | EPOLLERR | EPOLLHUP;
            EpollCtl( epollFD_, EPOLL_CTL_ADD, socks_[indexToPnid_[iPeer]], &event );
        }
    }

    if (trace_settings & (TRACE_SYNC | TRACE_SYNC_DETAIL))
    {
        for ( int i = 0; i < GetConfigPNodesCount(); i++ )
        {
            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                peer_t *peer = &p[indexToPnid_[i]];
                trace_printf( "%s@%d" " - socks_[%d]=%d, "
                              "peer->p_sending=%d, "
                              "peer->p_receiving=%d\n"
                            , method_name, __LINE__
                            , indexToPnid_[i]
                            , socks_[indexToPnid_[i]]
                            , peer->p_sending
                            , peer->p_receiving );
            }
        }
    }

    inBarrier_ = true;
    MonStats->BarrierWaitIncr( );

    static int sv_epoll_wait_timeout = -2;
    static int sv_epoll_retry_count = 1;
    if ( sv_epoll_wait_timeout == -2 )
    {
        char *lv_epoll_wait_timeout_env = getenv( "SQ_MON_EPOLL_WAIT_TIMEOUT" );
        if ( lv_epoll_wait_timeout_env )
        {
            // convert to milliseconds
            sv_epoll_wait_timeout = atoi( lv_epoll_wait_timeout_env ) * 1000;
            char *lv_epoll_retry_count_env = getenv( "SQ_MON_EPOLL_RETRY_COUNT" );
            if ( lv_epoll_retry_count_env )
            {
                sv_epoll_retry_count = atoi( lv_epoll_retry_count_env );
            }
            if ( sv_epoll_retry_count > 180 )
            {
                sv_epoll_retry_count = 180;
            }
        }
        else
        {
            // default to 64 seconds
            sv_epoll_wait_timeout = 16000;
            sv_epoll_retry_count = 4;
        }

        char buf[MON_STRING_BUF_SIZE];
        snprintf( buf, sizeof(buf)
                , "[%s@%d] EPOLL timeout wait_timeout=%d msecs, retry_count=%d\n"
                , method_name
                ,  __LINE__
                , sv_epoll_wait_timeout
                , sv_epoll_retry_count );

        mon_log_write( MON_CLUSTER_ALLGATHERSOCK_1, SQ_LOG_INFO, buf );
    }

    // do the work
    struct epoll_event events[2*GetConfigPNodesMax() + 1];
    while ( 1 )
    {
reconnected:
        bool checkConnections = false;
        bool doReconnect = false;
        bool resetConnections = false;
        int peerTimedoutCount = 0;
        int maxEvents = 2*GetConfigPNodesCount() - nsent - nrecv;
        if ( maxEvents == 0 ) break;
        int nw;
        peer_t *peer;

        while ( 1 )
        {
            nw = epoll_wait( epollFD_, events, maxEvents, sv_epoll_wait_timeout );
            if ( nw >= 0 || errno != EINTR ) break;
        }

        if ( nw == 0 )
        { // Timeout, no fd's ready
            for ( int iPeer = 0; iPeer < GetConfigPNodesCount(); iPeer++ )
            { // Check no IO completion on peers
                peer = &p[indexToPnid_[iPeer]];
                if ( (peer->p_receiving) || (peer->p_sending) )
                {
                    peerTimedoutCount++;
                    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                    {
                        trace_printf( "%s@%d - EPOLL timeout (%d) on: %s(%d), "
                                      "socks_[%d]=%d, "
                                      "peer->p_sending=%d, "
                                      "peer->p_receiving=%d\n"
                                    , method_name, __LINE__
                                    , peerTimedoutCount
                                    , Node[indexToPnid_[iPeer]]->GetName(), indexToPnid_[iPeer]
                                    , indexToPnid_[iPeer]
                                    , socks_[indexToPnid_[iPeer]]
                                    , peer->p_sending
                                    , peer->p_receiving );
                    }

                    if (peer->p_initial_check && !reconnecting)
                    { // Set the session timeout relative to now
                        peer->p_initial_check = false;
                        clock_gettime(CLOCK_REALTIME, &peer->znodeFailedTime);
                        peer->znodeFailedTime.tv_sec += sessionTimeout;
                        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                        {
                            trace_printf( "%s@%d" " - Znode Fail Time %ld(secs)\n"
                                        , method_name, __LINE__
                                        , peer->znodeFailedTime.tv_sec);
                        }
                    }

                    if ( IsRealCluster && peer->p_timeout_count < sv_epoll_retry_count )
                    {
                        peer->p_timeout_count++;
                        checkConnections = true;
                        if (peer->p_timeout_count == sv_epoll_retry_count)
                        {
                            resetConnections = true;
                        }
                    }
                    else
                    {
                        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                        {
                            trace_printf( "%s@%d" " - Peer timed out: %s(%d), "
                                          "socks_[%d]=%d, "
                                          "peer->p_timeout_count=%d\n"
                                        , method_name, __LINE__
                                        , Node[indexToPnid_[iPeer]]->GetName(), indexToPnid_[iPeer]
                                        , indexToPnid_[iPeer]
                                        , socks_[indexToPnid_[iPeer]]
                                        , peer->p_timeout_count );
                        }
                    }
                }
            } // Check no IO completion on peers

            if (checkConnections)
            {
                checkConnections = false;
                if (trace_settings & TRACE_RECOVERY)
                {
                    trace_printf( "%s@%d - Initializing AllgatherSockReconnect(),"
                                  " peerTimedoutCount=%d\n"
                                , method_name, __LINE__
                                , peerTimedoutCount );
                }
                // First, check ability to connect to all peers
                // An err returned will mean that connect failed with
                // at least one peer. No err implies that possible network
                // reset occurred and there is probably one dead connection
                // to a peer where no IOs will complete ever, so connections
                // to all peers must be reestablished.
                err = AllgatherSockReconnect( stats, false );
                if (err == MPI_SUCCESS)
                { // Connections to all peers are good
                    if (resetConnections)
                    { // Establish new connections on all peers
                        resetConnections = false;
                        err = AllgatherSockReconnect( stats, true );
                        // Redrive IOs on new peer connections
                        nsent = 0; nrecv = 0;
                        for ( int i = 0; i < GetConfigPNodesCount(); i++ )
                        {
                            peer = &p[indexToPnid_[i]];
                            if ( indexToPnid_[i] == MyPNID || socks_[indexToPnid_[i]] == -1 )
                            { // peer is me or not available
                                peer->p_sending = peer->p_receiving = false;
                                nsent++;
                                nrecv++;
                            }
                            else
                            {
                                peer->p_sending = peer->p_receiving = true;
                                peer->p_sent = peer->p_received = 0;
                                peer->p_n2recv = -1;
                                peer->p_buff = ((char *) rbuf) + (indexToPnid_[i] * CommBufSize);
                                struct epoll_event event;
                                event.data.fd = socks_[indexToPnid_[i]];
                                event.events = EPOLLIN | EPOLLOUT | EPOLLET | EPOLLRDHUP | EPOLLERR | EPOLLHUP;
                                EpollCtl( epollFD_, EPOLL_CTL_ADD, socks_[indexToPnid_[i]], &event );
                            }
                        }
                    } // (resetConnections)
                } // (err == MPI_SUCCESS)
                else
                {
                    for ( int i = 0; i < GetConfigPNodesCount(); i++ )
                    {
                        peer = &p[indexToPnid_[i]];
                        if ( indexToPnid_[i] != MyPNID && socks_[indexToPnid_[i]] == -1 )
                        { // peer is me or no longer available
                            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY) &&
                                (peer->p_sending || peer->p_receiving) )
                            {
                                trace_printf( "%s@%d No IO completion on %s(%d):socks_[%d]=%d, "
                                              "peer->p_sending=%d, "
                                              "peer->p_receiving=%d\n"
                                            , method_name, __LINE__
                                            , Node[indexToPnid_[i]]->GetName(), indexToPnid_[i]
                                            , indexToPnid_[i]
                                            , socks_[indexToPnid_[i]]
                                            , peer->p_sending
                                            , peer->p_receiving );
                            }
                            if (peer->p_sending)
                            {
                                nsent++;
                                peer->p_sending = false;
                            }
                            if (peer->p_receiving)
                            {
                                peer->p_receiving = false;
                                nrecv++;
                            }
                        }
                    }
                }
                doReconnect = true;
            } // (checkConnections)

            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                for ( int i = 0; i < GetConfigPNodesCount(); i++ )
                {
                    peer = &p[indexToPnid_[i]];
                    trace_printf( "%s@%d doReconnect=%d, %s(%d):socks_[%d]=%d, "
                                  "peer->p_sending=%d, "
                                  "peer->p_receiving=%d\n"
                                , method_name, __LINE__
                                , doReconnect
                                , Node[indexToPnid_[i]]->GetName(), indexToPnid_[i]
                                , indexToPnid_[i]
                                , socks_[indexToPnid_[i]]
                                , peer->p_sending
                                , peer->p_receiving );
                }
            }

            if (doReconnect)
            {
                reconnectSeqNum_ = seqNum_;
                reconnecting = true;
                if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                {
                    trace_printf( "%s@%d" " - Reconnecting! (reconnectSeqNum_=%lld)\n"
                                , method_name, __LINE__, reconnectSeqNum_ );
                }
                goto reconnected;
            }
        }  // ( nw == 0 )

        if ( nw < 0 )
        { // Got an error
            char ebuff[256];
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf), "[%s@%d] epoll_wait(%d, %d) error: %s\n",
                method_name, __LINE__, epollFD_, maxEvents,
                strerror_r( errno, ebuff, 256 ) );
            mon_log_write( MON_CLUSTER_ALLGATHERSOCK_3, SQ_LOG_CRIT, buf );
            MPI_Abort( MPI_COMM_SELF,99 );
        }

        // Process fd's which are ready to initiate an IO or completed IO
        for ( int iEvent = 0; iEvent < nw; iEvent++ )
        {
            bool stateChange = false;
            int fd = events[iEvent].data.fd;
            int iPeer;
            for ( iPeer = 0; iPeer < GetConfigPNodesCount(); iPeer++ )
            { // Find corresponding peer by matching socket fd
                if ( events[iEvent].data.fd == socks_[indexToPnid_[iPeer]] ) break;
            }
            if ( indexToPnid_[iPeer] < 0 || indexToPnid_[iPeer] >= GetConfigPNodesMax() || indexToPnid_[iPeer] == MyPNID
                || socks_[indexToPnid_[iPeer]] == -1
                || (!p[indexToPnid_[iPeer]].p_sending && !p[indexToPnid_[iPeer]].p_receiving) )
            {
                char buf[MON_STRING_BUF_SIZE];
                snprintf( buf, sizeof(buf)
                        , "[%s@%d] Invalid peer %d, "
                          "peer.p_sending=%d, "
                          "peer.p_receiving=%d\n"
                        , method_name, __LINE__
                        , indexToPnid_[iPeer]
                        , indexToPnid_[iPeer] >= GetConfigPNodesMax()?-1:p[indexToPnid_[iPeer]].p_sending
                        , indexToPnid_[iPeer] >= GetConfigPNodesMax()?-1:p[indexToPnid_[iPeer]].p_receiving );
                mon_log_write( MON_CLUSTER_ALLGATHERSOCK_4, SQ_LOG_CRIT, buf );
                MPI_Abort( MPI_COMM_SELF,99 );
            }
            peer_t *peer = &p[indexToPnid_[iPeer]];
            if ( (events[iEvent].events & EPOLLERR) ||
                 (events[iEvent].events & EPOLLHUP) ||
                 ( !(events[iEvent].events & (EPOLLIN|EPOLLOUT))) )
            {
                // An error has occurred on this fd, or the socket is not
                // ready for reading nor writing
                char buf[MON_STRING_BUF_SIZE];
                snprintf( buf, sizeof(buf)
                        , "[%s@%d] Error: peer=%d, events[%d].data.fd=%d, event[%d]=%s\n"
                        , method_name, __LINE__
                        , indexToPnid_[iPeer]
                        , iEvent
                        , events[iEvent].data.fd
                        , iEvent
                        , EpollEventString(events[iEvent].events) );
                mon_log_write( MON_CLUSTER_ALLGATHERSOCK_5, SQ_LOG_CRIT, buf );
                stats[indexToPnid_[iPeer]].MPI_ERROR = MPI_ERR_EXITED;
                err = MPI_ERR_IN_STATUS;
                if ( peer->p_sending )
                {
                    peer->p_sending = false;
                    nsent++;
                }
                if ( peer->p_receiving )
                {
                    peer->p_receiving = false;
                    nrecv++;
                }
                stateChange = true;
                goto early_exit;
            }
            if ( peer->p_receiving && events[iEvent].events & EPOLLIN )
            { // Got receive (read) completion
                int eagain_ok = 0;
read_again:
                char *r = &peer->p_buff[peer->p_received];
                int n2get;
                if ( peer->p_received >= hdrSize )
                {
                    n2get = peer->p_n2recv;
                }
                else
                {
                    n2get = hdrSize - peer->p_received;
                }
                int nr;
                while ( 1 )
                {
                    if (trace_settings & TRACE_SYNC_DETAIL)
                    {
                        trace_printf( "%s@%d - EPOLLIN from %s(%d),"
                                      " sending=%d,"
                                      " receiving=%d (%d)"
                                      " sent=%d,"
                                      " received=%d"
                                      " timeout_count=%d,"
                                      " initial_check=%d,"
                                      " n2recv=%d\n"
                                    , method_name, __LINE__
                                    , Node[indexToPnid_[iPeer]]->GetName(), indexToPnid_[iPeer]
                                    , peer->p_sending
                                    , peer->p_receiving, n2get
                                    , peer->p_sent
                                    , peer->p_received
                                    , peer->p_timeout_count
                                    , peer->p_initial_check
                                    , peer->p_n2recv );
                    }
                    nr = recv( fd, r, n2get, 0 );
                    if ( nr > 0 ) Meas.addSockAllGatherRcvdBytes( nr );
                    if ( nr >= 0 || errno == EINTR ) break;
                }
                if ( nr < 0 )
                {
                    if ( nr < 0 && eagain_ok && errno == EAGAIN )
                    {
                        // do nothing
                    }
                    else
                    {
                        // error, down socket
                        int err = errno;
                        char buf[MON_STRING_BUF_SIZE];
                        snprintf( buf, sizeof(buf)
                                , "[%s@%d] recv[%d](%d) error %d (%s)\n"
                                , method_name, __LINE__
                                , indexToPnid_[iPeer], nr , err, strerror(err) );
                        mon_log_write( MON_CLUSTER_ALLGATHERSOCK_6, SQ_LOG_CRIT, buf );
                        peer->p_receiving = false;
                        nrecv++;
                        if ( peer->p_sending )
                        {
                            peer->p_sending = false;
                            nsent++;
                        }
                        stats[indexToPnid_[iPeer]].MPI_ERROR = MPI_ERR_EXITED;
                        err = MPI_ERR_IN_STATUS;
                        stateChange = true;
                    }
                }
                else
                {
                    peer->p_received += nr;
                    if ( peer->p_received < hdrSize )
                    {
                        // do nothing
                    }
                    else
                    {
                        if ( peer->p_received == hdrSize )
                        {
                            // got the complete header, get buffer size
                            struct sync_buffer_def *sb;
                            sb = (struct sync_buffer_def *)peer->p_buff;
                            peer->p_n2recv = sb->msgInfo.msg_offset;
                            if ( peer->p_n2recv )
                            {
                                eagain_ok = 1;
                                goto read_again;
                            }
                        }
                        else
                        {
                            // reading buffer, update counters
                            peer->p_n2recv -= nr;
                        }
                        if ( peer->p_n2recv < 0 )
                        {
                            char buf[MON_STRING_BUF_SIZE];
                            snprintf( buf, sizeof(buf),
                                "[%s@%d] error n2recv %d\n",
                                method_name, __LINE__, peer->p_n2recv );
                            mon_log_write( MON_CLUSTER_ALLGATHERSOCK_7, SQ_LOG_CRIT, buf );
                            MPI_Abort( MPI_COMM_SELF,99 );
                        }
                        if ( peer->p_n2recv == 0 )
                        {
                            // this buffer is done
                            peer->p_receiving = false;
                            nrecv++;
                            stats[indexToPnid_[iPeer]].count = peer->p_received;
                            if (trace_settings & TRACE_SYNC_DETAIL)
                            {
                                trace_printf( "%s@%d - EPOLLOUT to %s(%d),"
                                              " sending=%d,"
                                              " receiving=%d (%d)"
                                              " sent=%d,"
                                              " received=%d"
                                              " timeout_count=%d,"
                                              " initial_check=%d,"
                                              " n2recv=%d\n"
                                            , method_name, __LINE__
                                            , Node[indexToPnid_[iPeer]]->GetName(), indexToPnid_[iPeer]
                                            , peer->p_sending
                                            , peer->p_receiving, n2get
                                            , peer->p_sent
                                            , peer->p_received
                                            , peer->p_timeout_count
                                            , peer->p_initial_check
                                            , peer->p_n2recv );
                            }
                            stateChange = true;
                        }
                    }
                }
            }
            if ( peer->p_sending  && events[iEvent].events & EPOLLOUT )
            { // Got send (write) completion
                char *s = &((char *)sbuf)[peer->p_sent];
                int n2send = nbytes - peer->p_sent;
                int ns;
                while ( 1 )
                {
                    if (trace_settings & TRACE_SYNC_DETAIL)
                    {
                        trace_printf( "%s@%d - EPOLLOUT to %s(%d),"
                                      " sending=%d (%d),"
                                      " receiving=%d"
                                      " sent=%d,"
                                      " received=%d"
                                      " timeout_count=%d,"
                                      " initial_check=%d,"
                                      " n2recv=%d\n"
                                    , method_name, __LINE__
                                    , Node[indexToPnid_[iPeer]]->GetName(), indexToPnid_[iPeer]
                                    , peer->p_sending, n2send
                                    , peer->p_receiving
                                    , peer->p_sent
                                    , peer->p_received
                                    , peer->p_timeout_count
                                    , peer->p_initial_check
                                    , peer->p_n2recv );
                    }
                    ns = send( fd, s, n2send, 0 );
                    if ( ns > 0 ) Meas.addSockAllGatherSentBytes( ns );
                    if ( ns >= 0 || errno != EINTR ) break;
                }
                if ( ns < 0 )
                {
                    // error, down socket
                    int err = errno;
                    char buf[MON_STRING_BUF_SIZE];
                    snprintf( buf, sizeof(buf)
                            , "[%s@%d] send[%d](%d) error=%d (%s)\n"
                            , method_name, __LINE__
                            , indexToPnid_[iPeer], ns, err, strerror(err) );
                    mon_log_write( MON_CLUSTER_ALLGATHERSOCK_8, SQ_LOG_CRIT, buf );
                    peer->p_sending = false;
                    nsent++;
                    if ( peer->p_receiving )
                    {
                        peer->p_receiving = false;
                        nrecv++;
                    }
                    stats[indexToPnid_[iPeer]].MPI_ERROR = MPI_ERR_EXITED;
                    err = MPI_ERR_IN_STATUS;
                    stateChange = true;
                }
                else
                {
                    peer->p_sent += ns;
                    if ( peer->p_sent == nbytes )
                    {
                        // finished sending to this destination
                        peer->p_sending = false;
                        nsent++;
                        if (trace_settings & TRACE_SYNC_DETAIL)
                        {
                            trace_printf( "%s@%d - EPOLLOUT to %s(%d),"
                                          " sending=%d (%d),"
                                          " receiving=%d"
                                          " sent=%d,"
                                          " received=%d"
                                          " timeout_count=%d,"
                                          " initial_check=%d,"
                                          " n2recv=%d\n"
                                        , method_name, __LINE__
                                        , Node[indexToPnid_[iPeer]]->GetName(), indexToPnid_[iPeer]
                                        , peer->p_sending, n2send
                                        , peer->p_receiving
                                        , peer->p_sent
                                        , peer->p_received
                                        , peer->p_timeout_count
                                        , peer->p_initial_check
                                        , peer->p_n2recv );
                        }
                        stateChange = true;
                    }
                }
            }
early_exit:
            if ( stateChange )
            {
                struct epoll_event event;
                event.data.fd = socks_[indexToPnid_[iPeer]];
                int op = 0;
                if ( !peer->p_sending && !peer->p_receiving )
                {
                    op = EPOLL_CTL_DEL;
                    event.events = 0;
                }
                else if ( peer->p_sending )
                {
                    op = EPOLL_CTL_MOD;
                    event.events = EPOLLOUT | EPOLLET | EPOLLRDHUP;
                }
                else if ( peer->p_receiving )
                {
                    op = EPOLL_CTL_MOD;
                    event.events = EPOLLIN | EPOLLET | EPOLLRDHUP;
                }
                if ( op == EPOLL_CTL_DEL || op == EPOLL_CTL_MOD )
                {
                    EpollCtl( epollFD_, op, fd, &event );
                }
            }
        }
    }

    MonStats->BarrierWaitDecr( );
    inBarrier_ = false;

    barrierCount_++;

    TRACE_EXIT;
    return err;
}

int CCluster::AllgatherSockReconnect( MPI_Status *stats, bool reestablishConnections )
{
    const char method_name[] = "CCluster::AllgatherSockReconnect";
    TRACE_ENTRY;

    int err = MPI_SUCCESS;
    int idst;
    int reconnectSock = -1;
    CNode *node;

    // Loop on each node in the cluster
    for ( int i = 0; i < GetConfigPNodesMax(); i++ )
    {
        // Loop on each adjacent node in the cluster
        for ( int j = i+1; j < GetConfigPNodesMax(); j++ )
        {
            if ( i == MyPNID )
            { // Current [i] node is my node, so connect to [j] node

                idst = j;
                node = Nodes->GetNode( idst );
                if (!node) continue;
                if (node->GetState() != State_Up)
                {
                    if (socks_[idst] != -1)
                    { // Peer socket is still active
                        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                        {
                            trace_printf( "%s@%d - Node %s (%d) is not up, "
                                          "removing old socket from epoll set, "
                                          "socks_[%d]=%d\n"
                                        , method_name, __LINE__
                                        , node->GetName(), node->GetPNid()
                                        , idst, socks_[idst] );
                        }
                        stats[idst].MPI_ERROR = MPI_ERR_EXITED;
                        stats[idst].count = 0;
                        err = MPI_ERR_IN_STATUS;
                        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                        {
                            trace_printf( "%s@%d - Setting Node %s (%d) status to "
                                          "stats[%d].MPI_ERROR=%s\n"
                                        , method_name, __LINE__
                                        , node->GetName(), node->GetPNid()
                                        , idst
                                        , ErrorMsg(stats[idst].MPI_ERROR) );
                        }

                        --currentNodes_;
                        // Clear bit in set of "up nodes"
                        upNodes_.upNodes[idst/MAX_NODE_BITMASK] &= ~(1ull << (idst%MAX_NODE_BITMASK));
            
                        // Remove old socket from epoll set, it may not be there
                        struct epoll_event event;
                        event.data.fd = socks_[idst];
                        event.events = 0;
                        EpollCtlDelete( epollFD_, socks_[idst], &event );
                        socks_[idst] = -1;
                    }
                    continue;
                }
                if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                {
                    trace_printf( "%s@%d - Pinging Node %s (%d) to see if it's up\n"
                                , method_name, __LINE__
                                , node->GetName(), node->GetPNid() );
                }
                if (PingSockPeer(node))
                {
                    reconnectSock = ConnectSockPeer( node, idst, reestablishConnections );
                    if (reconnectSock == -1)
                    {
                        stats[idst].MPI_ERROR = MPI_ERR_EXITED;
                        stats[idst].count = 0;
                        err = MPI_ERR_IN_STATUS;
                        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                        {
                            trace_printf( "%s@%d - Setting Node %s (%d) status to "
                                          "stats[%d].MPI_ERROR=%s\n"
                                        , method_name, __LINE__
                                        , node->GetName(), node->GetPNid()
                                        , idst
                                        , ErrorMsg(stats[idst].MPI_ERROR) );
                        }
                    }
                }
                else
                {
                    if (socks_[idst] != -1)
                    {
                        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                        {
                            trace_printf( "%s@%d - Node %s (%d) is not up, "
                                          "removing old socket from epoll set, "
                                          "socks_[%d]=%d\n"
                                        , method_name, __LINE__
                                        , node->GetName(), node->GetPNid()
                                        , idst, socks_[idst] );
                        }

                        --currentNodes_;
                        // Clear bit in set of "up nodes"
                        upNodes_.upNodes[idst/MAX_NODE_BITMASK] &= ~(1ull << (idst%MAX_NODE_BITMASK));
            
                        // Remove old socket from epoll set, it may not be there
                        struct epoll_event event;
                        event.data.fd = socks_[idst];
                        event.events = 0;
                        EpollCtlDelete( epollFD_, socks_[idst], &event );
                        socks_[idst] = -1;
                    }
                    reconnectSock = -1;
                    stats[idst].MPI_ERROR = MPI_ERR_EXITED;
                    stats[idst].count = 0;
                    err = MPI_ERR_IN_STATUS;
                    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                    {
                        trace_printf( "%s@%d - Setting Node %s (%d) status to "
                                      "stats[%d].MPI_ERROR=%s\n"
                                    , method_name, __LINE__
                                    , node->GetName(), node->GetPNid()
                                    , idst
                                    , ErrorMsg(stats[idst].MPI_ERROR) );
                    }
                }
            }
            else if ( j == MyPNID )
            { // Current [j] is my node, accept connection from peer [i] node

                idst = i;
                node = Nodes->GetNode( idst );
                if (!node) continue;
                if (node->GetState() != State_Up)
                {
                    if (socks_[idst] != -1)
                    { // Peer socket is still active
                        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                        {
                            trace_printf( "%s@%d - Node %s (%d) is not up, "
                                          "removing old socket from epoll set, "
                                          "socks_[%d]=%d\n"
                                        , method_name, __LINE__
                                        , node->GetName(), node->GetPNid()
                                        , idst, socks_[idst] );
                        }
                        stats[idst].MPI_ERROR = MPI_ERR_EXITED;
                        stats[idst].count = 0;
                        err = MPI_ERR_IN_STATUS;
                        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                        {
                            trace_printf( "%s@%d - Setting Node %s (%d) status to "
                                          "stats[%d].MPI_ERROR=%s\n"
                                        , method_name, __LINE__
                                        , node->GetName(), node->GetPNid()
                                        , idst
                                        , ErrorMsg(stats[idst].MPI_ERROR) );
                        }

                        --currentNodes_;
                        // Clear bit in set of "up nodes"
                        upNodes_.upNodes[idst/MAX_NODE_BITMASK] &= ~(1ull << (idst%MAX_NODE_BITMASK));
            
                        // Remove old socket from epoll set, it may not be there
                        struct epoll_event event;
                        event.data.fd = socks_[idst];
                        event.events = 0;
                        EpollCtlDelete( epollFD_, socks_[idst], &event );
                        socks_[idst] = -1;
                    }
                    continue;
                }
                if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                {
                    trace_printf( "%s@%d - Pinging Node %s (%d) to see if it's up\n"
                                , method_name, __LINE__
                                , node->GetName(), node->GetPNid() );
                }
                if (PingSockPeer(node))
                {
                    reconnectSock = AcceptSockPeer( node, idst, reestablishConnections );
                    if (reconnectSock == -1)
                    {
                        stats[idst].MPI_ERROR = MPI_ERR_EXITED;
                        stats[idst].count = 0;
                        err = MPI_ERR_IN_STATUS;
                        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                        {
                            trace_printf( "%s@%d - Setting Node %s (%d) status to "
                                          "stats[%d].MPI_ERROR=%s\n"
                                        , method_name, __LINE__
                                        , node->GetName(), node->GetPNid()
                                        , idst
                                        , ErrorMsg(stats[idst].MPI_ERROR) );
                        }
                    }
                }
                else
                {
                    if (socks_[idst] != -1)
                    {
                        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                        {
                            trace_printf( "%s@%d - Node %s (%d) is not up, "
                                          "removing old socket from epoll set, "
                                          "socks_[%d]=%d\n"
                                        , method_name, __LINE__
                                        , node->GetName(), node->GetPNid()
                                        , idst, socks_[idst] );
                        }

                        --currentNodes_;
                        // Clear bit in set of "up nodes"
                        upNodes_.upNodes[idst/MAX_NODE_BITMASK] &= ~(1ull << (idst%MAX_NODE_BITMASK));
            
                        // Remove old socket from epoll set, it may not be there
                        struct epoll_event event;
                        event.data.fd = socks_[idst];
                        event.events = 0;
                        EpollCtlDelete( epollFD_, socks_[idst], &event );
                        socks_[idst] = -1;
                    }
                    reconnectSock = -1;
                    stats[idst].MPI_ERROR = MPI_ERR_EXITED;
                    stats[idst].count = 0;
                    err = MPI_ERR_IN_STATUS;
                    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                    {
                        trace_printf( "%s@%d - Setting Node %s (%d) status to "
                                      "stats[%d].MPI_ERROR=%s\n"
                                    , method_name, __LINE__
                                    , node->GetName(), node->GetPNid()
                                    , idst
                                    , ErrorMsg(stats[idst].MPI_ERROR) );
                    }
                }
            }
            else
            {
                idst = -1;
            }
            if ( idst >= 0
              && reconnectSock != -1
              && socks_[idst] != -1
              && fcntl( socks_[idst], F_SETFL, O_NONBLOCK ) )
            {
                err = MPI_ERR_AMODE;
                char ebuff[256];
                char buf[MON_STRING_BUF_SIZE];
                snprintf( buf, sizeof(buf), "[%s@%d] fcntl(socks_[%d]=%d,F_SETFL,NONBLOCK) error: %s\n",
                    method_name, __LINE__,idst, socks_[idst], strerror_r( errno, ebuff, 256 ) );
                mon_log_write( MON_CLUSTER_ALLGATHERSOCKRECONN_1, SQ_LOG_CRIT, buf );
            }
        }
    }

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        for ( int i = 0; i < GetConfigPNodesCount(); i++ )
        {
            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                trace_printf( "%s@%d" " - socks_[%d]=%d, "
                              "stats[%d].MPI_ERROR=%s\n"
                            , method_name, __LINE__
                            , indexToPnid_[i]
                            , socks_[indexToPnid_[i]]
                            , indexToPnid_[i]
                            , ErrorMsg(stats[indexToPnid_[i]].MPI_ERROR) );
            }
        }
        trace_printf( "%s@%d - Returning err=%d\n"
                    , method_name, __LINE__, err );
    }

    TRACE_EXIT;
    return( err );
}

int CCluster::AcceptSockPeer( CNode *node, int peer, bool reestablishConnections )
{
    const char method_name[] = "CCluster::AcceptSockPeer";
    TRACE_ENTRY;

    int rc = MPI_SUCCESS;
    int reconnectSock = -1;
    struct hostent *he;

    // Get my host structure via my node name
    he = gethostbyname( MyNode->GetName() );
    if ( !he )
    {
        char ebuff[256];
        char buf[MON_STRING_BUF_SIZE];
        snprintf( buf, sizeof(buf)
                , "[%s@%d] gethostbyname(%s) error: %s\n"
                , method_name, __LINE__
                , MyNode->GetName()
                , strerror_r( h_errno, ebuff, 256 ) );
        mon_log_write( MON_CLUSTER_ACCEPTSOCKPEER_1, SQ_LOG_CRIT, buf );
        abort();
    }
    else
    {
        if (trace_settings & TRACE_RECOVERY)
        {
            trace_printf( "%s@%d Accepting server socket: from %s(%d), port=%d\n"
                        , method_name, __LINE__
                        , node->GetName(), node->GetPNid()
                        , MyNode->GetSyncSocketPort() );
        }

        // Accept connection from peer
        reconnectSock = AcceptSock( syncSock_ );
        if (reconnectSock != -1)
        {
            if (trace_settings & TRACE_RECOVERY)
            {
                trace_printf( "%s@%d Server %s(%d) accepted from client %s(%d), old socks_[%d]=%d, new socks_[%d]=%d\n"
                            , method_name, __LINE__
                            , MyNode->GetName(), MyPNID
                            , node->GetName(), node->GetPNid()
                            , peer, socks_[peer]
                            , peer, reconnectSock);
            }
        }
        else
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf), "[%s@%d] AcceptSock(%d) failed!\n",
                method_name, __LINE__, syncSock_ );
            mon_log_write( MON_CLUSTER_ACCEPTSOCKPEER_2, SQ_LOG_ERR, buf );
            rc = -1;
        }

        if (reestablishConnections)
        {
            if (socks_[peer] != -1)
            {
                // Remove old socket from epoll set, it may not be there
                struct epoll_event event;
                event.data.fd = socks_[peer];
                event.events = 0;
                EpollCtlDelete( epollFD_, socks_[peer], &event );
                if (node->GetState() != State_Up)
                {
                    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                    {
                        trace_printf( "%s@%d - Node %s (%d) is not up, "
                                      "removing old socket from epoll set, "
                                      "socks_[%d]=%d\n"
                                    , method_name, __LINE__
                                    , node->GetName(), node->GetPNid()
                                    , peer, socks_[peer] );
                    }
                    socks_[peer] = -1;
                }
            }
            if (reconnectSock != -1)
            {
                socks_[peer] = reconnectSock; // AcceptSockPeer
            }
        }
        else
        {
            if (reconnectSock != -1)
            {
                close( (int)reconnectSock );
            }
        }
    }

    TRACE_EXIT;
    return rc;
}

int CCluster::ConnectSockPeer( CNode *node, int peer, bool reestablishConnections )
{
    const char method_name[] = "CCluster::ConnectSockPeer";
    TRACE_ENTRY;

    int rc = MPI_SUCCESS;
    int reconnectSock = -1;
    unsigned char srcaddr[4], dstaddr[4];
    struct hostent *he;

    // Get my host structure via my node name
    he = gethostbyname( MyNode->GetName() );
    if ( !he )
    {
        char ebuff[256];
        char buf[MON_STRING_BUF_SIZE];
        snprintf( buf, sizeof(buf)
                , "[%s@%d] gethostbyname(%s) error: %s\n"
                , method_name, __LINE__
                , MyNode->GetName()
                , strerror_r( h_errno, ebuff, 256 ) );
        mon_log_write( MON_CLUSTER_CONNECTSOCKPEER_1, SQ_LOG_CRIT, buf );
        abort();
    }
    else
    {
        // Initialize my source address structure
        memcpy( srcaddr, he->h_addr, 4 );
        // Get peer's host structure via its node name
        he = gethostbyname( node->GetName() );
        if ( !he )
        {
            char ebuff[256];
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf),
                "[%s@%d] gethostbyname(%s) error: %s\n",
                method_name, __LINE__, node->GetName(),
                strerror_r( h_errno, ebuff, 256 ) );
            mon_log_write( MON_CLUSTER_CONNECTSOCKPEER_2, SQ_LOG_CRIT, buf );
            abort();
        }
        // Initialize peer's destination address structure
        memcpy( dstaddr, he->h_addr, 4 );

        if (trace_settings & TRACE_RECOVERY)
        {
            trace_printf( "%s@%d Creating client socket: src=%d.%d.%d.%d, "
                          "dst(%s)=%d.%d.%d.%d, dst port=%d\n"
                        , method_name, __LINE__
                        , (int)((unsigned char *)srcaddr)[0]
                        , (int)((unsigned char *)srcaddr)[1]
                        , (int)((unsigned char *)srcaddr)[2]
                        , (int)((unsigned char *)srcaddr)[3]
                        ,  node->GetName()
                        , (int)((unsigned char *)dstaddr)[0]
                        , (int)((unsigned char *)dstaddr)[1]
                        , (int)((unsigned char *)dstaddr)[2]
                        , (int)((unsigned char *)dstaddr)[3]
                        , sockPorts_[peer] );
        }
        // Connect to peer
        reconnectSock = MkCltSock( srcaddr, dstaddr, sockPorts_[peer] );
        if (reconnectSock != -1)
        {
            if (trace_settings & TRACE_RECOVERY)
            {
                trace_printf( "%s@%d Client %s(%d) connected to server %s(%d), old socks_[%d]=%d, new socks_[%d]=%d\n"
                            , method_name, __LINE__
                            , MyNode->GetName(), MyPNID
                            , node->GetName(), node->GetPNid()
                            , peer, socks_[peer]
                            , peer, reconnectSock);
            }
        }
        else
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf)
                    , "[%s@%d] MkCltSock() src=%d.%d.%d.%d, "
                      "dst(%s)=%d.%d.%d.%d failed!\n"
                    , method_name, __LINE__
                    , (int)((unsigned char *)srcaddr)[0]
                    , (int)((unsigned char *)srcaddr)[1]
                    , (int)((unsigned char *)srcaddr)[2]
                    , (int)((unsigned char *)srcaddr)[3]
                    ,  node->GetName()
                    , (int)((unsigned char *)dstaddr)[0]
                    , (int)((unsigned char *)dstaddr)[1]
                    , (int)((unsigned char *)dstaddr)[2]
                    , (int)((unsigned char *)dstaddr)[3] );
            mon_log_write( MON_CLUSTER_CONNECTSOCKPEER_3, SQ_LOG_ERR, buf );
            rc = -1;
        }

        if (reestablishConnections)
        {
            if (socks_[peer] != -1)
            {
                // Remove old socket from epoll set, it may not be there
                struct epoll_event event;
                event.data.fd = socks_[peer];
                event.events = 0;
                EpollCtlDelete( epollFD_, socks_[peer], &event );
                if (node->GetState() != State_Up)
                {
                    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                    {
                        trace_printf( "%s@%d - Node %s (%d) is not up, "
                                      "removing old socket from epoll set, "
                                      "socks_[%d]=%d\n"
                                    , method_name, __LINE__
                                    , node->GetName(), node->GetPNid()
                                    , peer, socks_[peer] );
                    }
                    socks_[peer] = -1;
                }
            }
            if (reconnectSock != -1)
            {
                socks_[peer] = reconnectSock; // ConnectSockPeer
            }
        }
        else
        {
            if (reconnectSock != -1)
            {
                close( (int)reconnectSock );
            }
        }
    }

    TRACE_EXIT;
    return( rc );
}

// When we get a communication error for a point-to-point monitor communicator
// verify that the other nodes in the cluster also lost communications
// with that monitor.  If all nodes concur we consider that monitor
// down.
void CCluster::ValidateClusterState( cluster_state_def_t nodestate[],
                                     bool haveDivergence)
{
    const char method_name[] = "CCluster::ValidateClusterState";

    exitedMons_t::iterator it;
    upNodes_t nodeMask;

    for ( int i =0; i < MAX_NODE_MASKS ; i++ )
    {
        nodeMask.upNodes[i] = 0;
    }

    for ( it = exitedMons_.begin(); it != exitedMons_.end(); )
    {
        if (trace_settings & (TRACE_SYNC | TRACE_RECOVERY | TRACE_INIT))
        {
            trace_printf("%s@%d checking exited pnid=%d, detecting pnid=%d, seqNum=%lld"
                         " (current seqNum_=%lld)\n", method_name, __LINE__,
                         it->exitedPnid, it->detectingPnid, it->seqNum, seqNum_);
        }

        if ( seqNum_ >= (it->seqNum + 2) )
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf), "[%s] Validating exited node %d, "
                      "detected by node %d at seq #%lld "
                      "(current seq # is %lld).\n",
                      method_name, it->exitedPnid, it->detectingPnid,
                      it->seqNum, seqNum_);
            mon_log_write(MON_CLUSTER_VALIDATE_STATE_1, SQ_LOG_ERR, buf);

            int concurringNodes = 0;

            // Check if all active nodes see the node as down.
            nodeMask.upNodes[it->exitedPnid/MAX_NODE_BITMASK] = 1ull << (it->exitedPnid%MAX_NODE_BITMASK);
            string setSeesUp;
            string setSeesDown;
            char nodeX[10];

            // Evaluate each active (up) node in the cluster
            int pnodesCount = 0;
            for (int index = 0;
                 index < GetConfigPNodesMax() && pnodesCount < currentNodes_;
                 ++index)
            {
                if ( nodestate[index].seq_num != 0 )
                {  // There is valid nodestate info from node "index"

                    pnodesCount++;

                    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                    {
                        trace_printf( "%s@%d down pnid= %d: nodestate[%d].nodeMask.upNodes[%d]=%llx, downNodeMask.upNodes[%d]=%llx\n"
                                    , method_name, __LINE__
                                    , it->exitedPnid
                                    , index, (it->exitedPnid/MAX_NODE_BITMASK)
                                    , nodestate[index].nodeMask.upNodes[it->exitedPnid/MAX_NODE_BITMASK]
                                    , (index/MAX_NODE_BITMASK)
                                    , nodeMask.upNodes[it->exitedPnid/MAX_NODE_BITMASK] );
                    }

                    if ((nodestate[index].nodeMask.upNodes[it->exitedPnid/MAX_NODE_BITMASK] &
                         nodeMask.upNodes[it->exitedPnid/MAX_NODE_BITMASK]) == 0)
                    {  // Node "pnid" sees the node as down

                        // temp trace
                        if (trace_settings & (TRACE_SYNC | TRACE_RECOVERY | TRACE_INIT))
                        {
                            trace_printf("%s@%d node %d concurs that node %d "
                                         "is down\n", method_name, __LINE__,
                                         /*indexToPnid_[index]*/ index, it->exitedPnid);
                        }

                        snprintf(nodeX, sizeof(nodeX), "%d, ", /*indexToPnid_[index]*/ index);
                        setSeesDown.append(nodeX);

                        ++concurringNodes;
                    }
                    else
                    {
                        // temp trace
                        if (trace_settings & (TRACE_SYNC | TRACE_RECOVERY | TRACE_INIT))
                        {
                            trace_printf("%s@%d node %d says node %d is up\n",
                                         method_name, __LINE__, /*indexToPnid_[index]*/ index,
                                         it->exitedPnid);
                        }

                        snprintf(nodeX, sizeof(nodeX), "%d, ", /*indexToPnid_[index]*/ index);
                        setSeesUp.append(nodeX);

                    }
                }
                else
                {
                    // temp trace
                    if (trace_settings & (TRACE_SYNC | TRACE_RECOVERY | TRACE_INIT))
                    {
                        trace_printf("%s@%d ignoring state from node %d\n",
                                     method_name, __LINE__, /*indexToPnid_[index]*/ index);
                    }
                }
            }

            if (trace_settings & (TRACE_SYNC | TRACE_RECOVERY | TRACE_INIT))
            {
                trace_printf("%s@%d concurringNodes=%d, currentNodes_=%d\n",
                             method_name, __LINE__, concurringNodes, currentNodes_);
            }

            if (concurringNodes == currentNodes_)
            {   // General agreement that node is down, proceed to mark it down

                CNode *downNode = Nodes->GetNode( it->exitedPnid );
                if (downNode && downNode->GetState() != State_Down)
                {
                    // temp trace
                    if (trace_settings & (TRACE_SYNC | TRACE_RECOVERY | TRACE_INIT))
                    {
                        trace_printf("%s@%d proceeding to mark node %d down\n",
                                     method_name, __LINE__, it->exitedPnid);
                    }

                    mem_log_write(CMonLog::MON_UPDATE_CLUSTER_3, it->exitedPnid);

                    HandleDownNode(it->exitedPnid);
                }
                else
                {
                    if (trace_settings & (TRACE_SYNC | TRACE_RECOVERY | TRACE_INIT))
                    {
                        trace_printf("%s@%d Node pnid=%d is already down\n"
                                    , method_name, __LINE__
                                    , it->exitedPnid);
                    }
                }
            }
            else if ( concurringNodes != 0 && !enqueuedDown_ )
            {   // Some monitors say the node is down, others don't.
                // This is not supposed to happen.  Enqueue request to
                // bring this node down.  All monitors will do the same
                // so the cluster will be brought down.
                if (setSeesUp.length() > 2)
                    setSeesUp.erase(setSeesUp.length()-2, 2);
                if (setSeesDown.length() > 2)
                    setSeesDown.erase(setSeesDown.length()-2, 2);
                char buf[MON_STRING_BUF_SIZE*2];
                snprintf( buf, sizeof(buf), "[%s] Lost connection to node "
                          "%d but only %d of %d nodes also lost the "
                          "connection.  See up: %s.  See down: %s.  So node "
                          "%d is going down (at seq #%lld).\n", method_name,
                          it->exitedPnid, concurringNodes, currentNodes_,
                          setSeesUp.c_str(), setSeesDown.c_str(),
                          MyPNID, seqNum_ );
                mon_log_write(MON_CLUSTER_VALIDATE_STATE_2, SQ_LOG_ERR, buf);

                mem_log_write(CMonLog::MON_UPDATE_CLUSTER_4, MyPNID,
                              it->exitedPnid);

                enqueuedDown_ = true;
                ReqQueue.enqueueDownReq(MyPNID);
            }

            if (trace_settings & (TRACE_SYNC | TRACE_RECOVERY | TRACE_INIT))
            {
                trace_printf("%s@%d removing exited pnid=%d, detecting pnid=%d, seqNum=%lld"
                             " (current seqNum_=%lld)\n", method_name, __LINE__,
                             it->exitedPnid, it->detectingPnid, it->seqNum, seqNum_);
            }
            // Delete current list element and advance to next one
            it = exitedMons_.erase( it );
        }
        else
        {   // Advance to next list element
            ++it;
        }
    }


    if ( haveDivergence )
    {
        for ( int i =0; i < MAX_NODE_MASKS ; i++ )
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf)
                    , "[%s] Cluster view divergence (at seq #%lld), "
                      "node %d sees set[%d]: %llx\n"
                    , method_name, seqNum_, MyPNID, i
                    , upNodes_.upNodes[i] );
            mon_log_write(MON_CLUSTER_VALIDATE_STATE_3, SQ_LOG_ERR, buf);
        }

        // For each "up node" (from local perspective)
        // go through nodestate for each other node. If any node
        // says the node is down, add an item to the exitedMons_ list
        // for examination during the next sync cycle (by which time
        // all nodes will have had a chance to detect the down monitor.)

        int pnodesCount2 = 0;
        for (int remIndex = 0;
             remIndex < GetConfigPNodesMax() && pnodesCount2 < currentNodes_;
             ++remIndex)
        {
            bool someExited = false;
            // No need to check local monitor's view of the cluster since
            // any down connections are handled directly when detected.
            if (/*indexToPnid_[remIndex]*/remIndex == MyPNID)
            {
                pnodesCount2++;
                continue;
            }

            // No need to check a remote monitor's view when node is down
            CNode *remoteNode = Nodes->GetNode( /*indexToPnid_[remIndex]*/remIndex );
            if ( ! remoteNode )
            {   //  node is not member of cluster
                if (trace_settings & (TRACE_SYNC | TRACE_RECOVERY | TRACE_INIT))
                {
                    trace_printf("%s@%d Skipping non-existing node "
                                 "pnid=%d\n",
                                 method_name, __LINE__,
                                 /*indexToPnid_[remIndex]*/remIndex);
                }
                continue;
            }
            else if (remoteNode->GetState() == State_Down)
            {   //  node is down
                if (trace_settings & (TRACE_SYNC | TRACE_RECOVERY | TRACE_INIT))
                {
                    trace_printf("%s@%d Skipping down node "
                                 "pnid=%d (%s)\n",
                                 method_name, __LINE__,
                                 /*indexToPnid_[remIndex]*/remIndex, remoteNode->GetName());
                }
                continue;
            }
            else
            {
                pnodesCount2++;
            }

            // Check if all active nodes see the node as up.
            nodeMask.upNodes[/*indexToPnid_[remIndex]*/remIndex/MAX_NODE_BITMASK] =
                1ull << (/*indexToPnid_[remIndex]*/remIndex%MAX_NODE_BITMASK);

            if ( upNodes_.upNodes[/*indexToPnid_[remIndex]*/remIndex/MAX_NODE_BITMASK] &
                 nodeMask.upNodes[/*indexToPnid_[remIndex]*/remIndex/MAX_NODE_BITMASK] )
            {  // This remote node sees node pnid as up
                int pnodesCount3 = 0;
                for (int exitedPNid = 0;
                     exitedPNid < GetConfigPNodesMax() && pnodesCount3 < currentNodes_;
                     ++exitedPNid)
                {
                    CNode *exitedNode = Nodes->GetNode( /*indexToPnid_[remIndex]*/exitedPNid );
                    if (  exitedNode &&
                         (/*indexToPnid_[remIndex]*/remIndex != exitedPNid) &&
                         (nodestate[remIndex].seq_num != 0) &&
                         (nodestate[exitedPNid].nodeMask.upNodes[/*indexToPnid_[remIndex]*/remIndex/MAX_NODE_BITMASK] &
                          nodeMask.upNodes[/*indexToPnid_[remIndex]*/remIndex/MAX_NODE_BITMASK]) == 0 )
                    {  // Node remIndex sees exitedPNid as down

                        pnodesCount3++;

                        if (trace_settings & (TRACE_SYNC | TRACE_RECOVERY | TRACE_INIT))
                        {
                            trace_printf("%s@%d Divergence, queueing "
                                         "monExited{%d, %d, %lld}\n",
                                         method_name, __LINE__, exitedPNid, /*indexToPnid_[remIndex]*/remIndex,
                                         seqNum_);
                        }

                        someExited = true;
                        monExited_t monExited = {exitedPNid, /*indexToPnid_[remIndex]*/remIndex, seqNum_};
                        exitedMons_.push_back( monExited );
                    }
                }
            }
            if (someExited)
            {
                // No need to look further for any other
                // monitor's view of node pnid.  When the
                // exitedMons_ element is processed all nodes
                // will be checked for concurrence.
                break;
            }
        }
    }
}

bool CCluster::ValidateSeqNum( cluster_state_def_t nodestate[] )
{
    const char method_name[] = "CCluster::ValidateSeqNum";

    unsigned long long seqNum;
    unsigned long long loSeqNum = seqNum_;
    unsigned long long hiSeqNum = seqNum_;
    unsigned long long seqNumBucket[MAX_NODES];
    int seqNumCount[MAX_NODES];
    int maxBucket = 0;
    bool found;
    int mostCountsIndex;

    if ( GetConfigPNodesCount() ==  1 ) return true;

    // Count occurrences of sequence numbers
    for (int pnid = 0; pnid < GetConfigPNodesMax(); pnid++)
    {
        CNode *node = Nodes->GetNode( pnid );
        if (!node) continue;
        if (node->GetState() != State_Up) continue;

        if ( pnid == MyPNID )
        {
            seqNum = nodestate[pnid].seq_num = seqNum_;
        }
        else
        {
            seqNum = nodestate[pnid].seq_num;
        }

        if (trace_settings & TRACE_SYNC)
        {
            trace_printf( "%s@%d seqNum_=%lld, nodestate[%d].seq_num=%lld\n"
                        , method_name, __LINE__
                        , seqNum_
                        , pnid
                        , nodestate[pnid].seq_num );
        }

        if (seqNum != 0)
        {
            loSeqNum = (seqNum < loSeqNum) ? seqNum : loSeqNum;
            hiSeqNum = (seqNum > hiSeqNum) ? seqNum : hiSeqNum;

            found = false;
            for (int i=0; i<maxBucket; ++i)
            {
                if ( seqNum == seqNumBucket[i] )
                {
                    ++seqNumCount[i];
                    found = true;
                    break;
                }
            }
            if ( ! found )
            {
                seqNumBucket[maxBucket] = seqNum;
                seqNumCount[maxBucket] = 1;
                ++maxBucket;
            }
        }
    }

    if ( maxBucket == 0 )
    {  // Normal case, all nodes have same sequence number
        mostCountsIndex = 0;
    }
    else
    {  // Look for majority sequence number
        int mostCounts = 0;
        mostCountsIndex = 0;
        for (int i=0; i<maxBucket; ++i)
        {
            if ( seqNumCount[i] > mostCounts )
            {
                mostCounts = seqNumCount[i];
                mostCountsIndex = i;
            }
        }
    }

    lowSeqNum_  = loSeqNum;
    highSeqNum_ = hiSeqNum;

    if (trace_settings & TRACE_SYNC)
    {
        if ( lowSeqNum_ != highSeqNum_ )
        {
            trace_printf( "%s@%d Most common seq num=%lld (%d nodes), "
                          "%d buckets, low=%lld, high=%lld, local seq num (%lld) did not match.\n"
                         , method_name, __LINE__
                         , seqNumBucket[mostCountsIndex]
                         , seqNumCount[mostCountsIndex]
                         , maxBucket
                         , lowSeqNum_
                         , highSeqNum_
                         , seqNum_ );
        }
    }

    // Fail if any sequence number does not match
    return( lowSeqNum_ == highSeqNum_ );
}

void CCluster::HandleDownNode( int pnid )
{
    const char method_name[] = "CCluster::HandleDownNode";
    TRACE_ENTRY;

    // Add to dead node name list
    CNode *downNode = Nodes->GetNode( pnid );
    assert(downNode);
    deadNodeList_.push_back( downNode );

    if (trace_settings & TRACE_INIT)
        trace_printf("%s@%d - Added down node to list, pnid=%d, name=(%s)\n", method_name, __LINE__, downNode->GetPNid(), downNode->GetName());

    // assign new leaders if needed
    AssignLeaders( pnid, downNode->GetName(), false );

    // Build available list of spare nodes
    CNode *spareNode;
    NodesList *spareNodesList = Nodes->GetSpareNodesList();
    NodesList::iterator itSn;
    for ( itSn = spareNodesList->begin(); itSn != spareNodesList->end() ; itSn++ )
    {
        spareNode = *itSn;
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            trace_printf( "%s@%d - %s (pnid=%d) is in available spare node list, state=%s, spare=%d, rank failure=%d\n"
                        , method_name, __LINE__, spareNode->GetName(), spareNode->GetPNid()
                        , StateString(spareNode->GetState()), spareNode->IsSpareNode(), spareNode->IsRankFailure());
        // if spare node is available
        if ( spareNode->IsSpareNode()    &&
             !spareNode->IsRankFailure() &&
             spareNode->GetState() == State_Up )
        {
            spareNodeVector_.push_back( spareNode );
            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                trace_printf("%s@%d - pnid=%d, name=(%s) is available Spare\n", method_name, __LINE__, spareNode->GetPNid(), spareNode->GetName());
        }
    }

    // Activate spare or down node
    NodesList::iterator itDn;
    for ( itDn = deadNodeList_.begin(); itDn != deadNodeList_.end() ; itDn++ )
    {
        downNode = *itDn;
        if ( Emulate_Down )
        {
            ReqQueue.enqueueDownReq( downNode->GetPNid() );
        }
        else
        {
            bool done = false;
            spareNode = NULL;
            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                trace_printf( "%s@%d - spare node vector size=%ld\n"
                            , method_name, __LINE__, spareNodeVector_.size());
            // Find available spare node for current down node
            for ( unsigned int ii = 0; ii < spareNodeVector_.size() && !done ; ii++ )
            {
                PNidVector sparePNids = spareNodeVector_[ii]->GetSparePNids();
                if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                    trace_printf( "%s@%d - spare pnids vector size=%ld\n"
                                , method_name, __LINE__, sparePNids.size());
                // Check each pnid it is configured to spare
                for ( unsigned int jj = 0; jj < sparePNids.size(); jj++ )
                {
                    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                        trace_printf( "%s@%d - %s (pnid=%d) is in spare node vector[%d], size=%ld\n"
                                    , method_name, __LINE__
                                    , spareNodeVector_[ii]->GetName()
                                    , spareNodeVector_[ii]->GetPNid()
                                    , jj, sparePNids.size());
                    // if this is a spare for the down node
                    if ( spareNodeVector_[ii]->IsSpareNode() &&
                         downNode->GetPNid() == sparePNids[jj] )
                    {
                        // assign it and remove it from the vector
                        spareNode = spareNodeVector_[ii];
                        spareNodeVector_.erase( spareNodeVector_.begin() + ii );
                        done = true;
                        break;
                    }
                }
            }

            if ( spareNode )
            {
                Nodes->RemoveFromSpareNodesList( spareNode );
                downNode->SetState( State_Takeover ); // change state so that pending requests could fail.
                spareNode->SetActivatingSpare( true );
                if ( spareNode->GetPNid() == MyPNID )
                {
                    ReqQueue.enqueueActivateSpareReq( spareNode, downNode, true );
                }
            }
            else
            {
                if ( downNode->IsSpareNode() )
                {
                    Nodes->RemoveFromSpareNodesList( downNode );
                }
                ReqQueue.enqueueDownReq( downNode->GetPNid() );
            }
        }
    }

    spareNodeVector_.clear();
    deadNodeList_.clear();

    TRACE_EXIT;
}

void CCluster::UpdateClusterState( bool &doShutdown,
                                   struct sync_buffer_def * syncBuf,
                                   MPI_Status *status,
                                   int sentChangeNid)
{
    const char method_name[] = "CCluster::UpdateClusterState";
    TRACE_ENTRY;

    struct sync_buffer_def *recvBuf;
#ifndef NAMESERVER_PROCESS
    struct sync_buffer_def *sendBuf = Nodes->GetSyncBuffer();
#endif
    STATE node_state;
    int change_nid;
    cluster_state_def_t nodestate[GetConfigPNodesMax()];
    bool clusterViewDivergence = false;


    // Populate nodestate array using node state info from "allgather"
    // along with local node state.
    for (int index = 0; index < GetConfigPNodesMax(); index++)
    {
        // Only process active nodes
        bool noComm;
        switch( CommType )
        {
            case CommType_InfiniBand:
                noComm = (comms_[index] == MPI_COMM_NULL) ? true : false;
                break;
            case CommType_Sockets:
                noComm = (socks_[index] == -1) ? true : false;
                break;
            default:
                // Programmer bonehead!
                abort();
        }

        if (noComm
         || status[index].MPI_ERROR != MPI_SUCCESS)
        {
            if (trace_settings & (TRACE_RECOVERY | TRACE_INIT))
            {
                if (!noComm)
                {
                    trace_printf( "%s@%d - Communication error from node %d, "
                                  " seq_num=#%lld\n"
                                , method_name, __LINE__, index
                                , seqNum_ );
                }
            }
            // Not an active node, set default values
            nodestate[index].node_state = State_Unknown;
            nodestate[index].change_nid = -1;
            nodestate[index].seq_num     = 0;
            for ( int i =0; i < MAX_NODE_MASKS ; i++ )
            {
                nodestate[index].nodeMask.upNodes[i] = 0;
            }
#ifdef NAMESERVER_PROCESS
            nodestate[index].monConnCount = -1;
#else
            nodestate[index].monProcCount = 0;
#endif

            continue;
        }

        recvBuf = (struct sync_buffer_def *)
            (((char *) syncBuf) + index * CommBufSize);

        if (trace_settings & TRACE_SYNC)
        {
            int nr;
            MPI_Get_count(&status[index], MPI_CHAR, &nr);
            trace_printf("%s@%d - Received %d bytes from node %d, "
                         ", seq_num=%lld, message count=%d\n",
                         method_name, __LINE__, nr, index,
                         recvBuf->nodeInfo.seq_num,
                         recvBuf->msgInfo.msg_count);
        }

        nodestate[index].node_state  = recvBuf->nodeInfo.node_state;
        nodestate[index].change_nid  = recvBuf->nodeInfo.change_nid;
        nodestate[index].seq_num     = recvBuf->nodeInfo.seq_num;
        nodestate[index].nodeMask    = recvBuf->nodeInfo.nodeMask;
#ifdef NAMESERVER_PROCESS
        nodestate[index].monConnCount = recvBuf->nodeInfo.monConnCount;
#else
        nodestate[index].monProcCount = recvBuf->nodeInfo.monProcCount;
#endif

        for ( int i =0; i < MAX_NODE_MASKS ; i++ )
        {
            if ( nodestate[index].nodeMask.upNodes[i] != upNodes_.upNodes[i] )
            {
                if (trace_settings & (TRACE_SYNC | TRACE_RECOVERY | TRACE_INIT))
                {
                    for ( int j =0; j < MAX_NODE_MASKS ; j++ )
                    {
                        trace_printf( "%s@%d - Divergence  (at seq #%lld), node %s "
                                      "(pnid=%d) sees cluster state[%d] %llx, local "
                                      "monitor sees %llx\n"
                                    , method_name, __LINE__
                                    , seqNum_
                                    , Node[index]->GetName()
                                    , index
                                    , j
                                    , nodestate[index].nodeMask.upNodes[j]
                                    , upNodes_.upNodes[j] );
                    }
                }
                clusterViewDivergence = true;
            }
        }

#ifndef NAMESERVER_PROCESS
        if (trace_settings & (TRACE_SYNC_DETAIL | TRACE_TMSYNC))
        {
           trace_printf( "%s@%d - Node %s (pnid=%d) TmSyncState=(%d)(%s)\n"
                       , method_name, __LINE__
                       , Node[index]->GetName()
                       , index
                       , recvBuf->nodeInfo.tmSyncState
                       , SyncStateString( recvBuf->nodeInfo.tmSyncState ));
        }
#endif

#ifndef NAMESERVER_PROCESS
        if ( Node[index]->GetTmSyncState() != recvBuf->nodeInfo.tmSyncState )
        {
            Node[index]->SetTmSyncState(recvBuf->nodeInfo.tmSyncState);
            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
            {
                trace_printf("%s@%d - Node %s (pnid=%d) TmSyncState updated"
                             " (%d)(%s)\n", method_name, __LINE__,
                             Node[index]->GetName(), index,
                             recvBuf->nodeInfo.tmSyncState,
                             SyncStateString( recvBuf->nodeInfo.tmSyncState ));
            }
        }
#endif

        // Check if we need to increase my node's shutdown level ...
        // all nodes should be at the highest level selected from any source
        if ( MyNode->GetShutdownLevel() < recvBuf->nodeInfo.sdLevel )
        {
            MyNode->SetShutdownLevel( recvBuf->nodeInfo.sdLevel );
            if (MyNode->GetState() == State_Up)
            {
                MyNode->SetState( State_Shutdown );
            }
            if (trace_settings & (TRACE_REQUEST | TRACE_SYNC))
                trace_printf("%s@%d - Node %s Shutdown Level updated (%d)\n",
                             method_name, __LINE__,
                             Node[index]->GetName(), recvBuf->nodeInfo.sdLevel);
        }

        Node[index]->SetInternalState( recvBuf->nodeInfo.internalState );
        if ( recvBuf->nodeInfo.internalState == State_Ready_To_Exit )
        {   // The node is exiting.  Don't communicate with it any more.
            if (trace_settings & (TRACE_REQUEST | TRACE_SYNC))
                trace_printf("%s@%d - Node %s (%d) ready to exit, setting comm "
                             "to null\n", method_name, __LINE__,
                             Node[index]->GetName(), index);

            switch( CommType )
            {
                case CommType_InfiniBand:
                    MPI_Comm_free( &comms_[index] );
                    break;
                case CommType_Sockets:
                    shutdown( socks_[index], SHUT_RDWR );
                    close( socks_[index] );
                    socks_[index] = -1;
                    break;
                default:
                    // Programmer bonehead!
                    abort();
            }
            Node[index]->SetState( State_Down );
            --currentNodes_;
            // Clear bit in set of "up nodes"
            upNodes_.upNodes[index/MAX_NODE_BITMASK] &= ~(1ull << (index%MAX_NODE_BITMASK));
        }
    }

    if ( (checkSeqNum_ || reconnectSeqNum_ != 0)
      && !ValidateSeqNum( nodestate )
      && !enqueuedDown_ )
    {
        if ( reconnectSeqNum_ == 0 && MyNode->GetState() == State_Up )
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf(buf, sizeof(buf), "[%s] Sync cycle sequence number (%lld) "
                     "incorrect.  Aborting!\n", method_name, seqNum_);
            mon_log_write(MON_CLUSTER_UPDTCLUSTERSTATE_1, SQ_LOG_CRIT, buf);
            mem_log_write(CMonLog::MON_UPDATE_CLUSTER_2, MyPNID);
            abort();
        }
    }

    nodestate[MyPNID].node_state = Node[MyPNID]->GetState();
    nodestate[MyPNID].change_nid = sentChangeNid;
    nodestate[MyPNID].seq_num = seqNum_;
    nodestate[MyPNID].nodeMask = upNodes_;
#ifdef NAMESERVER_PROCESS
    nodestate[MyPNID].monConnCount = Node[MyPNID]->GetMonConnCount();
#else
    nodestate[MyPNID].monProcCount = Node[MyPNID]->GetNumProcs();
#endif

    // Examine status returned from MPI receive requests
    for (int index = 0; index < GetConfigPNodesMax(); index++)
    {
        bool noComm;
        switch( CommType )
        {
            case CommType_InfiniBand:
                noComm = (comms_[index] == MPI_COMM_NULL) ? true : false;
                break;
            case CommType_Sockets:
                noComm = (socks_[index] == -1) ? true : false;
                break;
            default:
                // Programmer bonehead!
                abort();
        }
        if (noComm) continue;

        if (status[index].MPI_ERROR != MPI_SUCCESS)
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf(buf, sizeof(buf), "[%s] MPI communications error=%d "
                     "(%s) for node %d (at seq #%lld).\n", method_name,
                     status[index].MPI_ERROR, ErrorMsg(status[index].MPI_ERROR),
                     index,  seqNum_);
            mon_log_write(MON_CLUSTER_UPDTCLUSTERSTATE_2, SQ_LOG_ERR, buf);

            if ( status[index].MPI_ERROR == MPI_ERR_EXITED )
            {   // A monitor has gone away

                mem_log_write(CMonLog::MON_UPDATE_CLUSTER_1, index);

                switch( CommType )
                {
                    case CommType_InfiniBand:
                        MPI_Comm_free( &comms_[index] );
                        break;
                    case CommType_Sockets:
                        shutdown( socks_[index], SHUT_RDWR );
                        close( socks_[index] );
                        socks_[index] = -1;
                        break;
                    default:
                        // Programmer bonehead!
                        abort();
                }
                --currentNodes_;

                // Clear bit in set of "up nodes"
                upNodes_.upNodes[index/MAX_NODE_BITMASK] &= ~(1ull << (index%MAX_NODE_BITMASK));

                // Pretend node is still up until down node processing
                // completes.
                nodestate[index].node_state = State_Unknown;
                nodestate[index].change_nid  = -1;
                nodestate[index].seq_num     = 0;
                for ( int i =0; i < MAX_NODE_MASKS ; i++ )
                {
                    nodestate[index].nodeMask.upNodes[i] = 0;
                }
#ifdef NAMESERVER_PROCESS
                nodestate[index].monConnCount = -1;
#else
                nodestate[index].monProcCount = 0;
#endif

                if ( validateNodeDown_ )
                {
                    if (trace_settings & (TRACE_SYNC | TRACE_RECOVERY | TRACE_INIT))
                    {
                        trace_printf( "%s@%d Divergence, queueing "
                                      "monExited{%d, %d, %lld}\n"
                                    , method_name, __LINE__
                                    , index, MyPNID, seqNum_ );
                    }
                    // Save info for the exited monitor so can confirm
                    // that all monitors have the same view.
                    monExited_t monExited = {index, MyPNID, seqNum_};
                    exitedMons_.push_back( monExited );
                }
                else
                {
                    HandleDownNode(index);
                }
            }
        }
    }

    if ( validateNodeDown_ )
        ValidateClusterState( nodestate, clusterViewDivergence );

#ifndef NAMESERVER_PROCESS
    if (trace_settings & (TRACE_SYNC_DETAIL | TRACE_TMSYNC))
    {
       trace_printf( "%s@%d - Node %s (pnid=%d) TmSyncState=(%d)(%s)\n"
                   , method_name, __LINE__
                   , MyNode->GetName()
                   , MyPNID
                   , sendBuf->nodeInfo.tmSyncState
                   , SyncStateString( sendBuf->nodeInfo.tmSyncState ));
    }
#endif

    // Update our node states
    for (int index = 0; index < GetConfigPNodesMax(); index++)
    {
        node_state = (STATE)nodestate[index].node_state;
        change_nid = nodestate[index].change_nid;

        if ( index == MyPNID &&
             MyNode->GetState() == State_Merged && seqNum_ == 1)
        {   // Initial "allgather" for this re-integrated monitor.

            seqNum_ = EnsureAndGetSeqNum(nodestate);

            if (trace_settings & (TRACE_SYNC | TRACE_RECOVERY | TRACE_INIT))
            {
                trace_printf("%s@%d Completed initial allgather for pnid=%d, "
                             "state=%d(%s), seqNum_=%lld\n", method_name, __LINE__,
                             index, MyNode->GetState(),
                             StateString(MyNode->GetState()), seqNum_ );
            }

            // Queue the node up request for processing by a
            // worker thread.
            ReqQueue.enqueueUpReq( MyPNID, NULL, -1 );
        }

        if ( change_nid == MyPNID )
        {
            if( MyNode->GetState() == State_Down ||
                MyNode->GetState() == State_Merged ||
                MyNode->GetState() == State_Joining )
            {
                if (trace_settings & TRACE_RECOVERY)
                    trace_printf( "%s@%d enqueueing node up, state=%s\n",
                                  method_name, __LINE__,
                                  StateString(MyNode->GetState()) );

                // Queue the node up request for processing by a
                // worker thread.
                ReqQueue.enqueueUpReq( MyPNID, NULL, -1 );
            }
            else
            {   // This node is being "downed"

                if (trace_settings & TRACE_RECOVERY)
                    trace_printf( "%s@%d enqueueing node down, state=%s\n",
                                  method_name, __LINE__,
                                  StateString(MyNode->GetState()) );

                // Queue the node down request for processing by a
                // worker thread.
                ReqQueue.enqueueDownReq( MyPNID );
            }
        }
        else
        {
            // In a real cluster, existing monitors need to merge new
            // monitor.

            CNode *pnode = change_nid != -1 ? Nodes->GetNode( change_nid ) : NULL;
#ifdef NAMESERVER_PROCESS
            if ( change_nid != -1 && pnode )
#else
            if ( ! Emulate_Down && change_nid != -1 && pnode )
#endif
            {
                switch ( pnode->GetState() )
                {
                case State_Down:
                    if (trace_settings & TRACE_RECOVERY)
                        trace_printf( "%s@%d - change_nid=%d, state=%s, "
                                      "queueing up request\n",
                                      method_name, __LINE__ , change_nid,
                                      StateString(pnode->GetState()));

                    mem_log_write(CMonLog::MON_UPDATE_CLUSTER_5, change_nid);

                    // Queue the node up request for processing by a
                    // worker thread.
                    ReqQueue.enqueueUpReq( change_nid,
                                           (char *)pnode->GetName(),
                                           -1 );
                    break;
                case State_Merging:
                    if (trace_settings & TRACE_RECOVERY)
                        trace_printf( "%s@%d - change_nid=%d, state=%s, "
                                      "queueing up request\n",
                                      method_name, __LINE__ , change_nid,
                                      StateString(pnode->GetState()));

                    mem_log_write(CMonLog::MON_UPDATE_CLUSTER_6, change_nid);

                    switch( CommType )
                    {
                        case CommType_InfiniBand:
                            setNewComm(change_nid);
                            break;
                        case CommType_Sockets:
                            setNewSock(change_nid);
                            break;
                        default:
                            // Programmer bonehead!
                            MPI_Abort(MPI_COMM_SELF,99);
                    }
                    pnode->SetState( State_Merged );
                    ReqQueue.enqueueUpReq( change_nid,
                                           (char *)pnode->GetName(),
                                           -1 );
                    break;

                case State_Merged:
                case State_Joining:
                default:
                    if (trace_settings & TRACE_RECOVERY)
                        trace_printf( "%s@%d - change_nid=%d, state=%s, "
                                      "no action required.\n",
                                      method_name, __LINE__ , change_nid,
                                      StateString( pnode->GetState() ));
                    break;
                }
            }
        }
        switch ( node_state )
        {
        case State_Up:
        case State_Joining:
        case State_Merged:
        case State_Merging:
        case State_Initializing:
        case State_Unlinked:
        case State_Unknown:
           break;
        case State_Down:
            doShutdown = true;
            break;
        case State_Stopped:
        case State_Shutdown:
            if (trace_settings & TRACE_SYNC_DETAIL)
                trace_printf("%s@%d - Node %d is stopping.\n", method_name, __LINE__, index);
            Node[index]->SetState( (STATE) node_state );
            doShutdown = true;
            break;
        default:
            if (trace_settings & TRACE_SYNC)
                trace_printf("%s@%d - Node %d in unknown state (%d).\n",
                             method_name, __LINE__, index, node_state);
        }
    }

#ifdef NAMESERVER_PROCESS
    // Update min monConnCount
    int minConnCount = INT_MAX;
    int minConnPnid = -1;
    for (int index = 0; index < GetConfigPNodesMax(); index++)
    {
        int connCount = nodestate[index].monConnCount;
        if ( ( connCount >= 0 ) && ( connCount < minConnCount ) )
        {
            minConnPnid = index;
            minConnCount = connCount;
        }
    }
    myMonConnCount_ = nodestate[MyPNID].monConnCount;
    minMonConnCount_ = minConnCount;
    minMonConnPnid_ = minConnPnid;
#else
    if (NameServerEnabled)
    {
        clusterProcCount_ = 0;
        for (int index = 0; index < GetConfigPNodesMax(); index++)
        {
            clusterProcCount_ += nodestate[index].monProcCount;
        }
    }
#endif

    TRACE_EXIT;
}

bool CCluster::ProcessClusterData( struct sync_buffer_def * syncBuf,
                                   struct sync_buffer_def * sendBuf,
                                   bool deferredTmSync )
{
    const char method_name[] = "CCluster::ProcessClusterData";
    TRACE_ENTRY;

    // Using the data returned from Allgather, process replication data
    // from all nodes.  If there are any TmSync messages from other
    // nodes, defer processing until all other replicated data are
    // processed.
    struct internal_msg_def *msg;
    struct sync_buffer_def *msgBuf;
    bool haveDeferredTmSync = false;

    for (int i = 0; i < GetConfigPNodesMax(); i++)
    {
        bool noComm;
        switch( CommType )
        {
            case CommType_InfiniBand:
                noComm = (comms_[i] == MPI_COMM_NULL) ? true : false;
                break;
            case CommType_Sockets:
                noComm = (socks_[i] == -1) ? true : false;
                break;
            default:
                // Programmer bonehead!
                abort();
        }
        // Only process active nodes
        if (noComm && i != MyPNID) continue;

        if ( i == MyPNID )
        {   // Get pointer to message sent by this node
            msgBuf = sendBuf;
        }
        else
        {   // Compute pointer to receive buffer element for node "i"
            msgBuf = (struct sync_buffer_def *)
                (((char *) syncBuf) + i * CommBufSize);
        }

        if (trace_settings & TRACE_SYNC)
        {
            trace_printf("%s@%d - Buffer for node %d, swpRecCount_=%d, seq_num=%lld, "
                         "lastSeqNum_=%lld, msg_count=%d, msg_offset=%d\n",
                         method_name, __LINE__, i, swpRecCount_,
                         msgBuf->nodeInfo.seq_num,
                         lastSeqNum_,
                         msgBuf->msgInfo.msg_count,
                         msgBuf->msgInfo.msg_offset);
        }

        // if we have already processed buffer, skip it
        if (lastSeqNum_ >= msgBuf->nodeInfo.seq_num) continue;

        if (trace_settings & TRACE_SYNC)
        {
            trace_printf("%s@%d - Processing buffer for node %d, swpRecCount_=%d, seq_num=%lld, "
                         "lastSeqNum_=%lld, msg_count=%d, msg_offset=%d\n",
                         method_name, __LINE__, i, swpRecCount_,
                         msgBuf->nodeInfo.seq_num,
                         lastSeqNum_,
                         msgBuf->msgInfo.msg_count,
                         msgBuf->msgInfo.msg_offset);
        }

        // reset msg length to zero to initialize for PopMsg()
        msgBuf->msgInfo.msg_offset = 0;

#ifndef NAMESERVER_PROCESS
        if ( msgBuf->msgInfo.msg_count == 1
        && (( internal_msg_def *)msgBuf->msg)->type == InternalType_Sync )
        {
            if ( deferredTmSync )
            {   // This node has sent a TmSync message.  Process it now.
                if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                    trace_printf("%s@%d - Handling deferred TmSync messages for "
                                 "node %d\n", method_name, __LINE__, i);

                struct internal_msg_def *msg;
                msg = Nodes->PopMsg( msgBuf );

                if ( i == MyPNID )
                    HandleMyNodeMsg (msg, MyPNID);
                else
                    HandleOtherNodeMsg (msg, i);
            }
            else
            {
                // This node has sent a TmSync message.  Defer processing
                // until we handle all of the non-TmSync messages from
                // other nodes.
                haveDeferredTmSync = true;

                if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                    trace_printf("%s@%d - Deferring TmSync processing for node"
                                 " %d until replicated data is handled\n",
                                 method_name, __LINE__, i);
            }
        }
        else if ( !deferredTmSync )
#else
        if ( !deferredTmSync )
#endif
        {
            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                trace_printf("%s@%d - Handling messages for "
                             "node %d\n", method_name, __LINE__, i);
            do
            {
                // Get the next sync msg for the node
                msg = Nodes->PopMsg( msgBuf );
                if (msg->type == InternalType_Null) break;

                if ( i == MyPNID )
                    HandleMyNodeMsg (msg, MyPNID);
                else
                    HandleOtherNodeMsg (msg, i);
            }
            while ( true );
        }
    }

    TRACE_EXIT;

    return haveDeferredTmSync;
}

bool CCluster::checkIfDone (  )
{
    const char method_name[] = "CCluster::checkIfDone";
    TRACE_ENTRY;

    int nameServerCount = 0;
    CClusterConfig *clusterConfig = Nodes->GetClusterConfig();
    CNameServerConfigContainer *nameServerConfigContainer = NULL;

#ifdef NAMESERVER_PROCESS

    if (clusterConfig)
    {
        nameServerConfigContainer = Nodes->GetNameServerConfig();
        if (nameServerConfigContainer)
        {
            nameServerCount = nameServerConfigContainer->GetCount();
        }
    }
#else
    int myNameServerCount = 0;
    CNameServerConfig *nameServerConfig = NULL;

    if (NameServerEnabled && clusterConfig)
    {
        nameServerConfigContainer = Nodes->GetNameServerConfig();
        if (nameServerConfigContainer)
        {
            nameServerCount = nameServerConfigContainer->GetCount();
            if (IsRealCluster)
            {
                nameServerConfig = nameServerConfigContainer->GetConfig( Node_name );
                if (nameServerConfig)
                {
                    myNameServerCount = 1;
                }
            }
            else
            {
                if (nameServerCount && MyPNID < nameServerCount)
                {
                    myNameServerCount = 1;
                }
            }
        }
    }
#endif

#ifdef NAMESERVER_PROCESS
    if (trace_settings & (TRACE_PROCESS | TRACE_PROCESS_DETAIL | TRACE_SYNC))
        trace_printf("%s@%d - Node %d shutdown level=%d, state=%s.  Process "
                     "count=%d, internal state=%d, currentNodes_=%d, "
                     "local process count=%d, shutdownNameServer=%d, "
                     "nameServerCount=%d\n",
                     method_name, __LINE__, 
                     MyNode->GetPNid(),
                     MyNode->GetShutdownLevel(),
                     StateString(MyNode->GetState()),
                     Nodes->ProcessCount(),
                     MyNode->getInternalState(),
                     currentNodes_, 
                     MyNode->GetNumProcs(),
                     MyNode->IsShutdownNameServer(),
                     nameServerCount );

#else
    if (NameServerEnabled)
    {
        if (trace_settings & (TRACE_PROCESS | TRACE_PROCESS_DETAIL | TRACE_SYNC))
            trace_printf("%s@%d - Node %d shutdown level=%d, state=%s.  Cluster process "
                         "count=%d, internal state=%d, currentNodes_=%d, "
                         "local process count=%d\n",
                         method_name, __LINE__, MyNode->GetPNid(),
                         MyNode->GetShutdownLevel(),
                         StateString(MyNode->GetState()),
                         clusterProcCount_,
                         MyNode->getInternalState(),
                         currentNodes_, MyNode->GetNumProcs());
    }
    else
    {
        if (trace_settings & (TRACE_PROCESS | TRACE_PROCESS_DETAIL | TRACE_SYNC))
            trace_printf("%s@%d - Node %d shutdown level=%d, state=%s.  Process "
                         "count=%d, internal state=%d, currentNodes_=%d, "
                         "local process count=%d\n",
                         method_name, __LINE__, MyNode->GetPNid(),
                         MyNode->GetShutdownLevel(),
                         StateString(MyNode->GetState()),
                         Nodes->ProcessCount(),
                         MyNode->getInternalState(),
                         currentNodes_, MyNode->GetNumProcs());
    }
#endif            
    // Check if we are also done
    if (( MyNode->GetState() != State_Down    ) &&
        ( MyNode->GetState() != State_Stopped )   )
    {
        if ( MyNode->GetShutdownLevel() != ShutdownLevel_Undefined )
        {
#ifdef NAMESERVER_PROCESS
            if ( (Nodes->ProcessCount() <= nameServerCount )   // only Name Servers alive
                 && (MyNode->GetNumProcs() <= MAX_PRIMITIVES ) // only My Name Server alive
                 && MyNode->IsShutdownNameServer()   // monitor shutdown Name Server received
                 && !MyNode->isInQuiesceState() )    // post-quiescing will
                                                     // expire WDG (cluster)
            {
                if (trace_settings & (TRACE_PROCESS | TRACE_PROCESS_DETAIL | TRACE_SYNC))
                   trace_printf("%s@%d - Name Server signaled to exit.\n", method_name, __LINE__);
                MyNode->SetState( State_Stopped );
                MyNode->SetInternalState(State_Ready_To_Exit);

                // we need to sync one more time so other nodes see our state
                return false;
            }
#else
            if ( NameServerEnabled )
            {
                
                if ( clusterProcCount_ == 0 )  // all Name Servers exited
                {
                    if (trace_settings & (TRACE_PROCESS | TRACE_PROCESS_DETAIL | TRACE_SYNC))
                       trace_printf("%s@%d - Monitor signaled to exit.\n", method_name, __LINE__);
                    MyNode->SetState( State_Stopped );
                    MyNode->SetInternalState(State_Ready_To_Exit);
    
                    // we need to sync one more time so other nodes see our state
                    return false;
                }
                else if ( (clusterProcCount_ <= 
                            (currentNodes_ * (MAX_PRIMITIVES+1)) ) // only WDGs and Name Servers alive
                          && (MyNode->GetNumProcs() <=
                            (MAX_PRIMITIVES+1) )                   // only WDGs and Name Servers alive
                          && !MyNode->isInQuiesceState()    // post-quiescing will
                                                            // expire WDG (cluster)
                          && !waitForWatchdogExit_ )        // WDG not yet exiting
                {
                    if (trace_settings & (TRACE_PROCESS | TRACE_PROCESS_DETAIL | TRACE_SYNC))
                       trace_printf("%s@%d - Stopping watchdog process. "
                                    "(process count: cluster=%d, MyNode=%d)\n",
                                    method_name, __LINE__,
                                    Nodes->ProcessCount(), MyNode->ProcessCount());
    
                    waitForWatchdogExit_ = true;
                    // stop the watchdog timer first
                    HealthCheck.setState(MON_STOP_WATCHDOG);
                    // let the watchdog process exit
                    HealthCheck.setState(MON_EXIT_PRIMITIVES);
                }
                else if ( (MyNode->GetNumProcs() <=         // only My Name Server alive
                            myNameServerCount )
                          && !MyNode->isInQuiesceState()    // post-quiescing will
                                                            // expire WDG (cluster)
                          && !waitForNameServerExit_ )      // Name Server not yet exiting
                {
                    if (trace_settings & (TRACE_PROCESS | TRACE_PROCESS_DETAIL | TRACE_SYNC))
                       trace_printf("%s@%d - Stopping Name Server process. "
                                    "(process count: cluster=%d, MyNode=%d)\n",
                                    method_name, __LINE__,
                                    Nodes->ProcessCount(), MyNode->ProcessCount());
    
                    waitForNameServerExit_ = true;
                    int rc = NameServer->ProcessShutdown();
                    if (rc)
                    {
                        char la_buf[MON_STRING_BUF_SIZE];
                        snprintf( la_buf, sizeof(la_buf)
                                , "[%s] - Shutdown request to Name Server failed, node going down\n"
                                , method_name );
                        mon_log_write( MON_CLUSTER_CHECKIFDONE_1, SQ_LOG_ERR, la_buf );
                        ReqQueue.enqueueDownReq( MyPNID );
                    }
                }
            }
            else
            {
                if ( Nodes->ProcessCount() == 0 )  // all WDTs exited
                {
                    if (trace_settings & (TRACE_PROCESS | TRACE_PROCESS_DETAIL | TRACE_SYNC))
                       trace_printf("%s@%d - Monitor signaled to exit.\n", method_name, __LINE__);
                    MyNode->SetState( State_Stopped );
                    MyNode->SetInternalState(State_Ready_To_Exit);
    
                    // we need to sync one more time so other nodes see our state
                    return false;
                }
                else if ( (Nodes->ProcessCount() <=
                          (currentNodes_*MAX_PRIMITIVES))        // only WDGs alive
                          && !MyNode->isInQuiesceState()    // post-quiescing will
                                                            // expire WDG (cluster)
                          && !waitForWatchdogExit_ )        // WDG not yet exiting
                {
                    if (trace_settings & (TRACE_PROCESS | TRACE_PROCESS_DETAIL | TRACE_SYNC))
                       trace_printf("%s@%d - Stopping watchdog process.\n",
                                    method_name, __LINE__);
    
                    waitForWatchdogExit_ = true;
                    // stop the watchdog timer first
                    HealthCheck.setState(MON_STOP_WATCHDOG);
                    // let the watchdog process exit
                    HealthCheck.setState(MON_EXIT_PRIMITIVES);
                }
            }
#endif
        }
    }
    else if ( MyNode->GetShutdownLevel() != ShutdownLevel_Undefined
              && MyNode->GetState() == State_Down
              && MyNode->GetNumProcs() == 0)
    {
        if (trace_settings & (TRACE_PROCESS | TRACE_PROCESS_DETAIL | TRACE_SYNC))
            trace_printf("%s@%d - No processes remaining, monitor exiting.\n",
                         method_name, __LINE__);

        MyNode->SetState( State_Stopped );
        MyNode->SetInternalState(State_Ready_To_Exit);
        // we need to sync one more time so other nodes see our state
        return false;
    }

    MyNode->CheckShutdownProcessing();

    TRACE_EXIT;

    return ( MyNode->getInternalState() == State_Ready_To_Exit );
}


// Gather "Allgather" performance statistics
// Given the beginning and ending time of an "Allgather" operation, compute
// the elapsed time and increment the count for the appropriate range
// bucket.

const struct timespec CCluster::agBuckets_[] = {
    {0,         0},  // lowest
    {0,     20000},  // 20 us
    {0,     50000},  // 50 us
    {0,    500000},  // 500 us
    {0,   1000000},  // 1 ms
    {0,  10000000},  // 10 ms
    {0,  25000000},  // 25 ms
    {0,  50000000},  // 50 ms
    {0, 100000000},  // 100 ms
    {0, 500000000}}; // 500 ms
const int CCluster::agBucketsSize_ = sizeof(agBuckets_)/sizeof(timespec);

bool CCluster::agTimeStats(struct timespec & ts_begin,
                           struct timespec & ts_end)
{
    const char method_name[] = "CCluster::agTimeStats";
    bool slowAg = false;

    struct timespec timediff;
    if ( (ts_end.tv_nsec - ts_begin.tv_nsec )  < 0 )
    {
        timediff.tv_sec = ts_end.tv_sec - ts_begin.tv_sec - 1;
        timediff.tv_nsec = 1000000000 + ts_end.tv_nsec - ts_begin.tv_nsec;
    }
    else
    {
        timediff.tv_sec = ts_end.tv_sec - ts_begin.tv_sec;
        timediff.tv_nsec = ts_end.tv_nsec - ts_begin.tv_nsec;
    }

    if ( timediff.tv_sec > agMaxElapsed_.tv_sec
         || (timediff.tv_sec == agMaxElapsed_.tv_sec
             && timediff.tv_nsec > agMaxElapsed_.tv_nsec ))
        // Have a new maximum elapsed time
        agMaxElapsed_ = timediff;

    if ( timediff.tv_sec < agMinElapsed_.tv_sec
         || (timediff.tv_sec == agMinElapsed_.tv_sec
             && timediff.tv_nsec < agMinElapsed_.tv_nsec ))
        // Have a new minimum time
        agMinElapsed_ = timediff;

    for (int i=agBucketsSize_-1; i>=0; --i)
    {
        if (timediff.tv_sec > agBuckets_[i].tv_sec
            || (timediff.tv_sec == agBuckets_[i].tv_sec
                && timediff.tv_nsec > agBuckets_[i].tv_nsec ))
        {
            ++agElapsed_[i];
            if (i >= 7)
            {
                slowAg = true;
                if (trace_settings & TRACE_SYNC)
                {
                    trace_printf("%s@%d slow Allgather=(%ld, %ld) seqNum_=%lld, i=%d\n",
                                 method_name, __LINE__,
                                 timediff.tv_sec, timediff.tv_nsec, seqNum_, i);
                }
            }
            break;
        }
    }

    return slowAg;
}

// Display "Allgather" statistics
void CCluster::stats()
{
    const char method_name[] = "CCluster::stats";

    trace_printf("%s@%d Allgather min elapsed=%ld.%ld\n", method_name, __LINE__,
                 agMinElapsed_.tv_sec, agMinElapsed_.tv_nsec);

    trace_printf("%s@%d Allgather max elapsed=%ld.%ld\n", method_name, __LINE__,
                 agMaxElapsed_.tv_sec, agMaxElapsed_.tv_nsec);

    unsigned long int bucket;
    const char * unit;
    const char * range;
    for (int i=0; i<agBucketsSize_; ++i)
    {
        if ( i == (agBucketsSize_-1))
        {
            bucket = agBuckets_[i].tv_nsec;
            range = ">";
        }
        else
        {
            bucket = agBuckets_[i+1].tv_nsec;
            range = "<=";
        }
        bucket = bucket/1000;
        if (bucket < 1000)
            unit = "usec";
        else
        {
            bucket = bucket / 1000;
            if ( bucket < 1000 )
                unit = "msec";
            else
                unit = "???";
        }
        trace_printf("%s@%d bucket[%d]=%d (%s %ld %s)\n",
                     method_name, __LINE__, i, agElapsed_[i],
                     range, bucket, unit);
    }
}

bool CCluster::exchangeNodeData ( )
{
    const char method_name[] = "CCluster::exchangeNodeData";
    TRACE_ENTRY;

    bool result = false;

    // Record statistics (sonar counters)
    if (sonar_verify_state(SONAR_ENABLED | SONAR_MONITOR_ENABLED))
       MonStats->req_sync_Incr();

    ++swpRecCount_; // recursive count for this function

    bool doShutdown = false;
    bool lastAllgatherWithLastSyncBuffer = false;

    struct internal_msg_def *msg;
    MPI_Status status[GetConfigPNodesMax()];
    int err;
    struct sync_buffer_def *recv_buffer;
    struct sync_buffer_def *send_buffer = Nodes->GetSyncBuffer();
    unsigned long long savedSeqNum = 0;

    // if we are here in a second recursive call that occurred while
    // processing TMSync data, use the second receive buffer
    // else, use the first one.
    if (swpRecCount_ == 1)
    {
      recv_buffer = recvBuffer_;
    }
    else
    {
      // should not be here in more than one recursive call.
      assert(swpRecCount_ == 2);
      recv_buffer = recvBuffer2_;
    }

    // Initialize sync buffer header including node state
    msg = Nodes->InitSyncBuffer( send_buffer, seqNum_, upNodes_ );

    // Fill sync buffer based on queue of replication requests
    Replicator.FillSyncBuffer ( msg );

reconnected:

    if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
        trace_printf( "%s@%d - doing Allgather size=%d, swpRecCount_=%d, "
                      "message count=%d, message seq_num=%lld, "
                      "seqNum_=%lld, lastSeqNum_=%lld, lowSeqNum_=%lld, "
                      "highSeqNum_=%lld, reconnectSeqNum_=%lld\n"
                    , method_name, __LINE__
                    , Nodes->GetSyncSize()
                    , swpRecCount_
                    , send_buffer->msgInfo.msg_count
                    , send_buffer->nodeInfo.seq_num
                    , seqNum_
                    , lastSeqNum_
                    , lowSeqNum_
                    , highSeqNum_
                    , reconnectSeqNum_);

    struct timespec ts_ag_begin;
    clock_gettime(CLOCK_REALTIME, &ts_ag_begin);


    // Exchange info with other nodes
    err = Allgather(Nodes->GetSyncSize(), send_buffer, (char *)recv_buffer,
             0 /*seqNum_*/, status );

    struct timespec ts_ag_end;
    clock_gettime(CLOCK_REALTIME, &ts_ag_end);

    if (err != MPI_SUCCESS && err != MPI_ERR_IN_STATUS)
    {
        if (trace_settings & TRACE_SYNC)
        {
            trace_printf("%s@%d - unexpected Allgather error=%s (%d)\n",
                         method_name, __LINE__, ErrorMsg(err), err);
        }

        char buf[MON_STRING_BUF_SIZE];
        snprintf(buf, sizeof(buf), "[%s], Unexpected MPI communications "
                 "error=%s (%d).\n", method_name, ErrorMsg(err), err);
        mon_log_write(MON_CLUSTER_EXCHANGENODEDATA_1, SQ_LOG_ERR, buf);

        // Allgather() failed in a fundamental way, bring this node down
        if ( !enqueuedDown_ )
        {
            enqueuedDown_ = true;
            ReqQueue.enqueueDownReq(MyPNID);
        }
    }
    else
    {
        if (agTimeStats( ts_ag_begin, ts_ag_end))
        {  // Slow cycle, print info
            if ( trace_settings & TRACE_SYNC )
            {
                trace_printf("%s@%d - slow Allgather info: sync size=%d, message count=%d, MyPNID=%d\n",
                             method_name, __LINE__,  Nodes->GetSyncSize(),
                             send_buffer->msgInfo.msg_count, MyPNID);
                struct sync_buffer_def *msgBuf;
                int nr;

                for (int i = 0; i < GetConfigPNodesMax(); i++)
                {
                    bool noComm;
                    switch( CommType )
                    {
                        case CommType_InfiniBand:
                            noComm = (comms_[i] == MPI_COMM_NULL) ? true : false;
                            break;
                        case CommType_Sockets:
                            noComm = (socks_[i] == -1) ? true : false;
                            break;
                        default:
                            // Programmer bonehead!
                            abort();
                    }
                    // Only process active nodes
                    if (noComm) continue;

                    msgBuf = (struct sync_buffer_def *)
                        (((char *) recv_buffer) + i * CommBufSize);

                    MPI_Get_count(&status[i], MPI_CHAR, &nr);

                    trace_printf("%s@%d - slow Allgather info, pnid=%d: received bytes=%d, message count=%d, msg_offset=%d\n",
                                 method_name, __LINE__, i, nr,
                                 msgBuf->msgInfo.msg_count,
                                 msgBuf->msgInfo.msg_offset);
                }
            }
        }

        UpdateClusterState( doShutdown
                          , recv_buffer
                          , status
                          , send_buffer->nodeInfo.change_nid);

        if ( lastAllgatherWithLastSyncBuffer )
        {
            seqNum_ = savedSeqNum;
            lastAllgatherWithLastSyncBuffer = false;
            send_buffer = Nodes->GetSyncBuffer();

            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                trace_printf( "%s@%d - Resetting lastAllgatherWithLastSyncBuffer=%d\n"
                            , method_name, __LINE__
                            , lastAllgatherWithLastSyncBuffer);

            goto reconnected;
        }

        if ( reconnectSeqNum_ != 0 )
        {

            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                trace_printf( "%s@%d - Allgather IO retry, swpRecCount_=%d, "
                              "seqNum_=%lld, lastSeqNum_=%lld, lowSeqNum_=%lld, "
                              "highSeqNum_=%lld, reconnectSeqNum_=%lld\n"
                            , method_name, __LINE__
                            , swpRecCount_
                            , seqNum_
                            , lastSeqNum_
                            , lowSeqNum_
                            , highSeqNum_
                            , reconnectSeqNum_);

            // The Allgather() has executed a reconnect at reconnectSeqNum_.
            // The UpdateClusterState has set the lowSeqNum_and highSeqNum_
            // in the current IO exchange which will indicate whether there is
            // a mismatch in IOs between monitor processes. If there is a mismatch,
            // the lowSeqNum_and highSeqNum_ relative to our current seqNum_
            // will determine how to redrive the exchange of node data.
            if (seqNum_ > lowSeqNum_)
            { // A remote monitor did not receive our last SyncBuffer
                // Redo exchange with the previous SyncBuffer
                send_buffer = Nodes->GetLastSyncBuffer();
                savedSeqNum = seqNum_;
                seqNum_ = lastSeqNum_;
                // Indicate to follow up the next exchange with current SyncBuffer
                lastAllgatherWithLastSyncBuffer = true;
                lowSeqNum_ = highSeqNum_ = reconnectSeqNum_ = 0;

                if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                    trace_printf( "%s@%d - Setting lastAllgatherWithLastSyncBuffer=%d\n"
                                , method_name, __LINE__
                                , lastAllgatherWithLastSyncBuffer);

                goto reconnected;
            }
            else if (seqNum_ < highSeqNum_)
            { // The local monitor did not receive the last remote SyncBuffer
                // Redo exchange with the current SyncBuffer
                send_buffer = Nodes->GetSyncBuffer();
                lowSeqNum_ = highSeqNum_ = reconnectSeqNum_ = 0;

                if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                    trace_printf( "%s@%d - lastAllgatherWithLastSyncBuffer=%d\n"
                                , method_name, __LINE__
                                , lastAllgatherWithLastSyncBuffer);

                goto reconnected;
            }
            lowSeqNum_ = highSeqNum_ = reconnectSeqNum_ = 0;
        }
    }

    if ( ProcessClusterData( recv_buffer, send_buffer, false ) )
    {   // There is a TmSync message remaining to be handled
        ProcessClusterData( recv_buffer, send_buffer, true );
    }

    if (swpRecCount_ == 1)
    {
        // Save the sync buffer and corresponding sequence number we just processed
        // On reconnect we must resend the last buffer and the current buffer
        // to ensure dropped buffers are processed by all monitor processe in the
        // correct order
        Nodes->SaveMyLastSyncBuffer();
        lastSeqNum_ = seqNum_;

        // Increment count of "Allgather" calls.  If wrap-around, start again at 1.
        if ( ++seqNum_ == 0) seqNum_ = 1;
    }

    // Wake up any threads waiting on the completion of a sync cycle
    syncCycle_.wakeAll();

    if (doShutdown) result = checkIfDone( );

    if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
        trace_printf( "%s@%d - node data exchange completed, swpRecCount_=%d, "
                      "seqNum_=%lld, lastSeqNum_=%lld, reconnectSeqNum_=%lld\n"
                    , method_name, __LINE__
                    , swpRecCount_
                    , seqNum_
                    , lastSeqNum_
                    , reconnectSeqNum_);

    --swpRecCount_;

    TRACE_EXIT;

    return result;
}

#ifndef NAMESERVER_PROCESS
void CCluster::exchangeTmSyncData ( struct sync_def *sync, bool bumpSync )
{
    const char method_name[] = "CCluster::exchangeTmSyncData";
    TRACE_ENTRY;

    ++swpRecCount_; // recursive count for this function

    bool doShutdown = false;
    bool lastAllgatherWithLastSyncBuffer = false;

    struct internal_msg_def *msg;
    MPI_Status status[GetConfigPNodesMax()];
    int err;
    struct sync_buffer_def *recv_buffer;
    struct sync_buffer_def *send_buffer = Nodes->GetSyncBuffer();
    unsigned long long savedSeqNum = 0;

    // if we are here in a second recursive call that occurred while
    // processing TMSync data, use the second receive buffer
    // else, use the first one.
    if (swpRecCount_ == 1)
    {
      recv_buffer = recvBuffer_;
    }
    else
    {
      // should not be here in more than one recursive call.
      assert(swpRecCount_ == 2);
      recv_buffer = recvBuffer2_;
    }

    if (bumpSync)
    {
        // Save the sync buffer and corresponding sequence number we just processed
        // On reconnect we must resend the last buffer and the current buffer
        // to ensure dropped buffers are processed by all monitor processe in the
        // correct order
        Nodes->SaveMyLastSyncBuffer();
        lastSeqNum_ = seqNum_;

        // Increment count of "Allgather" calls.  If wrap-around, start again at 1.
        if ( ++seqNum_ == 0) seqNum_ = 1;

        if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
            trace_printf( "%s@%d - Bumping sequence number, "
                          "swpRecCount_=%d, seqNum_=%lld, lastSeqNum_=%lld\n"
                        , method_name, __LINE__
                        , swpRecCount_
                        , seqNum_
                        , lastSeqNum_);

    }

    // Initialize sync buffer header including node state
    msg = Nodes->InitSyncBuffer( send_buffer, seqNum_, upNodes_ );

    // Add tmsync data
    AddTmsyncMsg( send_buffer, sync, msg );

reconnected:

    if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
        trace_printf( "%s@%d - doing Allgather size=%d, swpRecCount_=%d, "
                      "message count=%d, message seq_num=%lld, "
                      "seqNum_=%lld, lastSeqNum_=%lld, lowSeqNum_=%lld, "
                      "highSeqNum_=%lld, reconnectSeqNum_=%lld\n"
                    , method_name, __LINE__
                    , Nodes->GetSyncSize()
                    , swpRecCount_
                    , send_buffer->msgInfo.msg_count
                    , send_buffer->nodeInfo.seq_num
                    , seqNum_
                    , lastSeqNum_
                    , lowSeqNum_
                    , highSeqNum_
                    , reconnectSeqNum_);

    struct timespec ts_ag_begin;
    clock_gettime(CLOCK_REALTIME, &ts_ag_begin);


    // Exchange info with other nodes
    err = Allgather(Nodes->GetSyncSize(), send_buffer, (char *)recv_buffer,
             0 /*seqNum_*/, status );

    struct timespec ts_ag_end;
    clock_gettime(CLOCK_REALTIME, &ts_ag_end);

    if (err != MPI_SUCCESS && err != MPI_ERR_IN_STATUS)
    {
        if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
        {
            trace_printf("%s@%d - unexpected Allgather error=%s (%d)\n",
                         method_name, __LINE__, ErrorMsg(err), err);
        }

        char buf[MON_STRING_BUF_SIZE];
        snprintf(buf, sizeof(buf), "[%s], Unexpected MPI communications "
                 "error=%s (%d).\n", method_name, ErrorMsg(err), err);
        mon_log_write(MON_CLUSTER_EXCHANGETMSYNC_1, SQ_LOG_ERR, buf);

        // Allgather() failed in a fundamental way, bring this node down
        if ( !enqueuedDown_ )
        {
            enqueuedDown_ = true;
            ReqQueue.enqueueDownReq(MyPNID);
        }
    }
    else
    {
        if (agTimeStats( ts_ag_begin, ts_ag_end))
        {  // Slow cycle, print info
            if ( trace_settings & TRACE_SYNC )
            {
                trace_printf("%s@%d - slow Allgather info: sync size=%d, message count=%d, MyPNID=%d\n",
                             method_name, __LINE__,  Nodes->GetSyncSize(),
                             send_buffer->msgInfo.msg_count, MyPNID);
                struct sync_buffer_def *msgBuf;
                int nr;

                for (int i = 0; i < GetConfigPNodesMax(); i++)
                {
                    bool noComm;
                    switch( CommType )
                    {
                        case CommType_InfiniBand:
                            noComm = (comms_[i] == MPI_COMM_NULL) ? true : false;
                            break;
                        case CommType_Sockets:
                            noComm = (socks_[i] == -1) ? true : false;
                            break;
                        default:
                            // Programmer bonehead!
                            abort();
                    }
                    // Only process active nodes
                    if (noComm) continue;

                    msgBuf = (struct sync_buffer_def *)
                        (((char *) recv_buffer) + i * CommBufSize);

                    MPI_Get_count(&status[i], MPI_CHAR, &nr);

                    trace_printf("%s@%d - slow Allgather info, pnid=%d: received bytes=%d, message count=%d, msg_offset=%d\n",
                                 method_name, __LINE__, i, nr,
                                 msgBuf->msgInfo.msg_count,
                                 msgBuf->msgInfo.msg_offset);
                }
            }
        }

        UpdateClusterState( doShutdown
                          , recv_buffer
                          , status
                          , send_buffer->nodeInfo.change_nid);

        if ( lastAllgatherWithLastSyncBuffer )
        {
            seqNum_ = savedSeqNum;
            lastAllgatherWithLastSyncBuffer = false;
            send_buffer = Nodes->GetSyncBuffer();

            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                trace_printf( "%s@%d - Resetting lastAllgatherWithLastSyncBuffer=%d\n"
                            , method_name, __LINE__
                            , lastAllgatherWithLastSyncBuffer);

            goto reconnected;
        }

        if ( reconnectSeqNum_ != 0 )
        {
            if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                trace_printf( "%s@%d - Allgather IO retry, swpRecCount_=%d, "
                              "seqNum_=%lld, lastSeqNum_=%lld, lowSeqNum_=%lld, "
                              "highSeqNum_=%lld, reconnectSeqNum_=%lld\n"
                            , method_name, __LINE__
                            , swpRecCount_
                            , seqNum_
                            , lastSeqNum_
                            , lowSeqNum_
                            , highSeqNum_
                            , reconnectSeqNum_);

            // The Allgather() has executed a reconnect at reconnectSeqNum_.
            // The UpdateClusterState has set the lowSeqNum_and highSeqNum_
            // in the current IO exchange which will indicate whether there is
            // a mismatch in IOs between monitor processes. If there is a mismatch,
            // the lowSeqNum_and highSeqNum_ relative to our current seqNum_
            // will determine how to redrive the exchange of node data.
            if (seqNum_ > lowSeqNum_)
            { // A remote monitor did not receive our last SyncBuffer
                // Redo exchange with the previous SyncBuffer
                send_buffer = Nodes->GetLastSyncBuffer();
                savedSeqNum = seqNum_;
                seqNum_ = lastSeqNum_;
                // Indicate to follow up the next exchange with current SyncBuffer
                lastAllgatherWithLastSyncBuffer = true;
                lowSeqNum_ = highSeqNum_ = reconnectSeqNum_ = 0;

                if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                    trace_printf( "%s@%d - Setting lastAllgatherWithLastSyncBuffer=%d\n"
                                , method_name, __LINE__
                                , lastAllgatherWithLastSyncBuffer);

                goto reconnected;
            }
            else if (seqNum_ < highSeqNum_)
            { // The local monitor did not receive the last remote SyncBuffer
                // Redo exchange with the current SyncBuffer
                send_buffer = Nodes->GetSyncBuffer();
                lowSeqNum_ = highSeqNum_ = reconnectSeqNum_ = 0;

                if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
                    trace_printf( "%s@%d - lastAllgatherWithLastSyncBuffer=%d\n"
                                , method_name, __LINE__
                                , lastAllgatherWithLastSyncBuffer);

                goto reconnected;
            }
            lowSeqNum_ = highSeqNum_ = reconnectSeqNum_ = 0;
        }
    }

    if ( ProcessClusterData( recv_buffer, send_buffer, false ) )
    {   // There is a TmSync message remaining to be handled
        ProcessClusterData( recv_buffer, send_buffer, true );
    }

    if (swpRecCount_ == 1)
    {
        // Save the sync buffer and corresponding sequence number we just processed
        // On reconnect we must resend the last buffer and the current buffer
        // to ensure dropped buffers are processed by all monitor processe in the
        // correct order
        Nodes->SaveMyLastSyncBuffer();
        lastSeqNum_ = seqNum_;

        // Increment count of "Allgather" calls.  If wrap-around, start again at 1.
        if ( ++seqNum_ == 0) seqNum_ = 1;
    }

    if (trace_settings & (TRACE_SYNC | TRACE_TMSYNC))
        trace_printf( "%s@%d - node data exchange completed, swpRecCount_=%d, "
                      "seqNum_=%lld, lastSeqNum_=%lld, reconnectSeqNum_=%lld\n"
                    , method_name, __LINE__
                    , swpRecCount_
                    , seqNum_
                    , lastSeqNum_
                    , reconnectSeqNum_);

    --swpRecCount_;

    TRACE_EXIT;
}
#endif

void CCluster::EpollCtl( int efd, int op, int fd, struct epoll_event *event )
{
    const char method_name[] = "CCluster::EpollCtl";
    TRACE_ENTRY;
#if 0
    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        int iPeer;
        for ( iPeer = 0; iPeer < GetConfigPNodesMax(); iPeer++ )
        { // Find corresponding peer by matching socket fd
            if ( fd == socks_[iPeer] ) break;
        }
        trace_printf( "%s@%d epoll_ctl( efd=%d,%s, fd=%d(%s), %s )\n"
                    , method_name, __LINE__
                    , efd
                    , EpollOpString(op)
                    , fd, Node[iPeer]->GetName()
                    , EpollEventString(event->events) );
    }
#endif
    int rc = epoll_ctl( efd, op, fd, event );
    if ( rc == -1 )
    {
        char ebuff[256];
        char buf[MON_STRING_BUF_SIZE];
        int iPeer;
        for ( iPeer = 0; iPeer < GetConfigPNodesMax(); iPeer++ )
        { // Find corresponding peer by matching socket fd
            if ( fd == socks_[iPeer] ) break;
        }
        snprintf( buf, sizeof(buf), "[%s@%d] epoll_ctl(efd=%d,%s, fd=%d(%s), %s) error: %s\n"
                , method_name, __LINE__
                , efd
                , EpollOpString(op)
                , fd, Node[iPeer]->GetName()
                , EpollEventString(event->events)
                , strerror_r( errno, ebuff, 256 ) );
        mon_log_write( MON_CLUSTER_EPOLLCTL_1, SQ_LOG_CRIT, buf );
        MPI_Abort( MPI_COMM_SELF,99 );
    }

    TRACE_EXIT;
    return;
}

void CCluster::EpollCtlDelete( int efd, int fd, struct epoll_event *event )
{
    const char method_name[] = "CCluster::EpollCtlDelete";
    TRACE_ENTRY;

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        int iPeer;
        for ( iPeer = 0; iPeer < GetConfigPNodesMax(); iPeer++ )
        { // Find corresponding peer by matching socket fd
            if ( fd == socks_[iPeer] ) break;
        }
        trace_printf( "%s@%d epoll_ctl( efd=%d,%s, fd=%d(%s), %s )\n"
                    , method_name, __LINE__
                    , efd
                    , EpollOpString(EPOLL_CTL_DEL)
                    , fd, Node[iPeer]->GetName()
                    , EpollEventString(event->events) );
    }

    // Remove old socket from epoll set, it may not be there
    int rc = epoll_ctl( efd, EPOLL_CTL_DEL, fd, event  );
    if ( rc == -1 )
    {
        int err = errno;
        if (err != ENOENT)
        {
            char ebuff[256];
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf), "[%s@%d] epoll_ctl(efd=%d, %s, fd=%d, %s) error: %s\n"
                    , method_name, __LINE__
                    , efd
                    , EpollOpString(EPOLL_CTL_DEL)
                    , fd
                    , EpollEventString(event->events)
                    , strerror_r( err, ebuff, 256 ) );
            mon_log_write( MON_CLUSTER_EPOLLCTLDELETE_1, SQ_LOG_CRIT, buf );
            MPI_Abort( MPI_COMM_SELF,99 );
        }
    }

    TRACE_EXIT;
    return;
}

void CCluster::InitClusterSocks( int worldSize, int myRank, char *nodeNames, int *rankToPnid )
{
    const char method_name[] = "CCluster::InitClusterSocks";
    TRACE_ENTRY;

    int serverSyncPort;
    CNode *node;

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        trace_printf( "%s@%d worldSize=%d, myRank=%d\n"
                    , method_name, __LINE__
                    , worldSize, myRank);
    }

    // Exchange ports with collective
    serverSyncPort = MyNode->GetSyncSocketPort();
    int rc = MPI_Allgather( &serverSyncPort, 1, MPI_INT,
        sockPorts_, 1, MPI_INT, MPI_COMM_WORLD );
    if ( rc != MPI_SUCCESS )
    {
        char buf[MON_STRING_BUF_SIZE];
        snprintf( buf, sizeof(buf), "[%s@%d] MPI_Allgather error=%s\n",
            method_name, __LINE__, ErrorMsg( rc ) );
        mon_log_write( MON_CLUSTER_INITCLUSTERSOCKS_3, SQ_LOG_CRIT, buf );
        MPI_Abort( MPI_COMM_SELF,99 );
    }
#ifdef NAMESERVER_PROCESS
    if ( !IsRealCluster )
    {
        for ( int i = 0; i < worldSize; i++ )
            sockPorts_[i] = syncPort_ + i;
    }
#endif

    char *n, nodeName[MPI_MAX_PROCESSOR_NAME];
    unsigned char srcaddr[4], dstaddr[4];
    struct hostent *he;
    if ( nodeNames )
    {
        n = &nodeNames[myRank*MPI_MAX_PROCESSOR_NAME];
    }
    else
    {
        strcpy( nodeName, "localhost" );
        n = nodeName;
    }
    // Get my host structure via my node name or localhost
    he = gethostbyname( n );
    if ( !he )
    {
        char ebuff[256];
        char buf[MON_STRING_BUF_SIZE];
        snprintf( buf, sizeof(buf), "[%s@%d] gethostbyname(%s) error: %s\n",
            method_name, __LINE__, n, strerror_r( h_errno, ebuff, 256 ) );
        mon_log_write( MON_CLUSTER_INITCLUSTERSOCKS_4, SQ_LOG_CRIT, buf );
        MPI_Abort( MPI_COMM_SELF,99 );
    }
    // Initialize my source address structure
    memcpy( srcaddr, he->h_addr, 4 );
    int idst;
    // Loop on each node in the cluster
    for ( int i = 0; i < worldSize; i++ )
    {
        // Loop on each adjacent node in the cluster
        for ( int j = i+1; j < worldSize; j++ )
        {
            if ( i == myRank )
            { // Current [i] node is my node, so connect to [j] node
                idst = j;
                if ( nodeNames )
                { // Real cluster
                    n = &nodeNames[j*MPI_MAX_PROCESSOR_NAME];
                    // Get peer's host structure via its node name
                    he = gethostbyname( n );
                    if ( !he )
                    {
                        char ebuff[256];
                        char buf[MON_STRING_BUF_SIZE];
                        snprintf( buf, sizeof(buf),
                            "[%s@%d] gethostbyname(%s) error: %s\n",
                            method_name, __LINE__, n,
                            strerror_r( h_errno, ebuff, 256 ) );
                        mon_log_write( MON_CLUSTER_INITCLUSTERSOCKS_5, SQ_LOG_CRIT, buf );
                        MPI_Abort( MPI_COMM_SELF,99 );
                    }
                    // Initialize peer's destination address structure
                    memcpy( dstaddr, he->h_addr, 4 );
                    node = Nodes->GetNode( n );
                    if ( node )
                    { // Save peer's port in its node object
                        node->SetSyncSocketPort(sockPorts_[j]);
                    }
                }
                else
                { // Virtual cluster. Same source and destination addresses
                    node = NULL;
                    memcpy( dstaddr, srcaddr, 4 );
                }

                if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                {
                    trace_printf( "%s@%d Creating client socket: src=%d.%d.%d.%d, dst(%s)=%d.%d.%d.%d, dst port=%d\n"
                                , method_name, __LINE__
                                , (int)((unsigned char *)srcaddr)[0]
                                , (int)((unsigned char *)srcaddr)[1]
                                , (int)((unsigned char *)srcaddr)[2]
                                , (int)((unsigned char *)srcaddr)[3]
                                ,  n
                                , (int)((unsigned char *)dstaddr)[0]
                                , (int)((unsigned char *)dstaddr)[1]
                                , (int)((unsigned char *)dstaddr)[2]
                                , (int)((unsigned char *)dstaddr)[3]
                                , sockPorts_[j] );
                }
                // Connect to peer
                socks_[rankToPnid[j]] = MkCltSock( srcaddr, dstaddr, sockPorts_[j] ); // InitClusterSocks
            }
            else if ( j == myRank )
            { // Current [j] peer my node, accept connection from peer [i] node
                if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                {
                    trace_printf( "%s@%d Accepting server socket: src=%d.%d.%d.%d, port=%d\n"
                                , method_name, __LINE__
                                , (int)((unsigned char *)srcaddr)[0]
                                , (int)((unsigned char *)srcaddr)[1]
                                , (int)((unsigned char *)srcaddr)[2]
                                , (int)((unsigned char *)srcaddr)[3]
                                , serverSyncPort );
                }

                idst = i;
                // Accept connection from peer [i]
                socks_[rankToPnid[i]] = AcceptSock( syncSock_ ); // InitClusterSocks
            }
            else
            {
                idst = -1;
            }
            if ( idst >= 0 && socks_[rankToPnid[idst]] < 0 )
            {
                char buf[MON_STRING_BUF_SIZE];
                if ( idst == i )
                {
                    snprintf( buf, sizeof(buf), "[%s@%d] mkcltsock src=%d.%d.%d.%d dst=%d.%d.%d.%d failed\n",
                        method_name, __LINE__,
                        (int)((unsigned char *)srcaddr)[0],
                        (int)((unsigned char *)srcaddr)[1],
                        (int)((unsigned char *)srcaddr)[2],
                        (int)((unsigned char *)srcaddr)[3],
                        (int)((unsigned char *)dstaddr)[0],
                        (int)((unsigned char *)dstaddr)[1],
                        (int)((unsigned char *)dstaddr)[2],
                        (int)((unsigned char *)dstaddr)[3] );
                }
                else
                {
                    snprintf( buf, sizeof(buf), "[%s@%d] acceptsock(%d) failed\n",
                        method_name, __LINE__, syncSock_ );
                }
                mon_log_write( MON_CLUSTER_INITCLUSTERSOCKS_6, SQ_LOG_CRIT, buf );
                MPI_Abort( MPI_COMM_SELF,99 );
            }
            if ( idst >= 0 && fcntl( socks_[rankToPnid[idst]], F_SETFL, O_NONBLOCK ) )
            {
                char ebuff[256];
                char buf[MON_STRING_BUF_SIZE];
                snprintf( buf, sizeof(buf), "[%s@%d] fcntl(NONBLOCK) error: %s\n",
                    method_name, __LINE__, strerror_r( errno, ebuff, 256 ) );
                mon_log_write( MON_CLUSTER_INITCLUSTERSOCKS_7, SQ_LOG_CRIT, buf );
                MPI_Abort( MPI_COMM_SELF,99 );
            }
            MPI_Barrier( MPI_COMM_WORLD );
        }
    }
    TRACE_EXIT;
}

void CCluster::InitServerSock( void )
{
    const char method_name[] = "CCluster::InitServerSock";
    TRACE_ENTRY;
    int serverCommPort = 0;
    int serverSyncPort = 0;
#ifdef NAMESERVER_PROCESS
    int mon2nsPort = 0;
#else
    int ptpPort = 0;
#endif
    int val = 0;

    unsigned char addr[4];
    struct hostent *he;

    he = gethostbyname( Node_name );
    if ( !he )
    {
        char ebuff[256];
        char buf[MON_STRING_BUF_SIZE];
        snprintf( buf, sizeof(buf)
                , "[%s@%d] gethostbyname(%s) error: %s\n"
                , method_name, __LINE__
                , Node_name, strerror_r( h_errno, ebuff, 256 ) );
        mon_log_write( MON_CLUSTER_INITSERVERSOCK_1, SQ_LOG_CRIT, buf );
        abort();
    }
    memcpy( addr, he->h_addr, 4 );

#ifdef NAMESERVER_PROCESS
    char *env = getenv ("NS_COMM_PORT");
#else
    char *env = getenv("MONITOR_COMM_PORT");
#endif
    if ( env )
    {
        val = atoi(env);
        if ( val > 0)
        {
            if ( !IsRealCluster )
            {
                val += MyPNID;
            }
            serverCommPort = val;
        }
    }

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        trace_printf( "%s@%d COMM_PORT Node_name=%s, env=%s, serverCommPort=%d, val=%d\n"
                    , method_name, __LINE__
                    , Node_name, env, serverCommPort, val );
    }

    commSock_ = MkSrvSock( &serverCommPort );
    if ( commSock_ < 0 )
    {
        char ebuff[256];
        char buf[MON_STRING_BUF_SIZE];
        snprintf( buf, sizeof(buf)
#ifdef NAMESERVER_PROCESS
                , "[%s@%d] MkSrvSock(NS_COMM_PORT=%d) error: %s\n"
#else
                , "[%s@%d] MkSrvSock(MONITOR_COMM_PORT=%d) error: %s\n"
#endif
                , method_name, __LINE__, serverCommPort
                , strerror_r( errno, ebuff, 256 ) );
        mon_log_write( MON_CLUSTER_INITSERVERSOCK_2, SQ_LOG_CRIT, buf );
        abort();
    }
    else
    {
        snprintf( MyCommPort, sizeof(MyCommPort)
                , "%d.%d.%d.%d:%d"
                , (int)((unsigned char *)addr)[0]
                , (int)((unsigned char *)addr)[1]
                , (int)((unsigned char *)addr)[2]
                , (int)((unsigned char *)addr)[3]
                , serverCommPort );
        MyNode->SetCommSocketPort( serverCommPort );
        MyNode->SetCommPort( MyCommPort );

        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            trace_printf( "%s@%d Initialized my comm socket port, "
                          "pnid=%d (%s:%s) (commPort=%s)\n"
                        , method_name, __LINE__
                        , MyPNID, MyNode->GetName(), MyCommPort
                        , MyNode->GetCommPort() );

    }

#ifdef NAMESERVER_PROCESS
    env = getenv("NS_SYNC_PORT");
#else
    env = getenv("MONITOR_SYNC_PORT");
#endif
    if ( env )
    {
        val = atoi(env);
        if ( val > 0)
        {
            if ( !IsRealCluster )
            {
                val += MyPNID;
            }
            syncPort_ = serverSyncPort = val;
        }
    }

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        trace_printf( "%s@%d SYNC_PORT Node_name=%s, env=%s, serverSyncPort=%d, val=%d\n"
                    , method_name, __LINE__
                    , Node_name, env, syncPort_, val );
    }

    syncSock_ = MkSrvSock( &serverSyncPort );
    if ( syncSock_ < 0 )
    {
        char ebuff[256];
        char buf[MON_STRING_BUF_SIZE];
        snprintf( buf, sizeof(buf)
#ifdef NAMESERVER_PROCESS
                , "[%s@%d] MkSrvSock(NS_SYNC_PORT=%d) error: %s\n"
#else
                , "[%s@%d] MkSrvSock(MONITOR_SYNC_PORT=%d) error: %s\n"
#endif
                , method_name, __LINE__, serverSyncPort
                , strerror_r( errno, ebuff, 256 ) );
        mon_log_write( MON_CLUSTER_INITSERVERSOCK_3, SQ_LOG_CRIT, buf );
        abort();
    }
    else
    {
        snprintf( MySyncPort, sizeof(MySyncPort)
                , "%d.%d.%d.%d:%d"
                , (int)((unsigned char *)addr)[0]
                , (int)((unsigned char *)addr)[1]
                , (int)((unsigned char *)addr)[2]
                , (int)((unsigned char *)addr)[3]
                , serverSyncPort );
        MyNode->SetSyncSocketPort( serverSyncPort );
        MyNode->SetSyncPort( MySyncPort );

        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            trace_printf( "%s@%d Initialized my sync socket port, "
                          "pnid=%d (%s:%s) (syncPort=%s)\n"
                        , method_name, __LINE__
                        , MyPNID, MyNode->GetName(), MySyncPort
                        , MyNode->GetSyncPort() );
    }

#ifdef NAMESERVER_PROCESS
    env = getenv("NS_M2N_COMM_PORT");
    if ( env )
    {
        val = atoi(env);
        if ( val > 0)
        {
            if ( !IsRealCluster )
            {
                val += MyPNID;
            }
            mon2nsPort = val;
        }
    }

    mon2nsSock_ = MkSrvSock( &mon2nsPort );
    if ( mon2nsSock_ < 0 )
    {
        char ebuff[256];
        char buf[MON_STRING_BUF_SIZE];
        snprintf( buf, sizeof(buf)
                , "[%s@%d] MkSrvSock(NS_M2N_COMM_PORT=%d) error: %s\n"
                , method_name, __LINE__, mon2nsPort
                , strerror_r( errno, ebuff, 256 ) );
        mon_log_write( MON_CLUSTER_INITSERVERSOCK_4, SQ_LOG_CRIT, buf );
        abort();
    }
    else
    {
        snprintf( MyMon2NsPort, sizeof(MyMon2NsPort)
                , "%d.%d.%d.%d:%d"
                , (int)((unsigned char *)addr)[0]
                , (int)((unsigned char *)addr)[1]
                , (int)((unsigned char *)addr)[2]
                , (int)((unsigned char *)addr)[3]
                , mon2nsPort );
        MyNode->SetMon2NsPort( MyMon2NsPort );
        MyNode->SetMon2NsSocketPort( mon2nsPort );

        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            trace_printf( "%s@%d Initialized my mon2ns comm socket port, "
                          "pnid=%d (%s:%s) (Mon2NsPort=%s, Mon2NsSocketPort=%d)\n"
                        , method_name, __LINE__
                        , MyPNID, MyNode->GetName(), MyMon2NsPort
                        , MyNode->GetMon2NsPort()
                        , MyNode->GetMon2NsSocketPort() );

    }
#else
    if (NameServerEnabled)
    {
        env = getenv("MON2MON_COMM_PORT");
        if ( env )
        {
            val = atoi(env);
            if ( val > 0)
            {
                ptpPort = val;
            }
        }
        else
        {
           char buf[MON_STRING_BUF_SIZE];
           snprintf( buf, sizeof(buf)
                   , "[%s@%d] MON2MON_COMM_PORT environment variable is not set!\n"
                   , method_name, __LINE__ );
           mon_log_write( MON_CLUSTER_INITSERVERSOCK_5, SQ_LOG_CRIT, buf );
           abort();
        }
    
        // For virtual env, add PNid to the port so we can still test without collisions of port numbers
        if (!IsRealCluster)
        {
            ptpPort += MyNode->GetPNid();
        }
    
        ptpSock_ = MkSrvSock( &ptpPort );
        if ( ptpSock_ < 0 )
        {
            char ebuff[MON_STRING_BUF_SIZE];
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf)
                    , "[%s@%d] MkSrvSock(MON2MON_COMM_PORT=%d) error: %s\n"
                    , method_name, __LINE__, ptpPort
                    , strerror_r( errno, ebuff, MON_STRING_BUF_SIZE ) );
            mon_log_write( MON_CLUSTER_INITSERVERSOCK_6, SQ_LOG_CRIT, buf );
            abort();
        }
        else
        {
            snprintf( MyPtPPort, sizeof(MyPtPPort)
                    , "%d.%d.%d.%d:%d"
                    , (int)((unsigned char *)addr)[0]
                    , (int)((unsigned char *)addr)[1]
                    , (int)((unsigned char *)addr)[2]
                    , (int)((unsigned char *)addr)[3]
                    , ptpPort );
            MyNode->SetPtPPort( MyPtPPort );
            MyNode->SetPtPSocketPort( ptpPort );
    
            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                trace_printf( "%s@%d Initialized my ptp socket port, "
                              "pnid=%d (%s:%s) (ptpPort=%s)\n"
                            , method_name, __LINE__
                            , MyPNID, MyNode->GetName(), MyPtPPort
                            , MyNode->GetPtPPort() );
    
        }
    }
#endif

    epollFD_ = epoll_create1( EPOLL_CLOEXEC );
    if ( epollFD_ < 0 )
    {
        char ebuff[256];
        char buf[MON_STRING_BUF_SIZE];
        snprintf( buf, sizeof(buf), "[%s@%d] epoll_create1() error: %s\n",
            method_name, __LINE__, strerror_r( errno, ebuff, 256 ) );
        mon_log_write( MON_CLUSTER_INITSERVERSOCK_7, SQ_LOG_CRIT, buf );
        MPI_Abort( MPI_COMM_SELF,99 );
    }

    TRACE_EXIT;
}

int CCluster::AcceptCommSock( void )
{
    const char method_name[] = "CCluster::AcceptCommSock";
    TRACE_ENTRY;

    int csock = AcceptSock( commSock_ );

    TRACE_EXIT;
    return( csock  );
}

int CCluster::AcceptSyncSock( void )
{
    const char method_name[] = "CCluster::AcceptSyncSock";
    TRACE_ENTRY;

    int csock = AcceptSock( syncSock_ );

    TRACE_EXIT;
    return( csock  );
}

#ifndef NAMESERVER_PROCESS
int CCluster::AcceptPtPSock( void )
{
    const char method_name[] = "CCluster::AcceptPtPSock";
    TRACE_ENTRY;

    int csock = AcceptSock( ptpSock_ );

    TRACE_EXIT;
    return( csock  );
}
#endif


int CCluster::AcceptSock( int sock )
{
    const char method_name[] = "CCluster::AcceptSock";
    TRACE_ENTRY;

#if defined(_XOPEN_SOURCE_EXTENDED)
#ifdef __LP64__
    socklen_t  size;    // size of socket address
#else
    size_t   size;      // size of socket address
#endif
#else
    int    size;        // size of socket address
#endif
    int csock; // connected socket
    struct sockaddr_in  sockinfo;   // socket address info

    size = sizeof(struct sockaddr *);
    if ( getsockname( sock, (struct sockaddr *) &sockinfo, &size ) )
    {
        char buf[MON_STRING_BUF_SIZE];
        int err = errno;
        snprintf(buf, sizeof(buf), "[%s], getsockname() failed, errno=%d (%s).\n",
                 method_name, err, strerror(err));
        mon_log_write(MON_CLUSTER_ACCEPTSOCK_1, SQ_LOG_ERR, buf);
        return ( -1 );
    }

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        unsigned char *addrp = (unsigned char *) &sockinfo.sin_addr.s_addr;
        trace_printf( "%s@%d - Accepting socket on addr=%d.%d.%d.%d,  port=%d\n"
                    , method_name, __LINE__
                    , addrp[0]
                    , addrp[1]
                    , addrp[2]
                    , addrp[3]
                    , (int) ntohs( sockinfo.sin_port ) );
    }

    while ( ((csock = accept( sock
                            , (struct sockaddr *) 0
                            , (socklen_t *) 0 ) ) < 0) && (errno == EINTR) );

    if ( csock > 0 )
    {
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
        {
            unsigned char *addrp = (unsigned char *) &sockinfo.sin_addr.s_addr;
            trace_printf( "%s@%d - Accepted socket on addr=%d.%d.%d.%d,  port=%d, sock=%d\n"
                        , method_name, __LINE__
                        , addrp[0]
                        , addrp[1]
                        , addrp[2]
                        , addrp[3]
                        , (int) ntohs( sockinfo.sin_port )
                        , csock );
        }

        int nodelay = 1;
        if ( setsockopt( csock
                       , IPPROTO_TCP
                       , TCP_NODELAY
                       , (char *) &nodelay
                       , sizeof(int) ) )
        {
            char buf[MON_STRING_BUF_SIZE];
            int err = errno;
            snprintf(buf, sizeof(buf), "[%s], setsockopt() failed, errno=%d (%s).\n",
                     method_name, err, strerror(err));
            mon_log_write(MON_CLUSTER_ACCEPTSOCK_2, SQ_LOG_ERR, buf);
            return ( -2 );
        }

        int reuse = 1;
        if ( setsockopt( csock
                       , SOL_SOCKET
                       , SO_REUSEADDR
                       , (char *) &reuse
                       , sizeof(int) ) )
        {
            char buf[MON_STRING_BUF_SIZE];
            int err = errno;
            snprintf(buf, sizeof(buf), "[%s], setsockopt() failed, errno=%d (%s).\n",
                     method_name, err, strerror(err));
            mon_log_write(MON_CLUSTER_ACCEPTSOCK_3, SQ_LOG_ERR, buf);
            return ( -2 );
        }
    }

    TRACE_EXIT;
    return ( csock );
}

int CCluster::Connect( const char *portName, bool doRetries )
{
    const char method_name[] = "CCluster::Connect";
    TRACE_ENTRY;

    int  sock;      // socket
    int  ret;       // returned value
    int  nodelay = 1; // sockopt reuse option
    int  reuse = 1; // sockopt reuse option
#if defined(_XOPEN_SOURCE_EXTENDED)
#ifdef __LP64__
    socklen_t  size;    // size of socket address
#else
    size_t   size;      // size of socket address
#endif
#else
    int    size;        // size of socket address
#endif
    static int retries = 0;      // # times to retry connect
    int    outer_failures = 0;   // # failed connect loops
    int    connect_failures = 0; // # failed connects
    char   *p;     // getenv results
    struct sockaddr_in  sockinfo; // socket address info
    struct hostent *he;
    char   host[1000];
    const char *colon;
    unsigned int port;

    colon = strstr(portName, ":");
    strcpy(host, portName);
    int len = colon - portName;
    host[len] = '\0';
    port = atoi(&colon[1]);
    size = sizeof(sockinfo);

    if ( !retries )
    {
        p = getenv( "HPMP_CONNECT_RETRIES" );
        if ( p ) retries = atoi( p );
        else retries = 5;
    }

    for ( ;; )
    {
        sock = socket( AF_INET, SOCK_STREAM, 0 );
        if ( sock < 0 )
        {
            char la_buf[MON_STRING_BUF_SIZE];
            int err = errno;
            sprintf( la_buf, "[%s], socket() failed! errno=%d (%s)\n"
                   , method_name, err, strerror( err ));
            mon_log_write(MON_CLUSTER_CONNECT_1, SQ_LOG_CRIT, la_buf);
            abort();
        }

        he = gethostbyname( host );
        if ( !he )
        {
            char ebuff[256];
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf), "[%s@%d] gethostbyname(%s) error: %s\n",
                method_name, __LINE__, host, strerror_r( h_errno, ebuff, 256 ) );
            mon_log_write( MON_CLUSTER_CONNECT_2, SQ_LOG_CRIT, buf );
            abort();
        }

        // Connect socket.
        memset( (char *) &sockinfo, 0, size );
        memcpy( (char *) &sockinfo.sin_addr, (char *) he->h_addr, 4 );
        sockinfo.sin_family = AF_INET;
        sockinfo.sin_port = htons( (unsigned short) port );

        // Note the outer loop uses "retries" from HPMP_CONNECT_RETRIES,
        // and has a yield between each retry, since it's more oriented
        // toward failures from network overload and putting a pause
        // between retries.  This inner loop should only iterate when
        // a signal interrupts the local process, so it doesn't pause
        // or use the same HPMP_CONNECT_RETRIES count.
        connect_failures = 0;
        ret = 1;
        while ( ret != 0 && connect_failures <= 10 )
        {
            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                if (doRetries)
                {
                    trace_printf( "%s@%d - Connecting to %s, addr=%d.%d.%d.%d, port=%d, connect_failures=%d\n"
                                , method_name, __LINE__
                                , portName
                                , (int)((unsigned char *)he->h_addr)[0]
                                , (int)((unsigned char *)he->h_addr)[1]
                                , (int)((unsigned char *)he->h_addr)[2]
                                , (int)((unsigned char *)he->h_addr)[3]
                                , port
                                , connect_failures );
                }
                else
                {
                    trace_printf( "%s@%d - Connecting to %s, addr=%d.%d.%d.%d, port=%d\n"
                                , method_name, __LINE__
                                , portName
                                , (int)((unsigned char *)he->h_addr)[0]
                                , (int)((unsigned char *)he->h_addr)[1]
                                , (int)((unsigned char *)he->h_addr)[2]
                                , (int)((unsigned char *)he->h_addr)[3]
                                , port );
                }
            }

            ret = connect( sock, (struct sockaddr *) &sockinfo, size );
            if ( ret == 0 ) break;
            if ( errno == EINTR )
            {
                ++connect_failures;
            }
#ifdef NAMESERVER_PROCESS
            else if ( errno == ECONNREFUSED )
            {
                ++connect_failures;
                sleep( 1 );
            }
#endif
            else
            {
                char la_buf[MON_STRING_BUF_SIZE];
                int err = errno;
                sprintf( la_buf, "[%s], connect(%s) failed! errno=%d (%s)\n"
                       , method_name, portName, err, strerror( err ));
                mon_log_write(MON_CLUSTER_CONNECT_3, SQ_LOG_ERR, la_buf);
                close(sock);
                return ( -1 );
            }
        }

        if ( ret == 0 ) break;

        if (doRetries == false)
        {
            close( sock );
            return( -1 );
        }

        // For large clusters, the connect/accept calls seem to fail occasionally,
        // no doubt do to the large number (1000's) of simultaneous connect packets
        // flooding the network at once.  So, we retry up to HPMP_CONNECT_RETRIES
        // number of times.
        if ( errno != EINTR )
        {
            if ( ++outer_failures > retries )
            {
                char la_buf[MON_STRING_BUF_SIZE];
                sprintf( la_buf, "[%s], connect(%s) exceeded retries! count=%d\n"
                       , method_name, portName, retries);
                mon_log_write(MON_CLUSTER_CONNECT_4, SQ_LOG_ERR, la_buf);
                close( sock );
                return ( -1 );
            }
            struct timespec req, rem;
            req.tv_sec = 0;
            req.tv_nsec = 500000;
            nanosleep( &req, &rem );
        }
        close( sock );
    }

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        trace_printf( "%s@%d - Connected to %s addr=%d.%d.%d.%d, port=%d, sock=%d\n"
                    , method_name, __LINE__
                    , host
                    , (int)((unsigned char *)he->h_addr)[0]
                    , (int)((unsigned char *)he->h_addr)[1]
                    , (int)((unsigned char *)he->h_addr)[2]
                    , (int)((unsigned char *)he->h_addr)[3]
                    , port
                    , sock );
    }

    if ( setsockopt( sock, IPPROTO_TCP, TCP_NODELAY, (char *) &nodelay, sizeof(int) ) )
    {
        char la_buf[MON_STRING_BUF_SIZE];
        int err = errno;
        sprintf( la_buf, "[%s], setsockopt() failed! errno=%d (%s)\n"
               , method_name, err, strerror( err ));
        mon_log_write(MON_CLUSTER_CONNECT_5, SQ_LOG_ERR, la_buf);
        close( sock );
        return ( -2 );
    }

    if ( setsockopt( sock, SOL_SOCKET, SO_REUSEADDR, (char *) &reuse, sizeof(int) ) )
    {
        char la_buf[MON_STRING_BUF_SIZE];
        int err = errno;
        sprintf( la_buf, "[%s], setsockopt() failed! errno=%d (%s)\n"
               , method_name, err, strerror( err ));
        mon_log_write(MON_CLUSTER_CONNECT_6, SQ_LOG_ERR, la_buf);
        close( sock );
        return ( -2 );
    }

    TRACE_EXIT;
    return ( sock );
}

#ifdef NAMESERVER_PROCESS
void CCluster::ConnectToMon2NsCommSelf( void )
{
    const char method_name[] = "CCluster::ConnectToMon2NsCommSelf";
    TRACE_ENTRY;

    Connect( MyNode->GetMon2NsSocketPort() );

    TRACE_EXIT;
}
#else
void CCluster::ConnectToPtPCommSelf( void )
{
    const char method_name[] = "CCluster::ConnectToPtPCommSelf";
    TRACE_ENTRY;

    Connect( MyNode->GetPtPSocketPort() );

    TRACE_EXIT;
}
#endif

void CCluster::ConnectToSelf( void )
{
    const char method_name[] = "CCluster::ConnectToSelf";
    TRACE_ENTRY;

    Connect( MyNode->GetCommSocketPort() );

    TRACE_EXIT;
}

void CCluster::Connect( int socketPort )
{
    const char method_name[] = "CCluster::Connect";
    TRACE_ENTRY;

    int  sock;     // socket
    int  ret;      // returned value
#if defined(_XOPEN_SOURCE_EXTENDED)
#ifdef __LP64__
    socklen_t  size;    // size of socket address
#else
    size_t   size;      // size of socket address
#endif
#else
    int    size;        // size of socket address
#endif
    static int retries = 0;       // # times to retry connect
    int     connect_failures = 0; // # failed connects
    char   *p;     // getenv results
    struct sockaddr_in  sockinfo; // socket address info
    struct hostent *he;

    size = sizeof(sockinfo);

    if ( !retries )
    {
        p = getenv( "HPMP_CONNECT_RETRIES" );
        if ( p ) retries = atoi( p );
        else retries = 5;
    }

    sock = socket( AF_INET, SOCK_STREAM, 0 );
    if ( sock < 0 )
    {
        char la_buf[MON_STRING_BUF_SIZE];
        int err = errno;
        sprintf( la_buf, "[%s], socket() failed! errno=%d (%s)\n"
               , method_name, err, strerror( err ));
        mon_log_write(MON_CLUSTER_CONNECTTOSELF_1, SQ_LOG_CRIT, la_buf);
        MPI_Abort( MPI_COMM_SELF,99 );
    }

    he = gethostbyname( "localhost" );
    if ( !he )
    {
        char ebuff[256];
        char buf[MON_STRING_BUF_SIZE];
        snprintf( buf, sizeof(buf), "[%s@%d] gethostbyname(%s) error: %s\n",
            method_name, __LINE__, "localhost", strerror_r( h_errno, ebuff, 256 ) );
        mon_log_write( MON_CLUSTER_CONNECTTOSELF_2, SQ_LOG_CRIT, buf );
        MPI_Abort( MPI_COMM_SELF,99 );
    }

    // Connect socket.
    memset( (char *) &sockinfo, 0, size );
    memcpy( (char *) &sockinfo.sin_addr, (char *) he->h_addr, 4 );
    sockinfo.sin_family = AF_INET;
    sockinfo.sin_port = htons( (unsigned short) socketPort );

    connect_failures = 0;
    ret = 1;
    while ( ret != 0 && connect_failures <= 10 )
    {
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
        {
            trace_printf( "%s@%d - Connecting to localhost addr=%d.%d.%d.%d, port=%d, connect_failures=%d\n"
                        , method_name, __LINE__
                        , (int)((unsigned char *)he->h_addr)[0]
                        , (int)((unsigned char *)he->h_addr)[1]
                        , (int)((unsigned char *)he->h_addr)[2]
                        , (int)((unsigned char *)he->h_addr)[3]
                        , socketPort
                        , connect_failures );
        }

        ret = connect( sock, (struct sockaddr *) &sockinfo, size );
        if ( ret == 0 ) break;
        if ( errno == EINTR )
        {
            ++connect_failures;
        }
        else
        {
            char la_buf[MON_STRING_BUF_SIZE];
            int err = errno;
            sprintf( la_buf, "[%s], connect() failed! errno=%d (%s)\n"
                   , method_name, err, strerror( err ));
            mon_log_write(MON_CLUSTER_CONNECTTOSELF_3, SQ_LOG_CRIT, la_buf);
            MPI_Abort( MPI_COMM_SELF,99 );
        }
    }

    close( sock );

    TRACE_EXIT;
}

int CCluster::MkSrvSock( int *pport )
{
    const char method_name[] = "CCluster::MkSrvSock";
    TRACE_ENTRY;

    int  sock;     // socket
    int  err;      // return code
#if defined(_XOPEN_SOURCE_EXTENDED)
#ifdef __LP64__
    socklen_t size; // size of socket address
#else
    size_t    size; // size of socket address
#endif
#else
    unsigned int size; // size of socket address
#endif
    struct sockaddr_in  sockinfo;   // socket address info
    sock = socket( AF_INET, SOCK_STREAM, 0 );
    if ( sock < 0 )
    {
        char la_buf[MON_STRING_BUF_SIZE];
        int err = errno;
        sprintf( la_buf, "[%s], socket() failed! errno=%d (%s)\n"
               , method_name, err, strerror( err ));
        mon_log_write(MON_CLUSTER_MKSRVSOCK_1, SQ_LOG_CRIT, la_buf);
        return ( -1 );
    }

    int    nodelay = 1;   // sockopt nodelay option
    if ( setsockopt( sock, IPPROTO_TCP, TCP_NODELAY, (char *) &nodelay, sizeof(int) ) )
    {
        char la_buf[MON_STRING_BUF_SIZE];
        int err = errno;
        sprintf( la_buf, "[%s], setsockopt() failed! errno=%d (%s)\n"
               , method_name, err, strerror( err ));
        mon_log_write(MON_CLUSTER_MKSRVSOCK_2, SQ_LOG_ERR, la_buf);
        close( sock );
        return ( -2 );
    }

    int    reuse = 1;   // sockopt reuse option
    if ( setsockopt( sock, SOL_SOCKET, SO_REUSEADDR, (char *) &reuse, sizeof(int) ) )
    {
        char la_buf[MON_STRING_BUF_SIZE];
        int err = errno;
        sprintf( la_buf, "[%s], setsockopt(SO_REUSEADDR) failed! errno=%d (%s)\n"
               , method_name, err, strerror( err ));
        mon_log_write(MON_CLUSTER_MKSRVSOCK_3, SQ_LOG_ERR, la_buf);
        close( sock );
        return ( -1 );
    }

    // Bind socket.
    size = sizeof(sockinfo);
    memset( (char *) &sockinfo, 0, size );
    sockinfo.sin_family = AF_INET;
    sockinfo.sin_addr.s_addr = htonl( INADDR_ANY );
    sockinfo.sin_port = htons( *pport );
    int lv_bind_tries = 0;
    do
    {
        if (lv_bind_tries > 0)
        {
            sleep(5);
        }
        err = bind( sock, (struct sockaddr *) &sockinfo, size );
        sched_yield( );
    } while ( err &&
             (errno == EADDRINUSE) &&
             (++lv_bind_tries < 4) );
    if ( err )
    {
        char la_buf[MON_STRING_BUF_SIZE];
        int err = errno;
        sprintf( la_buf, "[%s], bind() failed! port=%d, errno=%d (%s)\n"
               , method_name, *pport, err, strerror( err ));
        mon_log_write(MON_CLUSTER_MKSRVSOCK_4, SQ_LOG_CRIT, la_buf);
        close( sock );
        return ( -1 );
    }
    if ( pport )
    {
        if ( getsockname( sock, (struct sockaddr *) &sockinfo, &size ) )
        {
            char la_buf[MON_STRING_BUF_SIZE];
            int err = errno;
            sprintf( la_buf, "[%s], getsockname() failed! errno=%d (%s)\n"
                   , method_name, err, strerror( err ));
            mon_log_write(MON_CLUSTER_MKSRVSOCK_5, SQ_LOG_CRIT, la_buf);
            close( sock );
            return ( -1 );
        }

        *pport = (int) ntohs( sockinfo.sin_port );
    }
    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        unsigned char *addrp = (unsigned char *) &sockinfo.sin_addr.s_addr;
        trace_printf( "%s@%d listening on addr=%d.%d.%d.%d, port=%d\n"
                    , method_name, __LINE__
                    , addrp[0]
                    , addrp[1]
                    , addrp[2]
                    , addrp[3]
                    , pport?*pport:0);
    }

    int lv_retcode = SetKeepAliveSockOpt( sock );
    if ( lv_retcode != 0 )
    {
        return lv_retcode;
    }

    // Listen
    if ( listen( sock, SOMAXCONN ) )
    {
        char la_buf[MON_STRING_BUF_SIZE];
        int err = errno;
        sprintf( la_buf, "[%s], listen() failed! errno=%d (%s)\n"
               , method_name, err, strerror( err ));
        mon_log_write(MON_CLUSTER_MKSRVSOCK_6, SQ_LOG_CRIT, la_buf);
        close( sock );
        return ( -1 );
    }
    TRACE_EXIT;
    return ( sock );
}

int CCluster::MkCltSock( const char *portName )
{
    const char method_name[] = "CCluster::MkCltSock1";
    TRACE_ENTRY;

    int    sock;        // socket
    int    ret;         // returned value
    int    reuse = 1;   // sockopt reuse option
    int    nodelay = 1; // sockopt nodelay option
#if defined(_XOPEN_SOURCE_EXTENDED)
#ifdef __LP64__
    socklen_t  size;    // size of socket address
#else
    size_t   size;      // size of socket address
#endif
#else
    int    size;        // size of socket address
#endif
    static int retries = 0;      // # times to retry connect
    int    outer_failures = 0;   // # failed connect loops
    int    connect_failures = 0; // # failed connects
    char   *p;     // getenv results
    struct sockaddr_in  sockinfo;    // socket address info
    struct hostent *he;
    char   host[1000];
    const char *colon;
    unsigned int port;

    colon = strstr(portName, ":");
    strcpy(host, portName);
    int len = colon - portName;
    host[len] = '\0';
    port = atoi(&colon[1]);

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
        {
            trace_printf( "%s@%d - Connecting to %s:%d\n"
                        , method_name, __LINE__
                        , host
                        , port );
        }
    }

    size = sizeof(sockinfo);

    if ( !retries )
    {
        p = getenv( "HPMP_CONNECT_RETRIES" );
        if ( p ) retries = atoi( p );
        else retries = 5;
    }

    for ( ;; )
    {
        sock = socket( AF_INET, SOCK_STREAM, 0 );
        if ( sock < 0 )
        {
            char la_buf[MON_STRING_BUF_SIZE];
            int err = errno;
            snprintf( la_buf, sizeof(la_buf)
                    , "[%s], socket() failed! errno=%d (%s)\n"
                    , method_name, err, strerror( err ));
            mon_log_write(MON_CLUSTER_MKCLTSOCK_1, SQ_LOG_ERR, la_buf);
            return ( -1 );
        }

        he = gethostbyname( host );
        if ( !he )
        {
            char la_buf[MON_STRING_BUF_SIZE];
            int err = h_errno;
            snprintf( la_buf, sizeof(la_buf),
                      "[%s] gethostbyname(%s) failed! errno=%d (%s)\n"
                    , method_name, host, err, strerror( err ));
            mon_log_write(MON_CLUSTER_MKCLTSOCK_2, SQ_LOG_ERR, la_buf);
            close( sock );
            return ( -1 );
        }

        // Connect socket.
        memset( (char *) &sockinfo, 0, size );
        memcpy( (char *) &sockinfo.sin_addr, (char *) he->h_addr, 4 );
        sockinfo.sin_family = AF_INET;
        sockinfo.sin_port = htons( (unsigned short) port );

        // Note the outer loop uses "retries" from HPMP_CONNECT_RETRIES,
        // and has a yield between each retry, since it's more oriented
        // toward failures from network overload and putting a pause
        // between retries.  This inner loop should only iterate when
        // a signal interrupts the local process, so it doesn't pause
        // or use the same HPMP_CONNECT_RETRIES count.
        connect_failures = 0;
        ret = 1;
        while ( ret != 0 && connect_failures <= 10 )
        {
            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                trace_printf( "%s@%d - Connecting to %s addr=%d.%d.%d.%d, port=%d, connect_failures=%d\n"
                            , method_name, __LINE__
                            , host
                            , (int)((unsigned char *)he->h_addr)[0]
                            , (int)((unsigned char *)he->h_addr)[1]
                            , (int)((unsigned char *)he->h_addr)[2]
                            , (int)((unsigned char *)he->h_addr)[3]
                            , port
                            , connect_failures );
            }

            ret = connect( sock, (struct sockaddr *) &sockinfo, size );
            if ( ret == 0 ) break;
            if ( errno == EINTR )
            {
                ++connect_failures;
            }
            else
            {
                char la_buf[MON_STRING_BUF_SIZE];
                int err = errno;
                sprintf( la_buf, "[%s], connect() failed! errno=%d (%s)\n"
                       , method_name, err, strerror( err ));
                mon_log_write(MON_CLUSTER_MKCLTSOCK_3, SQ_LOG_ERR, la_buf);
                close(sock);
                return ( -1 );
            }
        }

        if ( ret == 0 ) break;

        // For large clusters, the connect/accept calls seem to fail occasionally,
        // no doubt do to the large number (1000's) of simultaneous connect packets
        // flooding the network at once.  So, we retry up to HPMP_CONNECT_RETRIES
        // number of times.
        if ( errno != EINTR )
        {
            if ( ++outer_failures > retries )
            {
                char la_buf[MON_STRING_BUF_SIZE];
                sprintf( la_buf, "[%s], connect() exceeded retries! count=%d\n"
                       , method_name, retries);
                mon_log_write(MON_CLUSTER_MKCLTSOCK_4, SQ_LOG_ERR, la_buf);
                close( sock );
                return ( -1 );
            }
            struct timespec req, rem;
            req.tv_sec = 0;
            req.tv_nsec = 500000;
            nanosleep( &req, &rem );
        }
        close( sock );
    }

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
    {
        trace_printf( "%s@%d - Connected to %s addr=%d.%d.%d.%d, port=%d, sock=%d\n"
                    , method_name, __LINE__
                    , host
                    , (int)((unsigned char *)he->h_addr)[0]
                    , (int)((unsigned char *)he->h_addr)[1]
                    , (int)((unsigned char *)he->h_addr)[2]
                    , (int)((unsigned char *)he->h_addr)[3]
                    , port
                    , sock );
    }

    if ( setsockopt( sock, IPPROTO_TCP, TCP_NODELAY, (char *) &nodelay, sizeof(int) ) )
    {
        char la_buf[MON_STRING_BUF_SIZE];
        int err = errno;
        sprintf( la_buf, "[%s], setsockopt() failed! errno=%d (%s)\n"
               , method_name, err, strerror( err ));
        mon_log_write(MON_CLUSTER_MKCLTSOCK_5, SQ_LOG_ERR, la_buf);
        close( sock );
        return ( -2 );
    }

    if ( setsockopt( sock, SOL_SOCKET, SO_REUSEADDR, (char *) &reuse, sizeof(int) ) )
    {
        char la_buf[MON_STRING_BUF_SIZE];
        int err = errno;
        sprintf( la_buf, "[%s], setsockopt() failed! errno=%d (%s)\n"
               , method_name, err, strerror( err ));
        mon_log_write(MON_CLUSTER_MKCLTSOCK_6, SQ_LOG_ERR, la_buf);
        close( sock );
        return ( -2 );
    }

    TRACE_EXIT;
    return ( sock );
}

int CCluster::SetKeepAliveSockOpt( int sock )
{
    const char method_name[] = "CCluster::SetKeepAliveSockOpt";
    TRACE_ENTRY;

    static int sv_keepalive = -1;
    static int sv_keepidle  = 120;
    static int sv_keepintvl = 12;
    static int sv_keepcnt   = 5;

    if ( sv_keepalive == -1 )
    {
        char *lv_keepalive_env = getenv( "SQ_MON_KEEPALIVE" );
        if ( lv_keepalive_env )
        {
            sv_keepalive = atoi( lv_keepalive_env );
        }
        if ( sv_keepalive == 1 )
        {
            char *lv_keepidle_env = getenv( "SQ_MON_KEEPIDLE" );
            if ( lv_keepidle_env )
            {
                sv_keepidle = atoi( lv_keepidle_env );
            }
            char *lv_keepintvl_env = getenv( "SQ_MON_KEEPINTVL" );
            if ( lv_keepintvl_env )
            {
                sv_keepintvl = atoi( lv_keepintvl_env );
            }
            char *lv_keepcnt_env = getenv( "SQ_MON_KEEPCNT" );
            if ( lv_keepcnt_env )
            {
                sv_keepcnt = atoi( lv_keepcnt_env );
            }
        }
    }

    if ( sv_keepalive == 1 )
    {
        if ( setsockopt( sock, SOL_SOCKET, SO_KEEPALIVE, &sv_keepalive, sizeof(int) ) )
        {
            char la_buf[MON_STRING_BUF_SIZE];
            int err = errno;
            sprintf( la_buf, "[%s], setsockopt so_keepalive() failed! errno=%d (%s)\n"
                   , method_name, err, strerror( err ) );
            mon_log_write( MON_CLUSTER_SETKEEPALIVESOCKOPT_1, SQ_LOG_ERR, la_buf );
            close( sock );
            return ( -2 );
        }

        if ( setsockopt( sock, SOL_TCP, TCP_KEEPIDLE, &sv_keepidle, sizeof(int) ) )
        {
            char la_buf[MON_STRING_BUF_SIZE];
            int err = errno;
            sprintf( la_buf, "[%s], setsockopt tcp_keepidle() failed! errno=%d (%s)\n"
                   , method_name, err, strerror( err ) );
            mon_log_write( MON_CLUSTER_SETKEEPALIVESOCKOPT_2, SQ_LOG_ERR, la_buf );
            close( sock );
            return ( -2 );
        }

        if ( setsockopt( sock, SOL_TCP, TCP_KEEPINTVL, &sv_keepintvl, sizeof(int) ) )
        {
            char la_buf[MON_STRING_BUF_SIZE];
            int err = errno;
            sprintf( la_buf, "[%s], setsockopt tcp_keepintvl() failed! errno=%d (%s)\n"
                   , method_name, err, strerror( err ) );
            mon_log_write( MON_CLUSTER_SETKEEPALIVESOCKOPT_3, SQ_LOG_ERR, la_buf );
            close( sock );
            return ( -2 );
        }

        if ( setsockopt( sock, SOL_TCP, TCP_KEEPCNT, &sv_keepcnt, sizeof(int) ) )
        {
            char la_buf[MON_STRING_BUF_SIZE];
            int err = errno;
            sprintf( la_buf, "[%s], setsockopt tcp_keepcnt() failed! errno=%d (%s)\n"
                   , method_name, err, strerror( err ) );
            mon_log_write( MON_CLUSTER_SETKEEPALIVESOCKOPT_4, SQ_LOG_ERR, la_buf );
            close( sock );
            return ( -2 );
        }
    }

    TRACE_EXIT;
    return ( 0 );
}

int CCluster::MkCltSock( unsigned char srcip[4], unsigned char dstip[4], int port )
{
    const char method_name[] = "CCluster::MkCltSock2";
    TRACE_ENTRY;

    int    sock;        // socket
    int    ret;         // returned value
    int    reuse = 1;   // sockopt reuse option
    int    nodelay = 1; // sockopt nodelay option
#if defined(_XOPEN_SOURCE_EXTENDED)
#ifdef __LP64__
    socklen_t  size;    // size of socket address
#else
    size_t   size;      // size of socket address
#endif
#else
    int    size;        // size of socket address
#endif
    static int retries = 0;      // # times to retry connect
    int    outer_failures = 0;   // # failed connect loops
    int    connect_failures = 0; // # failed connects
    char   *p;     // getenv results
    struct sockaddr_in  sockinfo;    // socket address info

    size = sizeof(sockinfo);

    if ( !retries )
    {
        p = getenv( "HPMP_CONNECT_RETRIES" );
        if ( p ) retries = atoi( p );
        else retries = 5;
    }

    for ( ;; )
    {
        sock = socket( AF_INET, SOCK_STREAM, 0 );
        if ( sock < 0 ) return ( -1 );

        // Bind local address if specified.
        if ( srcip )
        {
            memset( (char *) &sockinfo, 0, size );
            memcpy( (char *) &sockinfo.sin_addr,
                (char *) srcip, sizeof(srcip) );
            sockinfo.sin_family = AF_INET;
            sockinfo.sin_port = 0;
            if ( bind( sock, (struct sockaddr *) &sockinfo, size ) )
            {
                char la_buf[MON_STRING_BUF_SIZE];
                int err = errno;
                sprintf( la_buf, "[%s], bind() failed! errno=%d (%s)\n"
                       , method_name, err, strerror( err ));
                mon_log_write(MON_CLUSTER_MKCLTSOCK_7, SQ_LOG_ERR, la_buf);
                close( sock );
                return ( -1 );
            }
        }

        // Connect socket.
        memset( (char *) &sockinfo, 0, size );
        memcpy( (char *) &sockinfo.sin_addr, (char *) dstip, 4 );
        sockinfo.sin_family = AF_INET;
        sockinfo.sin_port = htons( (unsigned short) port );

        // Note the outer loop uses "retries" from HPMP_CONNECT_RETRIES,
        // and has a yield between each retry, since it's more oriented
        // toward failures from network overload and putting a pause
        // between retries.  This inner loop should only iterate when
        // a signal interrupts the local process, so it doesn't pause
        // or use the same HPMP_CONNECT_RETRIES count.
        connect_failures = 0;
        ret = 1;
        while ( ret != 0 && connect_failures <= 10 )
        {
            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            {
                trace_printf( "%s@%d - Connecting to addr=%d.%d.%d.%d, port=%d, connect_failures=%d\n"
                            , method_name, __LINE__
                            , (int)dstip[0]
                            , (int)dstip[1]
                            , (int)dstip[2]
                            , (int)dstip[3]
                            , port
                            , connect_failures );
            }
            ret = connect( sock, (struct sockaddr *) &sockinfo,
                size );
            if ( ret == 0 ) break;
            if ( errno == EINTR )
            {
                ++connect_failures;
            }
#ifdef NAMESERVER_PROCESS
            else if ( errno == ECONNREFUSED )
            {
                ++connect_failures;
                sleep( 1 );
            }
#endif
            else
            {
                char la_buf[MON_STRING_BUF_SIZE];
                int err = errno;
                sprintf( la_buf, "[%s], connect(%d.%d.%d.%d:%d) failed! errno=%d (%s)\n"
                       , method_name
                       , (int)((unsigned char *)dstip)[0]
                       , (int)((unsigned char *)dstip)[1]
                       , (int)((unsigned char *)dstip)[2]
                       , (int)((unsigned char *)dstip)[3]
                       , port
                       , err, strerror( err ));
                mon_log_write(MON_CLUSTER_MKCLTSOCK_8, SQ_LOG_ERR, la_buf);
                close(sock);
                return ( -1 );
            }
        }

        if ( ret == 0 ) break;

        // For large clusters, the connect/accept calls seem to fail occasionally,
        // no doubt do to the large number (1000's) of simultaneous connect packets
        // flooding the network at once.  So, we retry up to HPMP_CONNECT_RETRIES
        // number of times.
        if ( errno != EINTR )
        {
            if ( ++outer_failures > retries )
            {
                char la_buf[MON_STRING_BUF_SIZE];
                sprintf( la_buf, "[%s], connect() exceeded retries! count=%d\n"
                       , method_name, retries);
                mon_log_write(MON_CLUSTER_MKCLTSOCK_9, SQ_LOG_ERR, la_buf);
                close( sock );
                return ( -1 );
            }
            struct timespec req, rem;
            req.tv_sec = 0;
            req.tv_nsec = 500000;
            nanosleep( &req, &rem );
        }
        close( sock );
    }

    if ( setsockopt( sock, IPPROTO_TCP, TCP_NODELAY, (char *) &nodelay, sizeof(int) ) )
    {
        char la_buf[MON_STRING_BUF_SIZE];
        int err = errno;
        sprintf( la_buf, "[%s], setsockopt() failed! errno=%d (%s)\n"
               , method_name, err, strerror( err ));
        mon_log_write(MON_CLUSTER_MKCLTSOCK_10, SQ_LOG_ERR, la_buf);
        close( sock );
        return ( -2 );
    }

    if ( setsockopt( sock, SOL_SOCKET, SO_REUSEADDR, (char *) &reuse, sizeof(int) ) )
    {
        char la_buf[MON_STRING_BUF_SIZE];
        int err = errno;
        sprintf( la_buf, "[%s], setsockopt() failed! errno=%d (%s)\n"
               , method_name, err, strerror( err ));
        mon_log_write(MON_CLUSTER_MKCLTSOCK_11, SQ_LOG_ERR, la_buf);
        close( sock );
        return ( -2 );
    }

    int lv_retcode = SetKeepAliveSockOpt( sock );
    if ( lv_retcode != 0 )
    {
        return lv_retcode;
    }

    TRACE_EXIT;
    return ( sock );
}

int CCluster::ReceiveMPI(char *buf, int size, int source, MonXChngTags tag, MPI_Comm comm)
{
    const char method_name[] = "CCluster::ReceiveMPI";
    TRACE_ENTRY;

    MPI_Request request;
    MPI_Status status;
    int received = 0;

    int error = MPI_Irecv(buf, size, MPI_CHAR, source, tag, comm, &request);

    if (trace_settings & (TRACE_REQUEST | TRACE_INIT | TRACE_RECOVERY))
        trace_printf("%s@%d - Msg Received. Error = %d\n", method_name, __LINE__, error);

    if (!error)
    {
        while (!received)
        {
            error = MPI_Test(&request, &received, &status);

            if (!error)
            {
                if (trace_settings & (TRACE_REQUEST | TRACE_INIT | TRACE_RECOVERY))
                    trace_printf("%s@%d - Msg Received Test. Flag = %d\n", method_name, __LINE__, received);
            }
            else
            {
                usleep(10000); // sleep 10ms and try again
            }
         }
    }

    TRACE_EXIT;
    return error;
}

int CCluster::SendMPI(char *buf, int size, int source, MonXChngTags tag, MPI_Comm comm)
{
    const char method_name[] = "CCluster::SendMPI";
    TRACE_ENTRY;

    MPI_Request request;
    MPI_Status status;
    int sent = 0;

    int error = MPI_Isend(buf, size, MPI_CHAR, source, tag, comm, &request);

    if (trace_settings & (TRACE_REQUEST | TRACE_INIT | TRACE_RECOVERY))
        trace_printf("%s@%d - Msg Sent. Error = %d\n", method_name, __LINE__, error);

    if (!error)
    {
        while (!sent)
        {
            error = MPI_Test(&request, &sent, &status);

            if (!error)
            {
                if (trace_settings & (TRACE_REQUEST | TRACE_INIT | TRACE_RECOVERY))
                    trace_printf("%s@%d - Msg Sent Test. Flag = %d\n", method_name, __LINE__, sent);
            }
            else
            {
                usleep(10000); // sleep 10ms and try again
            }
         }
    }

    TRACE_EXIT;
    return error;
}

int CCluster::ReceiveSock(char *buf, int size, int sockFd, const char *desc)
{
    const char method_name[] = "CCluster::ReceiveSock";
    TRACE_ENTRY;

    bool    readAgain = false;
    int     error = 0;
    int     readCount = 0;
    int     received = 0;
    int     sizeCount = size;

    do
    {
        readCount = (int) recv( sockFd
                              , buf
                              , sizeCount
                              , 0 );
        if ( readCount > 0 ) Meas.addSockRcvdBytes( readCount );

        if (trace_settings & (TRACE_REQUEST | TRACE_INIT | TRACE_RECOVERY))
        {
            trace_printf( "%s@%d - recv(%d), sock=%d, readCount=%d, desc=%s\n"
                        , method_name, __LINE__
                        , sizeCount
                        , sockFd
                        , readCount
                        , desc );
        }

        if ( readCount > 0 )
        { // Got data
            received += readCount;
            buf += readCount;
            if ( received == size )
            {
                readAgain = false;
            }
            else
            {
                sizeCount -= readCount;
                readAgain = true;
            }
        }
        else if ( readCount == 0 )
        { // EOF
             error = ENODATA;
             readAgain = false;
        }
        else
        { // Got an error
            if ( errno != EINTR)
            {
                error = errno;
                readAgain = false;
            }
            else
            {
                readAgain = true;
            }
        }
    }
    while( readAgain );

    if (trace_settings & (TRACE_REQUEST | TRACE_INIT | TRACE_RECOVERY))
    {
        trace_printf( "%s@%d - recv(), received=%d, sock=%d, error=%d(%s), desc=%s\n"
                    , method_name, __LINE__
                    , received
                    , sockFd
                    , error, strerror(error)
                    , desc );
    }

    TRACE_EXIT;
    return error;
}

int CCluster::SendSock(char *buf, int size, int sockFd, const char *desc)
{
    const char method_name[] = "CCluster::SendSock";
    TRACE_ENTRY;

    bool    sendAgain = false;
    int     error = 0;
    int     sendCount = 0;
    int     sent = 0;

    do
    {
        sendCount = (int) send( sockFd
                              , buf
                              , size
                              , 0 );
        if ( sendCount > 0 ) Meas.addSockSentBytes( sendCount );

        if (trace_settings & (TRACE_REQUEST | TRACE_INIT | TRACE_RECOVERY))
        {
            trace_printf( "%s@%d - send(), sock=%d, sendCount=%d, desc=%s\n"
                        , method_name, __LINE__
                        , sockFd
                        , sendCount
                        , desc );
        }

        if ( sendCount > 0 )
        { // Sent data
            sent += sendCount;
            if ( sendCount == size )
            {
                 sendAgain = false;
            }
            else
            {
                sendAgain = true;
            }
        }
        else
        { // Got an error
            if ( errno != EINTR)
            {
                error = errno;
                sendAgain = false;
            }
            else
            {
                sendAgain = true;
            }
        }
    }
    while( sendAgain );

    if (trace_settings & (TRACE_REQUEST | TRACE_INIT | TRACE_RECOVERY))
    {
        trace_printf( "%s@%d - send(), sent=%d, sock=%d, error=%d(%s), desc=%s\n"
                    , method_name, __LINE__
                    , sent
                    , sockFd
                    , error, strerror(error)
                    , desc );
    }

    TRACE_EXIT;
    return error;
}