///////////////////////////////////////////////////////////////////////////////
//
// @@@ START COPYRIGHT @@@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
//
// @@@ END COPYRIGHT @@@
//
///////////////////////////////////////////////////////////////////////////////

#include <stdlib.h>
#include <errno.h>
#include <stdio.h>
#include <fcntl.h>
#include "/usr/include/linux/watchdog.h"
#include <sys/ioctl.h>
#include <sys/time.h>
#include <string.h>
#include <iostream>
#include <sys/types.h>
#include <unistd.h>

using namespace std;

#include "monlogging.h"
#include "monsonar.h"
#include "montrace.h"
#include "redirector.h"
#include "monitor.h"
#include "clusterconf.h"
#include "device.h"
#include "lock.h"
#include "lnode.h"
#include "pnode.h"
#include "mlio.h"
#include "nameserver.h"

#include "replicate.h"
#include "reqqueue.h"
#include "healthcheck.h"
#ifndef NAMESERVER_PROCESS
#include "ptpclient.h"
#endif

extern CReqQueue ReqQueue;
extern char MyPath[MAX_PROCESS_PATH];
extern int MyPNID;
extern bool IsRealCluster;
extern bool SpareNodeColdStandby;
extern bool Emulate_Down;

extern CConfigContainer *Config;
extern CMonitor *Monitor;
extern CNodeContainer *Nodes;
#ifndef NAMESERVER_PROCESS
extern CDeviceContainer *Devices;
#endif
extern CNode *MyNode;
extern CMonStats *MonStats;
#ifndef NAMESERVER_PROCESS
extern CRedirector Redirector;
#endif
extern CReplicate Replicator;
extern CHealthCheck HealthCheck;
extern CMonTrace *MonTrace;
extern bool IsAgentMode;
extern bool IAmIntegrating;
extern char Node_name[MPI_MAX_PROCESSOR_NAME];
extern CClusterConfig *ClusterConfig;

const char *StateString( STATE state);
#ifndef NAMESERVER_PROCESS
const char *SyncStateString( SyncState state);
extern CPtpClient *PtpClient;
extern CNameServer *NameServer;
extern CProcess *NameServerProcess;
extern bool NameServerEnabled;
#endif
extern CNameServerConfigContainer *NameServerConfig;

// The following defines are necessary for the new watchdog timer facility.  They should really be
// ultimately placed in watchdog.h in my opinion, especially so people know not to re-use values 16,17
// as they are specified for our higher resolution timer values.   These defines are used as parameters 
// to the ioctl calls.   Please not that the _IO[W]R_BAD  versions need to be used rather than the 
// mostly equivalent _IO[W]R version due to the fact that the latter version has some compile time parameter
// validation checking that is not liked by the INTEL compiler.   The BAD version avoids the check.  I've
// confirmed that the values passed in to satisfy the necessary space limit criteria for passing the test.
// They use GCC which does not experience the same compiler issue.

#define WATCHDOG_IOCTL_BASE     'W'

#define WDIOC_SQ_GETSUPPORT        _IOR(WATCHDOG_IOCTL_BASE, 0, struct watchdog_info)
#define WDIOC_SQ_GETSTATUS         _IOR(WATCHDOG_IOCTL_BASE, 1, int)
#define WDIOC_SQ_GETBOOTSTATUS     _IOR(WATCHDOG_IOCTL_BASE, 2, int)
#define WDIOC_SQ_GETTEMP           _IOR(WATCHDOG_IOCTL_BASE, 3, int)
#define WDIOC_SQ_SETOPTIONS        _IOR(WATCHDOG_IOCTL_BASE, 4, int)
#define WDIOC_SQ_KEEPALIVE         _IOR(WATCHDOG_IOCTL_BASE, 5, int)
#define WDIOC_SQ_SETTIMEOUT        _IOWR(WATCHDOG_IOCTL_BASE, 6, int)
#define WDIOC_SQ_GETTIMEOUT        _IOR(WATCHDOG_IOCTL_BASE, 7, int)


// The following defines specify the default values for the HA
// timers if the timer related environment variables are not defined.
// Defaults to 60 second Watchdog process timer expiration
#define WDT_KeepAliveTimerDefault 60

// Default interval used by GetSchedulingData (in milliseconds)
unsigned long int CNode::minSchedDataInterval_ = 500;

// Strings for use in obtaining /proc/meminfo data (see
// CNode::GetSchedulingData).  Spelling is important.  Must match
// items in /proc/meminfo.
const char * CNode::memInfoString_[] = {
    "MemTotal:",
    "MemFree:",
    "Buffers:",
    "Cached:",
    "SwapFree:",
    "Active:",
    "Inactive:",
    "Dirty:",
    "Writeback:",
    "VmallocUsed:"
};

size_t CNode::memInfoStringLen_[memFinalItem];


CNode::CNode( char *name, int pnid, int rank )
      :CLNodeContainer(this)
      ,CProcessContainer(true)
      ,pnid_(pnid)
      ,changeState_(false)
      ,numCores_(0)
      ,freeCache_(0)
      ,state_(rank == -1 ? State_Down : State_Up)
      ,phase_(Phase_Ready)
      ,killingNode_(false)
      ,dtmAborted_(false)
      ,smsAborted_(false)
      ,lastLNode_(NULL)
      ,lastSdLevel_(ShutdownLevel_Undefined)
      ,rankFailure_(false)
      ,creator_(false)
      ,creatorPid_(-1)
      ,creatorVerifier_(-1)
      ,joiningPhase_(JoiningPhase_Unknown)
      ,activatingSpare_(false)
      ,spareNode_(false)
      ,next_(NULL)
      ,prev_(NULL)
      ,rank_(rank)
#ifndef NAMESERVER_PROCESS
      ,tmSyncNid_(-1)
      ,tmSyncState_(SyncState_Null)
#endif
      ,shutdownLevel_(ShutdownLevel_Undefined)
      ,shutdownNameServer_(false)
      ,wdtKeepAliveTimerValue_(WDT_KeepAliveTimerDefault)
      ,zid_(pnid)
      ,commPort_("")
      ,syncPort_("")
#ifdef NAMESERVER_PROCESS
      ,mon2NsPort_("")
      ,mon2NsSocketPort_(-1)
      ,monConnCount_(0)
#else
      ,ptpPort_("")
      ,ptpSocketPort_(-1)
#endif
      ,commSocketPort_(-1)
      ,syncSocketPort_(-1)
      ,uniqStrId_(-1)
      ,procStatFile_(NULL)
      ,procMeminfoFile_(-1)
{
    const char method_name[] = "CNode::CNode";
    TRACE_ENTRY;

    // Add eyecatcher sequence as a debugging aid
    memcpy(&eyecatcher_, "PNOD", 4);

    char *p = getenv("SQ_SCHED_DATA_INTERVAL");
    if (p != NULL)
    {
        minSchedDataInterval_ = atoi(p);
    }
    if (trace_settings & TRACE_INIT)
    {
        trace_printf("%s@%d minSchedDataInterval_=%ld\n", method_name,
                     __LINE__, minSchedDataInterval_);
    }

    prevSchedData_.tv_sec = 0;
    prevSchedData_.tv_nsec = 0;

    STRCPY(name_, name);
    
    hostname_ = name;
    size_t pos = hostname_.find_first_of( ".:" );
    if ( pos != hostname_.npos )
    {
        hostname_.erase( hostname_.begin()+pos, hostname_.end() );
    }

    for (unsigned int i=0; i<memFinalItem; i++)
    {
        memInfoStringLen_[i] = strlen ( memInfoString_[i] );
    }

    if (pnid_ == MyPNID)
    {
        GetSchedulingData();
#ifdef HARD_AFFINITY
        int i;
        cpu_set_t mask;

        sched_getaffinity(getpid(), sizeof(affinityMask_), &affinityMask_);
        for (i=0; i < numCores_; i++)
        {
            if (CPU_ISSET(i, &affinityMask_))
            {
                if (trace_settings & TRACE_INIT)
                    trace_printf("%s@%d" " - Processor core #" "%d" " is usable" "\n", method_name, __LINE__, i);
            }
            else
            {
                if (trace_settings & TRACE_INIT)
                    trace_printf("%s@%d" " - Processor core #" "%d" " is not activated" "\n", method_name, __LINE__, i);
            }
        }
        if ( !IsRealCluster )
        {
            // set the monitor's hard affinity to a processor core
            CPU_ZERO(&mask);
            CPU_SET(MyPNID%numCores_, &mask);
            sched_setaffinity(getpid(), sizeof(mask), &mask);
        }
#else
        sched_getaffinity(getpid(), sizeof(affinityMask_), &affinityMask_);
#endif
    }
    
    CPU_ZERO( &excludedCoreMask_ );

    if (trace_settings & (TRACE_PROCESS | TRACE_PROCESS_DETAIL))
    {   // Display pidMap location, useful for understanding later trace output
        trace_printf("%s@%d PNid %d, pidMap_ (%p)\n",
                     method_name, __LINE__, GetPNid(), GetPidMap());
    }

    gettimeofday(&todStart_, NULL);

#ifndef NAMESERVER_PROCESS
    quiesceSendPids_ = new SQ_LocalIOToClient::bcastPids_t;
    quiesceExitPids_ = new SQ_LocalIOToClient::bcastPids_t;
#endif
    internalState_ = State_Default; 

    uniqStrId_ = Config->getMaxUniqueId ( pnid_ ) + 1;

    TRACE_EXIT;
}

CNode::CNode( char *name
            , int   pnid
            , int   rank
            , int   sparePNidCount
            , int   sparePNids[]
            , cpu_set_t &excludedCoreMask )
      :CLNodeContainer(this)
      ,CProcessContainer(true)
      ,pnid_(pnid)
      ,changeState_(false)
      ,numCores_(0)
      ,freeCache_(0)
      ,state_(rank == -1 ? State_Down : State_Up)
      ,phase_(Phase_Ready)
      ,killingNode_(false)
      ,dtmAborted_(false)
      ,smsAborted_(false)
      ,lastLNode_(NULL)
      ,lastSdLevel_(ShutdownLevel_Undefined)
      ,excludedCoreMask_(excludedCoreMask)
      ,rankFailure_(false)
      ,creator_(false)
      ,creatorPid_(-1)
      ,creatorVerifier_(-1)
      ,joiningPhase_(JoiningPhase_Unknown)
      ,activatingSpare_(false)
      ,spareNode_(true)
      ,next_(NULL)
      ,prev_(NULL)
      ,rank_(rank)
      ,tmSyncNid_(-1)
      ,tmSyncState_(SyncState_Suspended)
      ,shutdownLevel_(ShutdownLevel_Undefined)
      ,shutdownNameServer_(false)
      ,wdtKeepAliveTimerValue_(WDT_KeepAliveTimerDefault)
      ,zid_(-1)
      ,commPort_("")
      ,syncPort_("")
#ifdef NAMESERVER_PROCESS
      ,mon2NsPort_("")
      ,mon2NsSocketPort_(-1)
      ,monConnCount_(-1)
#else
      ,ptpPort_("")
      ,ptpSocketPort_(-1)
#endif
      ,commSocketPort_(-1)
      ,syncSocketPort_(-1)
      ,uniqStrId_(-1)
      ,procStatFile_(NULL)
      ,procMeminfoFile_(-1)
{
    const char method_name[] = "CNode::CNode";
    TRACE_ENTRY;

    // Add eyecatcher sequence as a debugging aid
    memcpy(&eyecatcher_, "PNOD", 4);

    STRCPY(name_, name);
    
    hostname_ = name;
    size_t pos = hostname_.find_first_of( ".:" );
    if ( pos != hostname_.npos )
    {
        hostname_.erase( hostname_.begin()+pos, hostname_.end() );
    }

    for (unsigned int i=0; i<memFinalItem; i++)
    {
        memInfoStringLen_[i] = strlen ( memInfoString_[i] );
    }

    if (pnid_ == MyPNID)
    {
        GetSchedulingData();
#ifdef HARD_AFFINITY
        sched_getaffinity(getpid(), sizeof(affinityMask_), &affinityMask_);
        int i;
        cpu_set_t mask;

        for (i=0; i < numCores_; i++)
        {
            if (CPU_ISSET(i, &affinityMask_))
            {
                if (trace_settings & TRACE_INIT)
                    trace_printf("%s@%d" " - Processor core #" "%d" " is usable" "\n", method_name, __LINE__, i);
            }
            else
            {
                if (trace_settings & TRACE_INIT)
                    trace_printf("%s@%d" " - Processor core #" "%d" " is not activated" "\n", method_name, __LINE__, i);
            }
        }
        if ( !IsRealCluster )
        {
            // set the monitor's hard affinity to a processor core
            CPU_ZERO(&mask);
            CPU_SET(MyPNID%numCores_, &mask);
            sched_setaffinity(getpid(), sizeof(mask), &mask);
        }
#else
        sched_getaffinity(getpid(), sizeof(affinityMask_), &affinityMask_);
#endif
    }

    CPU_ZERO( &excludedCoreMask_ );

    for ( int i = 0; i <  sparePNidCount; i++ )
    {
        sparePNids_.push_back( sparePNids[i] );
    }

    if (trace_settings & (TRACE_PROCESS | TRACE_PROCESS_DETAIL))
    {   // Display pidMap location, useful for understanding later trace output
        trace_printf("%s@%d PNid %d, pidMap_ (%p)\n",
                     method_name, __LINE__, GetPNid(), GetPidMap());
    }
    
    gettimeofday(&todStart_, NULL);

#ifndef NAMESERVER_PROCESS
    quiesceSendPids_ = new SQ_LocalIOToClient::bcastPids_t;
    quiesceExitPids_ = new SQ_LocalIOToClient::bcastPids_t;
#endif
    internalState_ = State_Default; 

    uniqStrId_ = Config->getMaxUniqueId ( pnid_ ) + 1;

    TRACE_EXIT;
}

CNode::~CNode( void )
{
    const char method_name[] = "CNode::~CNode";
    TRACE_ENTRY;

    if (trace_settings & (TRACE_INIT | TRACE_REQUEST))
    {
        trace_printf( "%s@%d pnid=%d\n", method_name, __LINE__, pnid_ );
    }

    // Alter eyecatcher sequence as a debugging aid to identify deleted object
    memcpy(&eyecatcher_, "pnod", 4);

    if ( procStatFile_ != NULL )
        fclose( procStatFile_ );

    if ( procMeminfoFile_ != -1 )
        close( procMeminfoFile_ );

#ifndef NAMESERVER_PROCESS
    if (quiesceSendPids_)
    {
        delete quiesceSendPids_;
    }

    if (quiesceExitPids_)
    {
        delete quiesceExitPids_;
    }
#endif
    
    TRACE_EXIT;
}

#ifndef NAMESERVER_PROCESS
void CNode::addToQuiesceSendPids( int pid, Verifier_t verifier )
{
    SQ_LocalIOToClient::pidVerifier_t pv;
    pv.pv.pid = pid;
    pv.pv.verifier = verifier;
    quiesceSendPids_->insert( pv.pnv );
}

void CNode::addToQuiesceExitPids( int pid, Verifier_t verifier )
{
    SQ_LocalIOToClient::pidVerifier_t pv;
    pv.pv.pid = pid;
    pv.pv.verifier = verifier;
    quiesceExitPids_->insert( pv.pnv );
}

void CNode::delFromQuiesceExitPids( int pid, Verifier_t verifier )
{
    SQ_LocalIOToClient::pidVerifier_t pv;
    pv.pv.pid = pid;
    pv.pv.verifier = verifier;
    quiesceExitPids_->erase( pv.pnv );
}
#endif

int CNode::AssignNid(void)
{
    const char method_name[] = "CNode::AssignNid";
    TRACE_ENTRY;

    CLNode *lnode = AssignLNode();
    
    TRACE_EXIT;
    if (lnode)
    {
        return( lnode->Nid );
    }
    else
    {
        return -1;
    }
}

CLNode *CNode::AssignLNode (void)
{
    CLNode *lnode;

    const char method_name[] = "CNode::AssignLNode";
    TRACE_ENTRY;

    assert(!spareNode_);

    // If logical nodes are configured
    lnode = GetFirstLNode();
    if ( lnode )
    {
        if ( !lastLNode_ )
        {
            lastLNode_ = lnode;
        }
        else if (lastLNode_->GetNextP())
        {
            lnode = lastLNode_->GetNextP();
        }
        else
        {
            lnode = lastLNode_;
        }
    }
    
    TRACE_EXIT;
    return lnode;
}

void CNode::CheckActivationPhase( void )
{
    int         tmCount = 0;
    CLNode     *lnode;
    CProcess   *process;
    bool        tmReady = false;

    const char method_name[] = "CNode::CheckActivationPhase";
    TRACE_ENTRY;

    // check for a TM process in each lnode
    lnode = GetFirstLNode();

    tmReady = lnode ? true : false;
    for ( ; lnode ; lnode = lnode->GetNextP() )
    {
        if ( lnode->GetState() == State_Up )
        {
            process = lnode->GetProcessLByType( ProcessType_DTM );
            if ( process )
            {
                tmCount++;
                if (trace_settings & (TRACE_INIT | TRACE_SYNC | TRACE_TMSYNC))
                    trace_printf("%s@%d - TM %s (pid=%d)\n", method_name, __LINE__, process->GetName(), process->GetPid());
            }
        }
        tmReady = (tmCount == GetLNodesCount()) ? true : false;
    }

    if ( tmReady )
    {
        if (trace_settings & (TRACE_INIT | TRACE_SYNC | TRACE_TMSYNC))
            trace_printf("%s@%d - Activation Phase_Ready on node %s, pnid=%d\n", method_name, __LINE__, GetName(), GetPNid());
        phase_ = Phase_Ready;
#ifndef NAMESERVER_PROCESS
        tmSyncState_ = SyncState_Null;
#endif
    }

    TRACE_EXIT;
}

void CNode::CheckShutdownProcessing( void )
{
    const char method_name[] = "CNode::CheckShutdownProcessing";
    TRACE_ENTRY;

#ifndef NAMESERVER_PROCESS
    struct message_def *msg;
    if ( shutdownLevel_ != lastSdLevel_ )
    {
        lastSdLevel_ = shutdownLevel_;
        msg = new struct message_def;
        msg->type = MsgType_Shutdown;
        msg->noreply = true;
        msg->u.request.type = ReqType_Notice;
        msg->u.request.u.shutdown.nid = MyPNID;
        msg->u.request.u.shutdown.pid = -1;
        msg->u.request.u.shutdown.level = shutdownLevel_;
        if (trace_settings & TRACE_SYNC)
           trace_printf("%s@%d" " - Broadcasting shutdown notice, level=" "%d" "\n", method_name, __LINE__, shutdownLevel_);
        char buf[MON_STRING_BUF_SIZE];
        sprintf(buf, "Broadcasting shutdown notice, level = %d\n", shutdownLevel_);
        mon_log_write(MON_NODE_SHUTDOWN_1, SQ_LOG_WARNING, buf);
        Bcast (msg);
        delete msg;
    }
#endif

    TRACE_EXIT;
}

#ifndef NAMESERVER_PROCESS
// In virtual node configuration, empty the quiescing pids so that new ones could be added.
void CNode::EmptyQuiescingPids()
{
    const char method_name[] = "CNode::EmptyQuiescingPids";
    TRACE_ENTRY;

    if (quiesceSendPids_)
    {
        quiesceSendPids_->clear();
    }

    if (quiesceExitPids_)
    {
        quiesceExitPids_->clear();
    }

    TRACE_EXIT;
}
#endif

#ifndef NAMESERVER_PROCESS
// Send quiescing notices to pids in the QiesceSendPids list. 
void CNode::SendQuiescingNotices( void )
{
    struct message_def *msg;

    const char method_name[] = "CNode::SendQuiescingNotices";
    TRACE_ENTRY;
    
    // send quiescing notices to TSEs and TMs
    msg = new struct message_def;
    msg->type = MsgType_NodeQuiesce;
    msg->noreply = true;
    msg->u.request.type = ReqType_Notice;
    msg->u.request.u.down.nid = MyPNID;
    strcpy(msg->u.request.u.down.node_name, GetNode()->GetName());

    if (trace_settings & TRACE_NOTICE)
           trace_printf("%s@%d" " - Broadcasting Quiesce notice to TSE and DTMs" "\n", method_name, __LINE__);

    // make a copy because the notice clear will delete the pids list if empty. 
    SQ_LocalIOToClient::bcastPids_t *quiesceSendPidsCopy = new SQ_LocalIOToClient::bcastPids_t;
    *quiesceSendPidsCopy = *quiesceSendPids_;

    SQ_theLocalIOToClient->putOnNoticeQueue( BCAST_PID
                                           , -1
                                           , msg
                                           , quiesceSendPidsCopy);

    TRACE_EXIT;
}
#endif

void CNode::SetState( STATE state )
{
    if ( state != state_ )
    {
        state_ = state;
        mem_log_write(CMonLog::MON_NODE_1, pnid_, state_); 
    }
}

void CNode::DeLink( CNode ** head, CNode ** tail )
{
    const char method_name[] = "CNode::DeLink";
    TRACE_ENTRY;
    if (*head == this)
        *head = next_;
    if (*tail == this)
        *tail = prev_;
    if (prev_)
        prev_->next_ = next_;
    if (next_)
        next_->prev_ = prev_;
    TRACE_EXIT;
}


void CNode::GetCpuStat ( void )
{
    char buffer[132];
    char name[25];
    long long num1, num2, num3, num4, num5, num6, num7;
    int count;
    int count2;
    int cpunum;

    // Holding area for cpu statistics, one set per core
    typedef struct {
        long long user;
        long long nice;
        long long sys;
        long long idle;
        long long wait;
        long long irq;
        long long sirq;
    } cpuStats_t;
    cpuStats_t cpuStats[MAX_CORES];

    const char method_name[] = "CNode::GetCpuStat";
    TRACE_ENTRY;


    // get processor statistics
    if ( procStatFile_ == NULL )
    {   // First time, open the file
        procStatFile_ = fopen("/proc/stat", "r");
        if (!procStatFile_)
        {
            if (trace_settings & TRACE_SYNC)
                trace_printf("%s@%d Cannot open /proc/stat, %s (%d)\n",
                             method_name, __LINE__, strerror(errno), errno);

            char buf[MON_STRING_BUF_SIZE];
            sprintf(buf, "[%s], Cannot open /proc/stat, %s (%d)\n",
                    method_name, strerror(errno), errno);

            mon_log_write(MON_NODE_GETCPUSTAT_1, SQ_LOG_ERR, buf);

            return;
        }
    }
    else
    {   // Other times, go back to the beginning.
        rewind ( procStatFile_ );
    }

    while( !feof(procStatFile_) )
    {
        fgets(buffer,132,procStatFile_);

        num1 = num2 = num3 = num4 = num5 = num6 = num7 = 0;
        count = sscanf(buffer, "%s %Ld %Ld %Ld %Ld %Ld %Ld %Ld",
                       (char *) &name, &num1, &num2, &num3, &num4, &num5,
                       &num6, &num7);

        if (count <= 0)
        {   // Invalid stats line
            continue;
        }

        if( strncmp( name, "btime", 5 ) == 0 )
        {
            bTime_ = (int) num1;
        }
        else if ( strncmp ( name, "cpu", 3) == 0 )
        {   // Got cpu statistics.  Determine which cpu.
            count2 = sscanf(name, "cpu%d", &cpunum);

            // Got cpu number, store the statistics for this cpu
            if (count2 == 1)
            {
                if (trace_settings & TRACE_SYNC_DETAIL)
                    trace_printf("%s@%d For cpu %d, user=%lld, nice=%lld, "
                                 "system=%lld, idle=%lld, iowait=%lld, "
                                 "irq=%lld, softirq=%lld\n", method_name,
                                 __LINE__, cpunum, num1, num2, num3, num4,
                                 num5, num6, num7);

                if (cpunum < MAX_CORES)
                {
                    cpuStats[cpunum].user = num1;
                    cpuStats[cpunum].nice = num2;
                    cpuStats[cpunum].sys  = num3;
                    cpuStats[cpunum].idle = num4;
                    cpuStats[cpunum].wait = num5;
                    cpuStats[cpunum].irq  = num6;
                    cpuStats[cpunum].sirq = num7;
                }
            }
        }
    }

    long long totUser, totNice, totSys, totIdle, totWait, totIrq, totSirq;
    int numCoresInLnode;
    int firstCoreInLnode;

    CLNode *lnode = GetFirstLNode();
    for ( ; lnode; lnode = lnode->GetNextP() )
    {
        numCoresInLnode = lnode->GetNumCores();
        firstCoreInLnode = lnode->GetFirstCore();

        // temp trace
        if (trace_settings & TRACE_SYNC_DETAIL)
            trace_printf("%s@%d pnode #%d, lnode #%d, numCoresInLnode=%d, "
                         "firstCoreInLnode=%d\n", method_name, __LINE__,
                         lnode->GetNode()->GetPNid(), lnode->Nid, numCoresInLnode,
                         firstCoreInLnode);

        // Aggregate stats for the logical node
        totUser = totNice = totSys = totIdle = totWait = totIrq = totSirq = 0;
        for (int i=firstCoreInLnode; i<(firstCoreInLnode+numCoresInLnode); ++i)
        {
            totUser += cpuStats[i].user;
            totNice += cpuStats[i].nice;
            totSys  += cpuStats[i].sys;
            totIdle += cpuStats[i].idle;
            totWait += cpuStats[i].wait;
            totIrq  += cpuStats[i].irq;
            totSirq += cpuStats[i].sirq;

            if (trace_settings & TRACE_SYNC_DETAIL)
                trace_printf("%s@%d pnode #%d, lnode #%d, core=%d, "
                             "totUser=%lld, totNice=%lld, totSys=%lld, "
                             "totIdle=%lld, totWait=%lld, totIrq=%lld, "
                             "totSirq=%lld\n", method_name, __LINE__,
                             lnode->GetNode()->GetPNid(), lnode->Nid, i, totUser,
                             totNice, totSys, totIdle, totWait, totIrq,
                             totSirq);

        }

        // Store average of stats over all cores comprising the logical node
        lnode->SetCpuUser   ( totUser / numCoresInLnode );
        lnode->SetCpuNice   ( totNice / numCoresInLnode );
        lnode->SetCpuSystem ( totSys  / numCoresInLnode );
        lnode->SetCpuIdle   ( totIdle / numCoresInLnode );
        lnode->SetCpuIowait ( totWait / numCoresInLnode );
        lnode->SetCpuIrq    ( totIrq  / numCoresInLnode );
        lnode->SetCpuSoftIrq( totSirq / numCoresInLnode );
    }

    TRACE_EXIT;
}

bool CNode::NextMemInfoLine( bufInfo_t &inBuf, char * dataline )
{
    char * eol;
    size_t bytes;
    ssize_t count;

    // check if another line is available in the buffer
    eol = strchr ( inBuf.bol, '\n' );

    if ( eol == NULL )
    {   // no, so read more data

        // move any residual data
        memcpy ( inBuf.buffer, inBuf.bol, inBuf.remBytes );

        // read additional data
        count = read(procMeminfoFile_, &inBuf.buffer[inBuf.remBytes],
                     inBuf.bufsize - inBuf.remBytes - 1);

        if ( count != 0 && count != -1 )
        {   // Update buffer data pointers and counts
            inBuf.buffer[inBuf.remBytes + count] = '\0';
            inBuf.remBytes += count;
            inBuf.bol = inBuf.buffer;

            eol = strchr ( inBuf.bol, '\n' );
        }
    }

    if ( eol != NULL )
    {   // Have another line, return data to caller

        bytes = eol - inBuf.bol;

        memcpy ( dataline, inBuf.bol, bytes );
        dataline[bytes] = '\0';

        inBuf.bol = ++eol;
        inBuf.remBytes -= (bytes + 1);

        return true;
    }
    else
    {   // No more data available
        return false;
    }
}

bool CNode::GetSchedulingData( void )
{
    static bool get_processor_info = true;

    const char method_name[] = "CNode::GetSchedulingData";
    TRACE_ENTRY;

    if( pnid_ != MyPNID )
    {
        char buf[MON_STRING_BUF_SIZE];
        sprintf(buf, "[CNode::GetSchedulingData], Invalid call for remote node.\n");
        mon_log_write(MON_NODE_GETSCHDATA_1, SQ_LOG_ERR, buf);
    }
    else
    {
       // Getting scheduling data is time consuming so don't do it too
        // often.
        struct timespec now;
        long int secs;
        long int nsecs;
        unsigned long long int msecs;

        clock_gettime(CLOCK_REALTIME, &now);
        if ( (now.tv_nsec - prevSchedData_.tv_nsec )  < 0 )
        {
            secs = now.tv_sec - prevSchedData_.tv_sec - 1;
            nsecs = 1000000000 + now.tv_nsec - prevSchedData_.tv_nsec;
        }
        else
        {
            secs = now.tv_sec - prevSchedData_.tv_sec;
            nsecs = now.tv_nsec - prevSchedData_.tv_nsec;
        }
        msecs = (secs * 1000) + (nsecs / 1000000);
        if ( msecs < minSchedDataInterval_ )
        {
            if (trace_settings & TRACE_SYNC)
                trace_printf("%s@%d Defer refreshing scheduling data\n",
                             method_name, __LINE__);
            // Wait longer to get scheduling data again
            return false;
        }
        // Remember time we last got scheduling data
        prevSchedData_ = now;

        // get memory information    
        if ( procMeminfoFile_ == -1 )
        {   // First time, open the file
            procMeminfoFile_ = open("/proc/meminfo", O_RDONLY);
            if ( procMeminfoFile_ == -1 )
            {
                if (trace_settings & TRACE_SYNC)
                    trace_printf("%s@%d Cannot open /proc/meminfo, %s (%d)\n",
                                 method_name, __LINE__, strerror(errno), errno);

                char buf[MON_STRING_BUF_SIZE];
                sprintf(buf, "[%s], Cannot open /proc/meminfo, %s (%d)\n",
                        method_name, strerror(errno), errno);
                mon_log_write(MON_NODE_GETSCHDATA_7, SQ_LOG_ERR, buf);
            }
        }
        else
        {   // Other times, go back to the beginning.
            lseek ( procMeminfoFile_, 0, SEEK_SET );
        }

        if ( procMeminfoFile_ != -1 )
        {
            bool foundProcMem[memFinalItem];
            for (unsigned int i=0; i<memFinalItem; i++)
                foundProcMem[i] = false;

            int totalProcMemFound = 0;
            unsigned long int value;

            char readBuffer[132];
            char memLine[80];

            bufInfo_t bufInfo;
            readBuffer[0] = '\0';
            bufInfo.buffer = readBuffer;
            bufInfo.bol = readBuffer;
            bufInfo.bufsize = sizeof(readBuffer);
            bufInfo.remBytes = 0;

            while  ( NextMemInfoLine ( bufInfo, memLine ) )
            {
                for (unsigned int i=0; i<memFinalItem; i++)
                {
                    if ( strncmp( memLine, memInfoString_[i],
                                  memInfoStringLen_[i]) == 0 )
                    {
                        errno = 0;
                        value = strtoul ( &memLine[memInfoStringLen_[i]],
                                          NULL, 10 );
                        if ( errno == 0 )
                        {
                            memInfoData_[i] = value;
                        }
                        else
                        {   // Unexpectedly could not convert data
                            // value. (Will retain previous value.)
                            char buf[MON_STRING_BUF_SIZE];
                            sprintf(buf, "%s, possible /proc/meminfo/%s "
                                    " conversion error, memLine='%s'. errno=%d"
                                    "\n", method_name, memInfoString_[i],
                                    memLine, errno);
                            mon_log_write(MON_NODE_GETSCHDATA_3, SQ_LOG_ERR,
                                          buf);
                        }
                        foundProcMem[i] = true;
                        ++totalProcMemFound;
                        break;
                    }
                }

                if (totalProcMemFound == memFinalItem)
                    break;
            }

            freeCache_ = memInfoData_[memFree] + memInfoData_[memBuffers]
                + memInfoData_[memCached];
            if (totalProcMemFound != memFinalItem)
            {   // Failed to find some data

                for (unsigned int i=0; i<memFinalItem; i++)
                {
                    if (!foundProcMem[i])
                    {   // Did not obtain this memory counter
                        char buf[MON_STRING_BUF_SIZE];
                        sprintf(buf, "%s, %s not in /proc/meminfo.\n",
                                method_name, memInfoString_[i]);
                        mon_log_write(MON_NODE_GETSCHDATA_2, SQ_LOG_ERR, buf);
                    }
                }
            }
        }

        if( get_processor_info )
        {
            // get processor information, one time only
            FILE *proccpu = NULL;
            char buffer[132];
            numCores_ = 0;
            proccpu = fopen("/proc/cpuinfo", "r");
    
            while( !feof(proccpu) )
            {    
                fgets(buffer,132,proccpu);
                if( strncmp( buffer, "processor\t:", 11 ) == 0 )
                {
                    numCores_++;
                }
            }

            get_processor_info = false;
            if (trace_settings & TRACE_SYNC_DETAIL)
               trace_printf("%s@%d" " - NumCores=" "%d" "\n", method_name, __LINE__, GetNumCores());
            fclose( proccpu );
        }

       if (trace_settings & TRACE_SYNC_DETAIL)
            trace_printf("%s@%d - Processor memory=%d kB, FreeMemory=%d kB, buffers=%d kB, cached=%d kB, FreeSwap=%d kB, Active=%d kB, Inactive=%d kB, Dirty=%d kB, Writeback=%d kB, VmallocUsed=%d kB\n", method_name, __LINE__, memInfoData_[memTotal], memInfoData_[memFree], memInfoData_[memBuffers], memInfoData_[memCached], memInfoData_[memSwapFree], memInfoData_[memActive], memInfoData_[memInactive], memInfoData_[memDirty], memInfoData_[memWriteback], memInfoData_[memVmallocUsed]);

       GetCpuStat();
    }

    TRACE_EXIT;

    return true;
}


strId_t CNode::GetStringId( char *candidate, CLNode *targetLNode, bool clone )
{
    const char method_name[] = "CNode::GetStringId";
    strId_t id;

    TRACE_ENTRY;

    if ( ! Config->findUniqueString ( pnid_, candidate, id ) )
    {   // The string is not in the configuration database, add it
        id.id  = uniqStrId_++;
        id.nid = pnid_;

        if (trace_settings & TRACE_PROCESS)
        {
            trace_printf("%s@%d - Adding unique string id=[%d,%d] (%s), targetLnode=%p, targetNid=%d\n",
                         method_name, __LINE__, id.nid, id.id, candidate, 
                         targetLNode, targetLNode?targetLNode->GetNid():-1 );
        }

        Config->addUniqueString(id.nid, id.id, candidate);

#ifndef NAMESERVER_PROCESS
        if (NameServerEnabled)
        {
            if (targetLNode != NULL && !clone &&
                !MyNode->IsMyNode(targetLNode->GetNid()))
            {
                // Forward the unique string to the target node
                int rc = PtpClient->AddUniqStr( id.nid
                                              , id.id
                                              , candidate
                                              , targetLNode->GetNid()
                                              , targetLNode->GetNode()->GetName() );
                if (rc)
                {
                    char la_buf[MON_STRING_BUF_SIZE];
                    snprintf( la_buf, sizeof(la_buf)
                            , "[%s] - Can't send unique string "
                              "to target node %s, nid=%d\n"
                            , method_name
                            , targetLNode->GetNode()->GetName()
                            , targetLNode->GetNid() );
                    mon_log_write(MON_NODE_GETSTRINGID_1, SQ_LOG_ERR, la_buf);
                }
            }
        }
        else
#endif
        {
#ifdef NAMESERVER_PROCESS
            clone = clone;  // Make compiler happy!
            targetLNode = targetLNode;  // Make compiler happy!
#endif
            CReplUniqStr *repl = new CReplUniqStr ( id.nid, id.id, candidate );
            Replicator.addItem(repl);
        }
    }
    // temp trace
    else
    {
        if (trace_settings & TRACE_PROCESS)
        {
            trace_printf("%s@%d - unique string id=[%d,%d] (%s)\n",
                         method_name, __LINE__, id.nid, id.id, candidate );
        }

#ifndef NAMESERVER_PROCESS
        if (NameServerEnabled)
        {
            if (targetLNode != NULL && !clone &&
                !MyNode->IsMyNode(targetLNode->GetNid()))
            {
                // Forward the unique string to the target node
                int rc = PtpClient->AddUniqStr( id.nid
                                              , id.id
                                              , candidate
                                              , targetLNode->GetNid()
                                              , targetLNode->GetNode()->GetName());
                if (rc)
                {
                    char la_buf[MON_STRING_BUF_SIZE];
                    snprintf( la_buf, sizeof(la_buf)
                            , "[%s] - Can't send unique string "
                              "to target node %s, nid=%d\n"
                            , method_name
                            , targetLNode->GetNode()->GetName()
                            , targetLNode->GetNid() );
                    mon_log_write(MON_NODE_GETSTRINGID_2, SQ_LOG_ERR, la_buf);
                }
            }
        }
#endif
    }

    TRACE_EXIT;

    return id;
}

CNode *CNode::Link( CNode * entry )
{
    const char method_name[] = "CNode::Link";
    TRACE_ENTRY;
    next_ = entry;
    entry->prev_ = this;

    TRACE_EXIT;
    return entry;
}

void CNode::MoveLNodes( CNode *spareNode )
{
    CLNode *lnode;
    const char method_name[] = "CNode::MoveLNodes";
    TRACE_ENTRY;

    // Move the logical nodes
    lnode = GetFirstLNode();
    for ( ; lnode; lnode = lnode->GetNextP() )
    {
        // adjust each logical node to reference the 
        // new physical node's logical nodes container
        lnode->SetLNodeContainer( spareNode->GetLNodeContainer() );
    }
    spareNode->SetFirstLNode( GetFirstLNode() );
    spareNode->SetLastLNode( GetLastLNode() );
    spareNode->SetLNodesCount( GetLNodesCount() );
    SetFirstLNode( NULL );
    SetLastLNode( NULL );
    SetLNodesCount( 0 );
    lastLNode_ = NULL;

    // Move the physical node's process list
    nameMap_t *nameMap = spareNode->GetNameMap();
    nameMap->clear();
    pidMap_t *pidMap = spareNode->GetPidMap();
    pidMap->clear();
    spareNode->SetNameMap( GetNameMap() );
    SetNameMap( nameMap );
    spareNode->SetPidMap( GetPidMap() );
    SetPidMap( pidMap );
    spareNode->SetFirstProcess( GetFirstProcess() );
    spareNode->SetLastProcess( GetLastProcess() );
    spareNode->SetNumProcs( GetNumProcs() );
    spareNode->ResetSpareNode();
    // Reset the state of the down node, it is now a spare node
    SetFirstProcess( NULL );
    SetLastProcess( NULL );
    SetNumProcs( 0 );
    SetSpareNode();

    // Move zone identifier
    spareNode->SetZone( GetZone() );
    SetZone( -1 );
    
    TRACE_EXIT;
    return;
}

#ifndef NAMESERVER_PROCESS
void CNode::SetAffinity( int nid, pid_t pid, PROCESSTYPE type )
{
    CLNode  *lnode = Nodes->GetLNode( nid );

    const char method_name[] = "CNode::SetAffinity";
    TRACE_ENTRY;

    if ( lnode )
    {
        lnode->SetAffinity (pid, type);
    }
    else
    {
        abort();
    }

    TRACE_EXIT;
}

void CNode::SetAffinity( CProcess *process )
{
    CLNode  *lnode = Nodes->GetLNode( process->GetNid() );

    const char method_name[] = "CNode::SetAffinity";
    TRACE_ENTRY;

    if ( lnode )
    {
        lnode->SetAffinity( process );
    }
    else
    {
        abort();
    }

    TRACE_EXIT;
}

void CNode::StartNameServerProcess( void )
{
    const char method_name[] = "CNode::StartNameServerProcess";
    TRACE_ENTRY;

    char path[MAX_SEARCH_PATH];
    char *ldpath = NULL; // = getenv("LD_LIBRARY_PATH");
    char filename[MAX_PROCESS_PATH];
    char name[MAX_PROCESS_NAME];
    char stdout[MAX_PROCESS_PATH];
    
    snprintf( name, sizeof(name), "$TNS%d", MyNode->GetZone() );
    snprintf( stdout, sizeof(stdout), "stdout_TNS%d", MyNode->GetZone() );

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
       trace_printf("%s@%d" " - Creating NameService Process\n", method_name, __LINE__);

    strcpy(path,getenv("PATH"));
    strcat(path,":");
    strcat(path,MyPath);
    strcpy(filename,"trafns");
    ldpath = getenv("LD_LIBRARY_PATH");
    strId_t pathStrId = MyNode->GetStringId ( path );
    strId_t ldpathStrId = MyNode->GetStringId ( ldpath );
    strId_t programStrId = MyNode->GetStringId ( filename );

    int result;
    NameServerProcess  = CreateProcess( NULL, //parent
                                        MyNode->AssignNid(),
                                        ProcessType_NameServer,
                                        0,  //debug
                                        0,  //priority
                                        0,  //backup
                                        true, //unhooked
                                        name,
                                        pathStrId,
                                        ldpathStrId,
                                        programStrId,
                                        (char *) "", //infile,
                                        stdout, //outfile,
                                        0, //tag
                                        result
                                        );
    if ( NameServerProcess )
    {
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
           trace_printf("%s@%d" " - NameService Process created\n", method_name, __LINE__);
    }
    else
    {
        char la_buf[MON_STRING_BUF_SIZE];
        sprintf(la_buf, "[%s], NameService Process creation failed.\n", method_name);
        mon_log_write( MON_NODE_STARTNAMESERVER_1, SQ_LOG_ERR, la_buf );
    }

    TRACE_EXIT;
}

void CNode::StartWatchdogProcess( void )
{
    const char method_name[] = "CNode::StartWatchdogProcess";
    TRACE_ENTRY;

    char path[MAX_SEARCH_PATH];
    char *ldpath = NULL; // = getenv("LD_LIBRARY_PATH");
    char filename[MAX_PROCESS_PATH];
    char name[MAX_PROCESS_NAME];
    char stdout[MAX_PROCESS_PATH];
    CProcess * watchdogProcess;
    
    snprintf( name, sizeof(name), "$WDG%d", MyNode->GetZone() );
    snprintf( stdout, sizeof(stdout), "stdout_WDG%d", MyNode->GetZone() );

    // The following variables are used to retrieve the proper startup and keepalive environment variable
    // values, and to use as arguments for the lower level ioctl calls that interface with the watchdog 
    // timer package.

    char *WDT_KeepAliveTimerValueC;

    // If the SQ_WDT_KEEPALIVETIMERVALUE are not defined in this case,
    // we will use the default values defined above.

    if (!(WDT_KeepAliveTimerValueC = getenv("SQ_WDT_KEEPALIVETIMERVALUE")))
    {
        wdtKeepAliveTimerValue_ = WDT_KeepAliveTimerDefault;
    }
    else
    {
        wdtKeepAliveTimerValue_ = atoi(WDT_KeepAliveTimerValueC);	
    }

    //Displays the startup and keep alive timer values in use for a given run.
    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
       trace_printf("%s@%d" " - KeepAlive Timer in seconds =%d\n", method_name, __LINE__, (wdtKeepAliveTimerValue_));

    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
       trace_printf("%s@%d" " - Creating Watchdog Process\n", method_name, __LINE__);

    strcpy(path,getenv("PATH"));
    strcat(path,":");
    strcat(path,MyPath);
    strcpy(filename,"sqwatchdog");
    ldpath = getenv("LD_LIBRARY_PATH");
    strId_t pathStrId = MyNode->GetStringId ( path );
    strId_t ldpathStrId = MyNode->GetStringId ( ldpath );
    strId_t programStrId = MyNode->GetStringId ( filename );

    int result;
    watchdogProcess  = CreateProcess( NULL, //parent
                                      MyNode->AssignNid(),
                                      ProcessType_Watchdog,
                                      0,  //debug
                                      0,  //priority
                                      0,  //backup
                                      true, //unhooked
                                      name,
                                      pathStrId,
                                      ldpathStrId,
                                      programStrId,
                                      (char *) "", //infile,
                                      stdout, //outfile,
                                      0, //tag
                                      result
                                      );
    if ( watchdogProcess )
    {
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
           trace_printf("%s@%d" " - Watchdog Process created\n", method_name, __LINE__);

        gettimeofday(&todStart_, NULL);
    }
    else
    {
        char la_buf[MON_STRING_BUF_SIZE];
        sprintf(la_buf, "[CNode::StartWatchdogProcess], Watchdog Process creation failed. Timer disabled\n");
        mon_log_write( MON_NODE_STARTWATCHDOG_1, SQ_LOG_ERR, la_buf );
    }

    TRACE_EXIT;
}

void CNode::StartPStartDProcess( void )
{
    const char method_name[] = "CNode::StartPStartDProcess";
    TRACE_ENTRY;

    char path[MAX_SEARCH_PATH];
    char *ldpath = NULL;
    char filename[MAX_PROCESS_PATH];
    char name[MAX_PROCESS_NAME];
    char stdout[MAX_PROCESS_PATH];
    CProcess * pstartdProcess;
    
    snprintf( name, sizeof(name), "$PSD%d", MyNode->GetZone() );
    snprintf( stdout, sizeof(stdout), "stdout_PSD%d", MyNode->GetZone() );

    strcpy(path,getenv("PATH"));
    strcat(path,":");
    strcat(path,MyPath);
    strcpy(filename,"pstartd");
    ldpath = getenv("LD_LIBRARY_PATH");
    strId_t pathStrId = MyNode->GetStringId ( path );
    strId_t ldpathStrId = MyNode->GetStringId ( ldpath );
    strId_t programStrId = MyNode->GetStringId ( filename );

    int result;
    pstartdProcess  = CreateProcess( NULL, //parent
                                      MyNode->AssignNid(),
                                      ProcessType_PSD,
                                      0,  //debug
                                      0,  //priority
                                      0,  //backup
                                      true, //unhooked
                                      name,
                                      pathStrId,
                                      ldpathStrId,
                                      programStrId,
                                      (char *) "", //infile,
                                      stdout, //outfile,
                                      0, //tag
                                      result
                                      );
    if ( pstartdProcess )
    {
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
           trace_printf("%s@%d - pstartd process created\n",
                        method_name, __LINE__);
    }
    else
    {
        char buf[MON_STRING_BUF_SIZE];
        sprintf(buf, "[CNode::StartPStartDProcess], pstartd creation failed.\n");
        mon_log_write( MON_NODE_STARTPSTARTD_1, SQ_LOG_ERR, buf );
    }

    TRACE_EXIT;
}

void CNode::StartPStartDPersistent( void )
{
    const char method_name[] = "CNode::StartPStartDPersistent";
    TRACE_ENTRY;

    if ( trace_settings & (TRACE_RECOVERY | TRACE_REQUEST | TRACE_INIT) )
    {
        trace_printf( "%s@%d - Sending start persistent processes event %d (%s)\n"
                    , method_name, __LINE__
                    , PStartD_StartPersist, name_ );
    }

    int tmCount = 0;
    CNode *node;
    CLNode *lnode;
    CProcess *process;

    // Any DTMs running in cluster?
    for ( int i=0; !tmCount && i < Nodes->GetPNodesCount(); i++ )
    {
        node = Nodes->GetNodeByMap( i );
        lnode = node->GetFirstLNode();
        for ( ; lnode; lnode = lnode->GetNextP() )
        {
            CProcess *process = lnode->GetProcessLByType( ProcessType_DTM );
            if ( process ) tmCount++;
        }
    }

    if ( tmCount )
    {
        lnode = GetFirstLNode();
        for ( ; lnode ; lnode = lnode->GetNextP() )
        {
            if ( lnode->GetState() == State_Up )
            {
                // Send local PSD process event to start persistent processes 
                // that don't require transactions
                process = lnode->GetProcessLByType( ProcessType_PSD );
                if ( process && process->IsFirstInstance() )
                {
                    char nidString[6];
                    sprintf(nidString,"%d",lnode->GetNid());
                    process->GenerateEvent( PStartD_StartPersist, sizeof(nidString), nidString );
            
                    if ( trace_settings & 
                        (TRACE_RECOVERY | TRACE_REQUEST | TRACE_INIT) )
                    {
                        trace_printf( "%s@%d - Sent event %d (%s) to PSD %s (pid=%d)\n"
                                    , method_name, __LINE__
                                    , PStartD_StartPersist, nidString
                                    , process->GetName(), process->GetPid());
                    }
                }
            }
        }
    }
    
    TRACE_EXIT;
}

void CNode::StartPStartDPersistentDTM( int nid )
{
    const char method_name[] = "CNode::StartPStartDPersistentDTM";
    TRACE_ENTRY;

    assert( GetState() == State_Up );

    int tmCount = 0;
    CNode *node;
    CLNode *lnode;
    CProcess *process;

    // Any DTMs running in cluster?
    for ( int i=0; !tmCount && i < Nodes->GetPNodesCount(); i++ )
    {
        node = Nodes->GetNodeByMap( i );
        lnode = node->GetFirstLNode();
        for ( ; lnode; lnode = lnode->GetNextP() )
        {
            process = lnode->GetProcessLByType( ProcessType_DTM );
            if ( process ) tmCount++;
        }
    }

    if ( tmCount )
    {
        lnode = GetFirstLNode();
        for ( ; lnode ; lnode = lnode->GetNextP() )
        {
            if ( lnode->GetNid() == nid )
            {
                // Send local PSD process event to start persistent processes 
                // that do require transactions
                process = lnode->GetProcessLByType( ProcessType_PSD );
                if ( process )
                {
                    char nidString[6];
                    sprintf( nidString, "%d", nid );
                    process->GenerateEvent( PStartD_StartPersistDTM, sizeof(nidString), nidString );
            
                    if ( trace_settings & 
                        (TRACE_RECOVERY | TRACE_REQUEST | TRACE_INIT) )
                    {
                        trace_printf( "%s@%d - Sending event %d (%s) to PSD %s (pid=%d)\n"
                                    , method_name, __LINE__
                                    , PStartD_StartPersistDTM, nidString
                                    , process->GetName(), process->GetPid());
                    }
                }
            }
        }
    }
    
    TRACE_EXIT;
}

void CNode::StartSMServiceProcess( void )
{
    const char method_name[] = "CNode::StartSMServiceProcess";
    TRACE_ENTRY;

    char path[MAX_SEARCH_PATH];
    char *ldpath = NULL;
    char filename[MAX_PROCESS_PATH];
    char name[MAX_PROCESS_NAME];
    char stdout[MAX_PROCESS_PATH];
    CProcess *smsProcess;
    
    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
       trace_printf("%s@%d" " - Creating SMService Process\n", method_name, __LINE__);

    snprintf( name, sizeof(name), "$SMS%03d", MyNode->GetZone() );
    snprintf( stdout, sizeof(stdout), "stdout_SMS%03d", MyNode->GetZone() );

    strcpy(path,getenv("PATH"));
    strcat(path,":");
    strcat(path,MyPath);
    strcpy(filename,"smservice");
    ldpath = getenv("LD_LIBRARY_PATH");
    strId_t pathStrId = MyNode->GetStringId ( path );
    strId_t ldpathStrId = MyNode->GetStringId ( ldpath );
    strId_t programStrId = MyNode->GetStringId ( filename );

    int result;
    smsProcess  = CreateProcess( NULL, //parent
                                 MyNode->AssignNid(),
                                 ProcessType_SMS,
                                 0,  //debug
                                 0,  //priority
                                 0,  //backup
                                 true, //unhooked
                                 name,
                                 pathStrId,
                                 ldpathStrId,
                                 programStrId,
                                 (char *) "", //infile,
                                 stdout, //outfile,
                                 0, //tag
                                 result
                                 );
    if ( smsProcess )
    {
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
           trace_printf("%s@%d - smservice process (%s) created\n",
                        method_name, __LINE__, name);
    }
    else
    {
        char buf[MON_STRING_BUF_SIZE];
        sprintf(buf, "[%s], smservice process (%s)creation failed.\n", 
                method_name, name);
        mon_log_write( MON_NODE_STARTSMSERVICE_1, SQ_LOG_CRIT, buf );
    }

    TRACE_EXIT;
}
#endif

CNodeContainer::CNodeContainer( void )
               :CLNodeContainer(NULL)
               ,Node(NULL)
               ,pnodeCount_(0)
               ,indexToPnid_(NULL)
               ,clusterConfig_(NULL)
               ,nameServerConfig_(NULL)
               ,head_(NULL)
               ,tail_(NULL)
               ,syncBufferFreeSpace_(MAX_SYNC_SIZE)
               ,lastSyncBuffer_(NULL)
               ,SyncBuffer(NULL)
{
    const char method_name[] = "CNodeContainer::CNodeContainer";
    TRACE_ENTRY;

    // Add eyecatcher sequence as a debugging aid
    memcpy(&eyecatcher_, "NCTR", 4);

    lastSyncBuffer_ = new struct sync_buffer_def;
    SyncBuffer = new struct sync_buffer_def;

    // Load cluster configuration
    LoadConfig();

    // Allocate logical and physical node arrays
    Node = new CNode *[clusterConfig_->GetPNodesConfigMax()];
    LNode = new CLNode *[clusterConfig_->GetLNodesConfigMax()];
    indexToPnid_ = new int[clusterConfig_->GetPNodesConfigMax()];
    for (int i = 0; i < clusterConfig_->GetPNodesConfigMax(); i++ )
    {
        Node[i] = NULL;
        indexToPnid_[i] = -1;
    }
    indexToNid_ = new int[clusterConfig_->GetLNodesConfigMax()];
    for (int i = 0; i < clusterConfig_->GetLNodesConfigMax(); i++ )
    {
        LNode[i] = NULL;
        indexToNid_[i] = -1;
    }

    TRACE_EXIT;
}

CNodeContainer::~CNodeContainer( void )
{
    CNode *node = head_;

    const char method_name[] = "CNodeContainer::~CNodeContainer";
    TRACE_ENTRY;
    if (lastSyncBuffer_)
    {
        delete lastSyncBuffer_;
    }
    if (SyncBuffer)
    {
        delete SyncBuffer;
    }
    while (head_)
    {
        node->DeLink (&head_, &tail_);
        delete node;
        node = head_;
    }

    if (clusterConfig_)
    {
        delete clusterConfig_;
        clusterConfig_ = NULL;
    }
    if (nameServerConfig_)
    {
        delete nameServerConfig_;
    }
    if (Node)
    {
        delete [] Node;
    }
    if (LNode)
    {
        delete [] LNode;
    }

    // Alter eyecatcher sequence as a debugging aid to identify deleted object
    memcpy(&eyecatcher_, "nctr", 4);
    
    TRACE_EXIT;
}


int CNodeContainer::GetPNid( char *nodeName )
{
    const char method_name[] = "CNodeContainer::GetPNid";
    TRACE_ENTRY;

    int pnid = clusterConfig_->GetPNid( nodeName );

    TRACE_EXIT;
    return( pnid );
}

void CNodeContainer::AddedNode( CNode *node )
{
    const char method_name[] = "CNodeContainer::AddedNode";
    TRACE_ENTRY;

    if (trace_settings & (TRACE_REQUEST | TRACE_SYNC))
    {
        trace_printf( "%s@%d - node %s, pnid=%d, zone=%d\n"
                    , method_name, __LINE__
                    , node->GetName(), node->GetPNid(), node->GetZone() );
    }

    assert( node->GetState() == State_Down );

    // Broadcast node added notice to local processes
    CLNode *lnode = node->GetFirstLNode();
    for ( ; lnode; lnode = lnode->GetNextP() )
    {
        lnode->Added();
    }

    TRACE_EXIT;
}

#ifndef NAMESERVER_PROCESS
CProcess *CNodeContainer::AddCloneProcess( ProcessInfoNs_reply_def *processInfo )
{
    const char method_name[] = "CNodeContainer::AddNode";
    TRACE_ENTRY;

    CLNode   *lnode = Nodes->GetLNode(processInfo->nid);
    CNode    *node = lnode->GetNode();

    strId_t pathStrId = MyNode->GetStringId ( processInfo->path, lnode, true );
    strId_t ldpathStrId = MyNode->GetStringId (processInfo->ldpath, lnode, true );
    strId_t programStrId = MyNode->GetStringId ( processInfo->program, lnode, true );

    CProcess *process = node->CloneProcess( processInfo->nid
                                          , processInfo->type
                                          , processInfo->priority
                                          , processInfo->backup
                                          , processInfo->unhooked
                                          , processInfo->process_name
                                          , processInfo->port_name
                                          , processInfo->pid
                                          , processInfo->verifier
                                          , processInfo->parent_nid
                                          , processInfo->parent_pid
                                          , processInfo->parent_verifier
                                          , processInfo->event_messages
                                          , processInfo->system_messages
                                          , pathStrId
                                          , ldpathStrId
                                          , programStrId
//                                          , processInfo->pathStrId
//                                          , processInfo->ldpathStrId
//                                          , processInfo->programStrId
                                          , processInfo->infile
                                          , processInfo->outfile
                                          , &processInfo->creation_time
                                          , -1 );//processInfo->origPNidNs_);

    TRACE_EXIT;
    return(process);
}
#endif

CNode *CNodeContainer::AddNode( int pnid )
{
    const char method_name[] = "CNodeContainer::AddNode";
    TRACE_ENTRY;

    CNode        *node = NULL;
    CPNodeConfig *pnodeConfig = clusterConfig_->GetPNodeConfig( pnid );
    if (pnodeConfig)
    {
        node = GetNode(pnid);
        if (!node)
        {
            node = new CNode( (char *)pnodeConfig->GetName()
                            , pnodeConfig->GetPNid(), -1 );
            assert( node != NULL );
    
            if ( node )
            {
                // Add node to monitor's view of the cluster
                AddNode( node );

                // Broadcast node added notice to local processes
                AddedNode( node );
            }
            else
            {
                char buf[MON_STRING_BUF_SIZE];
                snprintf( buf, sizeof(buf)
                        , "[%s@%d] Could not allocate physical node object "
                          ", pnid=%d\n"
                        , method_name, __LINE__, pnid);
                mon_log_write(MON_NODE_ADDNODE_1, SQ_LOG_ERR, buf);
            }
        }
        else
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf)
                    , "[%s@%d] Found existing physical node "
                      "in monitor's view of cluster, pnid=%d\n"
                    , method_name, __LINE__, pnid);
            mon_log_write(MON_NODE_ADDNODE_2, SQ_LOG_ERR, buf);
        }       
    }
    else
    {
        char buf[MON_STRING_BUF_SIZE];
        snprintf( buf, sizeof(buf)
                , "[%s@%d] Could not find physical node in configuration, "
                  "pnid=%d\n"
                , method_name, __LINE__, pnid);
        mon_log_write(MON_NODE_ADDNODE_3, SQ_LOG_ERR, buf);
    }

    TRACE_EXIT;
    return(node);
}

void CNodeContainer::AddNode( CNode *node )
{
    const char method_name[] = "CNodeContainer::AddNode";
    TRACE_ENTRY;

    assert( node != NULL );

    if ( node )
    {
        if (trace_settings & TRACE_INIT)
        {
            trace_printf( "%s@%d - Adding physical node object "
                          "(pnid=%d, nodename=%s) to container, "
                          "pnodesCount=%d, lnodesCount=%d\n"
                        , method_name, __LINE__
                        , node->GetPNid()
                        , node->GetName()
                        , GetPNodesCount()
                        , GetLNodesCount() );
        }

        // add physical node to physical nodes array
        Node[node->GetPNid()] = node;
        pnodeCount_++;

        if (head_ == NULL)
        {
            head_ = tail_ = node;
        }
        else
        {
            tail_ = tail_->Link(node);
        }
#ifdef NAMESERVER_PROCESS
        AddLNodes( node );
#else
        // now add logical nodes to physical node
        if (IAmIntegrating)
        {
            // do nothing.
            // lnodes will be created in the revive phase
        }
        else
        {
            AddLNodes( node );
        }
#endif

        if (trace_settings & TRACE_INIT)
        {
            trace_printf( "%s@%d - Added physical node object "
                          "(pnid=%d, nodename=%s) to container, "
                          "pnodesCount=%d, lnodesCount=%d\n"
                        , method_name, __LINE__
                        , node->GetPNid()
                        , node->GetName()
                        , GetPNodesCount()
                        , GetLNodesCount() );
        }
    }

    TRACE_EXIT;
}

void CNodeContainer::AddNodes( )
{
    const char method_name[] = "CNodeContainer::AddNodes";
    TRACE_ENTRY;

    CNode *node;
    int pnid;
    int rank;
    int *sparePNids = NULL;
    
    //  only relevant on a workstation acting as a single node
    const char* envVar = getenv("SQ_MAX_RANK"); 
    int maxNode;
    if (envVar != NULL)
        maxNode = atoi (envVar);
    else
        maxNode = MAX_NODES; 

    CPNodeConfig *pnodeConfig = clusterConfig_->GetFirstPNodeConfig();
    for ( ; pnodeConfig; pnodeConfig = pnodeConfig->GetNext() )
    {
        pnid = pnodeConfig->GetPNid();
        rank = 0;

        if ( pnodeConfig->IsSpareNode() )
        {
            sparePNids = new int [pnodeConfig->GetSparesCount()+1]; 
            assert( sparePNids != NULL );

            pnodeConfig->GetSpareList( sparePNids );
            // add the spare node's pnid to the spare set
            sparePNids[pnodeConfig->GetSparesCount()] = pnid;
            node = new CNode( (char *)pnodeConfig->GetName()
                            , pnid
                            , rank 
                            , pnodeConfig->GetSparesCount()+1
                            , sparePNids
                            , pnodeConfig->GetExcludedCoreMask() );
            assert( node != NULL );
            for ( int i = 0; i < (pnodeConfig->GetSparesCount()+1) ; i++ )
            {
                if ( Node[sparePNids[i]] && sparePNids[i] != pnid)
                {
                    // Replicate the spare set in each memmber of the set
                    Node[sparePNids[i]]->SetSparePNids( node->GetSparePNids() );
                }
            }
        }
        else
        {
            if (pnid >= maxNode) // only for workstation acting as single node
            {
                rank = -1; // -1 creates node in down state
            }
            node = new CNode( (char *)pnodeConfig->GetName(), pnid, rank );
            assert( node != NULL );
        }
        
        if ( node )
        {
            AddNode( node );
        }

        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
        {
            if ( pnodeConfig->IsSpareNode() )
            {
                for ( int i = 0; i < (pnodeConfig->GetSparesCount()+1) ; i++ )
                {
                    if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                        trace_printf( "%s@%d - spare set member %s (pnid=%d), spare set size=%d, contains:\n"
                                    , method_name, __LINE__
                                    , Node[sparePNids[i]]->GetName()
                                    , Node[sparePNids[i]]->GetPNid()
                                    , (pnodeConfig->GetSparesCount()+1));
                    PNidVector sparePNidsVector = Node[sparePNids[i]]->GetSparePNids();
                    for ( unsigned int jj = 0; jj < sparePNidsVector.size(); jj++ )
                    {
                        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                            trace_printf( "%s@%d - pnid=%d in spare pnids vector, size=%ld\n"
                                        , method_name, __LINE__
                                        , sparePNids[jj]
                                        , sparePNidsVector.size());
                    }
                }
            }
        }

        if ( sparePNids != NULL )
        {
            delete [] sparePNids;
            sparePNids = NULL;
        }
    }

    TRACE_EXIT;
}

void CNodeContainer::AddLNodes( CNode  *node )
{
    const char method_name[] = "CNodeContainer::AddLNodes";
    TRACE_ENTRY;

    CLNode         *lnode;
    CPNodeConfig   *pnodeConfig;
    CLNodeConfig   *lnodeConfig;

    pnodeConfig = clusterConfig_->GetPNodeConfig( node->GetPNid() );
    
    if (pnodeConfig )
    {
        // Create logical nodes configured for the physical node passed in
        lnodeConfig = pnodeConfig->GetFirstLNodeConfig();
        if (lnodeConfig)
        {
            for ( ; lnodeConfig; lnodeConfig = lnodeConfig->GetNextP() )
            {
                // add logical node to 
                //   logical nodes array and logical nodes container
                lnode = AddLNode( lnodeConfig, node );
                LNode[lnodeConfig->GetNid()] = lnode;
                // add logical node to physical node's logical node container 
                node->AddLNodeP( lnode );
            }
        }
        else
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf)
                    , "[%s@%d] Could not find logical node in "
                      "configuration of physical node, pnid=%d\n"
                    , method_name, __LINE__, node->GetPNid() );
            mon_log_write(MON_NODE_ADDLNODES_1, SQ_LOG_ERR, buf);
        }
    }
    else
    {
        char buf[MON_STRING_BUF_SIZE];
        snprintf( buf, sizeof(buf)
                , "[%s@%d] Could not find physical node in configuration, "
                  "pnid=%d\n"
                , method_name, __LINE__, node->GetPNid());
        mon_log_write(MON_NODE_ADDLNODES_2, SQ_LOG_ERR, buf);
    }


    TRACE_EXIT;
}

// add configured lnodes of physical node node2 to node1. 
void CNodeContainer::AddLNodes( CNode  *node1, CNode *node2 )
{
    const char method_name[] = "CNodeContainer::AddLNodes(node1, node2)";
    TRACE_ENTRY;

    CLNode         *lnode;
    CPNodeConfig   *pnodeConfig;
    CLNodeConfig   *lnodeConfig;

    pnodeConfig = clusterConfig_->GetPNodeConfig( node2->GetPNid() );
    if (pnodeConfig )
    {
        // Create logical nodes configured for the physical node passed in
        lnodeConfig = pnodeConfig->GetFirstLNodeConfig();
        if (lnodeConfig)
        {
            for ( ; lnodeConfig; lnodeConfig = lnodeConfig->GetNextP() )
            {
                // add logical node to 
                //   logical nodes array and logical nodes container
                lnode = AddLNode( lnodeConfig, node1 );
                LNode[lnodeConfig->GetNid()] = lnode;
                // add logical node to physical node's logical node container 
                node1->AddLNodeP( lnode );
            }
        }
        else
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf)
                    , "[%s@%d] Could not find logical node in "
                      "configuration of physical node, pnid=%d\n"
                    , method_name, __LINE__, node2->GetPNid() );
            mon_log_write(MON_NODE_ADDLNODES_3, SQ_LOG_ERR, buf);
            abort();
        }
    }
    else
    {
        char buf[MON_STRING_BUF_SIZE];
        snprintf( buf, sizeof(buf)
                , "[%s@%d] Could not find physical node in configuration, "
                  "pnid=%d\n"
                , method_name, __LINE__, node2->GetPNid());
        mon_log_write(MON_NODE_ADDLNODES_4, SQ_LOG_ERR, buf);
        abort();
    }

    TRACE_EXIT;
}

void CNodeContainer::AddToSpareNodesList( int pnid )
{
    const char method_name[] = "CNodeContainer::AddToSpareNodesList";
    TRACE_ENTRY;

    if (trace_settings & TRACE_INIT)
        trace_printf("%s@%d - adding pnid=%d to spare node list\n", method_name, __LINE__, pnid);

    if ( SpareNodeColdStandby )
    {
        spareNodesList_.push_back( Node[pnid] );
    }
    else
    {
        if ( Node[pnid]->GetState() == State_Up && Node[pnid]->IsSpareNode() )
        {
            spareNodesList_.push_back( Node[pnid] );
        }
    }

    if (trace_settings & TRACE_INIT)
    {
        CNode *spareNode;
        NodesList::iterator itSn;
        for ( itSn = spareNodesList_.begin(); itSn != spareNodesList_.end() ; itSn++ ) 
        {
            spareNode = *itSn;
            trace_printf("%s@%d - pnid=%d is in spare node list\n", method_name, __LINE__, spareNode->GetPNid());
        }
    }

    TRACE_EXIT;
}

int CNodeContainer::PackSpareNodesList( intBuffPtr_t &buffer )
{
    const char method_name[] = "CNodeContainer::PackSpareNodesList";
    TRACE_ENTRY;

    NodesList::iterator itSn;
    for ( itSn = spareNodesList_.begin(); itSn != spareNodesList_.end() ; itSn++ )
    {
        *buffer = (*itSn)->GetPNid();

        if (trace_settings & TRACE_INIT)
            trace_printf("%s@%d - packing spare node pnid=%d \n", method_name, __LINE__, *buffer);

        ++buffer;
    }

    TRACE_EXIT;
    return spareNodesList_.size();
}

void CNodeContainer::UnpackSpareNodesList( intBuffPtr_t &buffer, int spareNodesCount )
{
    const char method_name[] = "CNodeContainer::UnpackSpareNodesList";
    TRACE_ENTRY;

    // make sure the list is empty.
    assert(spareNodesList_.size() == 0);

    // reset spareNode_ flag in all nodes.
    for ( int pnid = 0; pnid < GetPNodesCount(); pnid++ )
    {
        Node[indexToPnid_[pnid]]->ResetSpareNode();
    }

    for ( int i=0; i < spareNodesCount; i++ )
    {
        if (trace_settings & TRACE_INIT)
            trace_printf("%s@%d - unpacking spare node pnid=%d \n", method_name, __LINE__, *buffer);

        spareNodesList_.push_back( Node[indexToPnid_[*buffer]] );
        Node[indexToPnid_[*buffer]]->SetSpareNode();
        ++buffer;
    }

    TRACE_EXIT;
    return;
}

int CNodeContainer::PackNodeMappings( intBuffPtr_t &buffer )
{
    const char method_name[] = "CNodeContainer::PackNodeMappings";
    TRACE_ENTRY;

    int pnid, pnidConfig;
    CLNode *lnode = NULL;
    CLNodeConfig *lnodeConfig = NULL;
    int count = 0;

    // go thru all configured physical nodes; and for their logical nodes,
    // find the physical nodes on which they are running.
    CPNodeConfig *pnodeConfig = clusterConfig_->GetFirstPNodeConfig();
    for ( ; pnodeConfig; pnodeConfig = pnodeConfig->GetNext() )
    {
        pnidConfig = pnodeConfig->GetPNid();
        lnodeConfig = pnodeConfig->GetFirstLNodeConfig();
        if ( lnodeConfig )
        {
            lnode = Nodes->GetLNode( lnodeConfig->GetNid() );
            pnid =  lnode->GetNode()->GetPNid();

            *buffer++ = pnidConfig;
            *buffer++ = pnid;

            ++count;

            if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
                trace_printf("%s@%d - Packing node mapping, pnidConfig=%d, pnid=%d \n",
                            method_name, __LINE__, pnidConfig, pnid);
        }
    }

    TRACE_EXIT;

    return count;
}

void CNodeContainer::UnpackNodeMappings( intBuffPtr_t &buffer, int nodeMapCount )
{
    const char method_name[] = "CNodeContainer::UnpackNodeMappings";
    TRACE_ENTRY;

    int pnid, pnidConfig;

    // lock sync thread since we are making a change the monitor's
    // operational view of the cluster
    if ( !Emulate_Down )
    {
        Monitor->EnterSyncCycle();
    }

    for (int count = 0; count < nodeMapCount; count++)
    {
        pnidConfig = *buffer++;
        pnid = *buffer++;

        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            trace_printf("%s@%d - Unpacking node mapping, pnidConfig=%d, pnid=%d \n",
                        method_name, __LINE__, pnidConfig, pnid);

#ifndef NAMESERVER_PROCESS
        Nodes->AddLNodes( Nodes->GetNode(pnid), Nodes->GetNode(pnidConfig) );
#endif
    }

    UpdateCluster();

    // unlock sync thread
    if ( !Emulate_Down )
    {
        Monitor->ExitSyncCycle();
    }

    TRACE_EXIT;
    return;
}

void CNodeContainer::PackZids( intBuffPtr_t &buffer )
{
    const char method_name[] = "CNodeContainer::PackZids";
    TRACE_ENTRY;

    // Pack in pairs of pnid:zid to account for missing pnid(s)
    for ( int index = 0; index < GetPNodesCount(); index++ )
    {
        *buffer = Node[indexToPnid_[index]]->GetPNid();
        ++buffer;
        *buffer = Node[indexToPnid_[index]]->GetZone();
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            trace_printf( "%s@%d - Packing zid=%d for indexToPnid_[%d]=%d\n"
                         , method_name, __LINE__
                         , *buffer
                         , index
                         , indexToPnid_[index]);
        ++buffer;
    }

    TRACE_EXIT;
    return;
}

void CNodeContainer::UnpackZids( intBuffPtr_t &buffer )
{
    const char method_name[] = "CNodeContainer::UnpackZids";
    TRACE_ENTRY;

    // Unpack in pairs of pnid:zid to account for missing pnid(s)
    for ( int index = 0; index < GetPNodesCount(); index++ )
    {
        int pnid = *buffer;
        ++buffer;
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY))
            trace_printf( "%s@%d - Unpacking zid=%d pnid=%d\n"
                         , method_name, __LINE__
                         , *buffer
                         , pnid);
        Node[pnid]->SetZone(*buffer);
        ++buffer;
    }

    TRACE_EXIT;
    return;
}

CLNode *CNodeContainer::AssignLNode( CProcess *requester, PROCESSTYPE type, int nid, int not_nid)
{
    int not_zone;
    CLNode *lnode = NULL;
    char la_buf[MON_STRING_BUF_SIZE];

    const char method_name[] = "CNodeContainer::AssignLNode";
    TRACE_ENTRY;

    not_zone = (not_nid == -1?-1:LNode[not_nid]->GetZone());

    if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS))
       trace_printf("%s@%d" " - nid=%d, not_nid=%d, not_zone=%d\n", method_name, __LINE__, nid, not_nid, not_zone);

    switch (type)
    {
    case ProcessType_Backout:
    case ProcessType_VolumeRecovery:
        if (nid == -1)
        {
            lnode = SelectLNode(requester, ZoneType_Storage, nid, not_zone, true);
        }
        else
        {
            if ((LNode[nid]->GetState() == State_Up || 
                 LNode[nid]->GetState() == State_Shutdown) &&
                !LNode[nid]->IsKillingNode() &&
                (LNode[nid]->GetZoneType() == ZoneType_Any ||
                 LNode[nid]->GetZoneType() == ZoneType_Backend ||
                 LNode[nid]->GetZoneType() == ZoneType_Storage) )
            {
                lnode = LNode[nid]; // use the configured nid passed in
            }
        }
        break;
    case ProcessType_TSE:
    case ProcessType_ASE:
        if (nid == -1)
        {
            lnode = SelectLNode(requester, ZoneType_Storage, nid, not_zone, true);
        }
        else
        {
            if ((LNode[nid]->GetState() == State_Up || 
                 LNode[nid]->GetState() == State_Shutdown) &&
                !LNode[nid]->IsKillingNode() &&
                (LNode[nid]->GetZoneType() == ZoneType_Any ||
                 LNode[nid]->GetZoneType() == ZoneType_Backend ||
                 LNode[nid]->GetZoneType() == ZoneType_Storage) )
            {
                lnode = LNode[nid]; // use the configured nid passed in
            }
        }
        break;
    case ProcessType_DTM:
    case ProcessType_SPX:
    case ProcessType_SSMP:
    case ProcessType_Watchdog:
    case ProcessType_PSD:
        if (nid == -1)
        {
            sprintf(la_buf, "[CNodeContainer::AssignLNode], Can't dynamically assign Nid for type (DTM, SPX, or Watchdog) assuming nid=0.\n");
            mon_log_write(MON_NODECONT_ASSIGN_LNODE, SQ_LOG_ERR, la_buf);

            nid = 0;
        }
        if ((LNode[nid]->GetState() == State_Up || 
             LNode[nid]->GetState() == State_Shutdown) &&
            !LNode[nid]->IsKillingNode() )
        {
            lnode = LNode[nid]; // use the configured nid passed in
        }
        break;
    case ProcessType_Generic:
        if (nid == -1)
        {
            lnode = SelectLNode(requester, ZoneType_Aggregation, nid, not_zone, true);
        }
        else
        {
            if ((LNode[nid]->GetState() == State_Up || 
                 LNode[nid]->GetState() == State_Shutdown) &&
                !LNode[nid]->IsKillingNode() )
            {
                lnode = LNode[nid]; // use the configured nid passed in
            }
        }
        break;
    case ProcessType_MXOSRVR:
        if (nid == -1)
        {
            lnode = SelectLNode(requester, ZoneType_Edge, nid, not_zone, false);
        }
        else
        {
            if ((LNode[nid]->GetState() == State_Up || 
                 LNode[nid]->GetState() == State_Shutdown)  &&
                !LNode[nid]->IsKillingNode() &&
                (LNode[nid]->GetZoneType() == ZoneType_Any ||
                 LNode[nid]->GetZoneType() == ZoneType_Frontend ||
                 LNode[nid]->GetZoneType() == ZoneType_Edge) )
            {
                lnode = LNode[nid]; // use the configured nid passed in
            }
        }
        break;
    default:
        if (trace_settings & TRACE_REQUEST)
           trace_printf("%s@%d" " - ProcessType=" "%d" "\n", method_name, __LINE__, type);
        lnode = SelectLNode(requester, ZoneType_Any, nid, not_zone, true);
    }

    TRACE_EXIT;
    return lnode;
}

void CNodeContainer::AvgNodeData(ZoneType type, int *avg_pcount, unsigned int *avg_memory)
{
    int     lnodes = 0;
    CNode  *node;
    
    const char method_name[] = "CNodeContainer::AvgNodeData";
    TRACE_ENTRY;

    CLNode *lnode = Nodes->GetFirstLNode();
    for ( ; lnode ; lnode = lnode->GetNext() )
    {
        node = lnode->GetNode();
        // average only over node of requested type and available logical nodes
        if ( (lnode->GetZoneType() & type) &&
             (lnode->GetState() == State_Up &&
             !lnode->IsKillingNode()) )
        {
            *avg_pcount += lnode->GetNumProcs();
            *avg_memory += (node->GetFreeCache() + node->GetFreeSwap());
            lnodes++;
        }
    }

    if ( lnodes )
    {
        *avg_pcount /= lnodes;
        *avg_memory /= lnodes;
    }
    else
    {
        *avg_pcount = 0;
        *avg_memory = 0;
    }

    TRACE_EXIT;
}

void CNodeContainer::ChangedNode( CNode *node )
{
    const char method_name[] = "CNodeContainer::ChangedNode";
    TRACE_ENTRY;

    if (trace_settings & (TRACE_REQUEST | TRACE_SYNC))
    {
        trace_printf( "%s@%d - node_name=%s, pnid=%d, zone=%d\n"
                    , method_name, __LINE__
                    , node->GetName()
                    , node->GetPNid()
                    , node->GetZone() );
    }

    assert( node->GetState() == State_Down );

    CClusterConfig *clusterConfig = Nodes->GetClusterConfig();
    CLNodeConfig   *lnodeConfig = NULL;

    // Broadcast node changed notice to local processes
    CLNode *lnode = node->GetFirstLNode();
    for ( ; lnode; lnode = lnode->GetNextP() )
    {
        lnodeConfig = clusterConfig->GetLNodeConfig( lnode->GetNid() );
        lnode->Changed( lnodeConfig );
    }

    TRACE_EXIT;
}

#ifndef NAMESERVER_PROCESS
void CNodeContainer::CancelDeathNotification( int nid
                                            , int pid
                                            , int verifier
                                            , _TM_Txid_External trans_id)
{
    CNode *node;
    
    const char method_name[] = "CNodeContainer::CancelDeathNotification";
    TRACE_ENTRY;

    for ( node=head_; node; node=node->GetNext() )
    {
        node->CLNodeContainer::CancelDeathNotification( nid
                                                      , pid
                                                      , verifier
                                                      , trans_id);
    }

    TRACE_EXIT;
}
#endif
   
#ifndef NAMESERVER_PROCESS
CProcess *CNodeContainer::CloneProcessNs( int nid
                                        , int pid
                                        , Verifier_t verifier )
{
    const char method_name[] = "CNodeContainer::CloneProcessNs";
    TRACE_ENTRY;

    CProcess *process = NULL;

    struct message_def msg;
    msg.type = MsgType_Service;
    msg.noreply = false;
    msg.reply_tag = REPLY_TAG;
    msg.u.request.type = ReqType_ProcessInfoNs;

    struct ProcessInfo_def *processInfo = &msg.u.request.u.process_info;
    processInfo->nid = -1;
    processInfo->pid = -1;
    processInfo->verifier = -1;
    processInfo->process_name[0] = 0;
    processInfo->target_nid = nid;
    processInfo->target_pid = pid;
    processInfo->target_verifier = verifier;
    processInfo->target_process_name[0] = 0;
    processInfo->target_process_pattern[0] = 0;
    processInfo->type = ProcessType_Undefined;
    
    int error = NameServer->ProcessInfoNs(&msg); // in reqQueue thread (CExternalReq)
    if (error == 0)
    {
        if ( (msg.type == MsgType_Service) &&
             (msg.u.reply.type == ReplyType_ProcessInfoNs) )
        {
            if ( msg.u.reply.u.process_info_ns.return_code == MPI_SUCCESS )
            {
                process = AddCloneProcess( &msg.u.reply.u.process_info_ns );
            }
            else
            {
                char buf[MON_STRING_BUF_SIZE];
                snprintf( buf, sizeof(buf),
                          "[%s] ProcessInfo failed, rc=%d\n"
                        , method_name, msg.u.reply.u.process_info_ns.return_code );
                mon_log_write( MON_NODE_CLONEPROCESSNS_1, SQ_LOG_ERR, buf );
            }
        }
        else
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf),
                      "[%s], Invalid MsgType(%d)/ReplyType(%d) for "
                      "ProcessInfoNs\n"
                    , method_name, msg.type, msg.u.reply.type );
            mon_log_write( MON_NODE_CLONEPROCESSNS_2, SQ_LOG_ERR, buf );
        }
    }
    else
    {
        char la_buf[MON_STRING_BUF_SIZE];
        snprintf( la_buf, sizeof(la_buf)
                , "[%s] - Process info request to Name Server failed\n"
                , method_name );
        mon_log_write( MON_NODE_CLONEPROCESSNS_3, SQ_LOG_ERR, la_buf );
    }

    TRACE_EXIT;
    return( process );
}
#endif
   
#ifndef NAMESERVER_PROCESS
CProcess *CNodeContainer::CloneProcessNs( const char *name, Verifier_t verifier )
{
    const char method_name[] = "CNodeContainer::CloneProcessNs";
    TRACE_ENTRY;

    CProcess *process = NULL;

    struct message_def msg;
    msg.type = MsgType_Service;
    msg.noreply = false;
    msg.reply_tag = REPLY_TAG;
    msg.u.request.type = ReqType_ProcessInfoNs;

    struct ProcessInfo_def *processInfo = &msg.u.request.u.process_info;
    processInfo->nid = -1;
    processInfo->pid = -1;
    processInfo->verifier = -1;
    processInfo->process_name[0] = 0;
    processInfo->target_nid = -1;
    processInfo->target_pid = -1;
    processInfo->target_verifier = verifier;
    STRCPY( processInfo->target_process_name, name);
    processInfo->target_process_pattern[0] = 0;
    processInfo->type = ProcessType_Undefined;

    int error = NameServer->ProcessInfoNs(&msg); // in reqQueue thread (CExternalReq)
    if (error == 0)
    {
        if ( (msg.type == MsgType_Service) &&
             (msg.u.reply.type == ReplyType_ProcessInfoNs) )
        {
            if ( msg.u.reply.u.process_info_ns.return_code == MPI_SUCCESS )
            {
                process = AddCloneProcess( &msg.u.reply.u.process_info_ns );
            }
            else
            {
                char buf[MON_STRING_BUF_SIZE];
                snprintf( buf, sizeof(buf),
                          "[%s] ProcessInfo failed, rc=%d\n"
                        , method_name, msg.u.reply.u.process_info_ns.return_code );
                mon_log_write( MON_NODE_CLONEPROCESSNS_4, SQ_LOG_ERR, buf );
            }
        }
        else
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf),
                      "[%s], Invalid MsgType(%d)/ReplyType(%d) for "
                      "ProcessInfo\n"
                    , method_name, msg.type, msg.u.reply.type );
            mon_log_write( MON_NODE_CLONEPROCESSNS_5, SQ_LOG_ERR, buf );
        }
    }
    else
    {
        char la_buf[MON_STRING_BUF_SIZE];
        snprintf( la_buf, sizeof(la_buf)
                , "[%s] - Process info request to Name Server failed\n"
                , method_name );
        mon_log_write( MON_NODE_CLONEPROCESSNS_6, SQ_LOG_ERR, la_buf );
    }

    TRACE_EXIT;
    return( process );
}
#endif
   
#ifndef NAMESERVER_PROCESS
void CNodeContainer::DeleteCloneProcess( CProcess *process )
{
    const char method_name[] = "CNodeContainer::DeleteCloneProcess";
    TRACE_ENTRY;

    CNode *node;
    node = Nodes->GetLNode(process->GetNid())->GetNode();
    node->DelFromNameMap ( process );
    node->DelFromPidMap ( process );
    node->DeleteFromList( process );

    TRACE_EXIT;
}
#endif

void CNodeContainer::DeletedNode( CNode *node )
{
    const char method_name[] = "CNodeContainer::DeletedNode";
    TRACE_ENTRY;

    if (trace_settings & (TRACE_REQUEST | TRACE_SYNC))
    {
        trace_printf( "%s@%d - node %s, pnid=%d, zone=%d\n"
                    , method_name, __LINE__
                    , node->GetName(), node->GetPNid(), node->GetZone() );
    }

    assert( node->GetState() == State_Down );

    // Broadcast node deleted notice to local processes
    CLNode *lnode = node->GetFirstLNode();
    for ( ; lnode; lnode = lnode->GetNextP() )
    {
        lnode->Deleted();
    }

    TRACE_EXIT;
}

bool CNodeContainer::DeleteNode( int pnid )
{
    const char method_name[] = "CNodeContainer::DeleteNode";
    TRACE_ENTRY;

    int rs = true;

    CNode *pnode = GetNode( pnid );
    if ( pnode )
    {
        // Broadcast node deleted notice to local processes
        Nodes->DeletedNode( pnode );

        // Now delete it from the monitor's view
        Nodes->DeleteNode( pnode );
        // Verify it was deleted, sanity check!
        assert( (pnode = Nodes->GetNode( pnid )) == NULL );
    }
    else
    {
        char buf[MON_STRING_BUF_SIZE];
        snprintf( buf, sizeof(buf)
                , "[%s@%d] Could not find physical node "
                  "in monitor's view of cluster, pnid=%d\n"
                , method_name, __LINE__, pnid);
        mon_log_write(MON_NODE_DELETENODE_1, SQ_LOG_ERR, buf);
        rs = false;
    }

    TRACE_EXIT;
    return( rs );
}

void CNodeContainer::DeleteNode( CNode  *node )
{
    const char method_name[] = "CNodeContainer::DeleteNode";
    TRACE_ENTRY;

    int pnid = node->GetPNid();

    if (trace_settings & (TRACE_INIT | TRACE_REQUEST))
    {
        trace_printf( "%s@%d deleting (pnid=%d), pnodesCount=%d, lnodesCount=%d\n"
                     , method_name, __LINE__
                     , pnid
                     , GetPNodesCount()
                     , GetLNodesCount() );
    }

    // lock sync thread
    if ( !Emulate_Down )
    {
        Monitor->EnterSyncCycle();
    }

    // delete logical nodes owned by physical node
    DeleteNodeLNodes( node );

    // delete physical node and remove from physical nodes array
    node->DeLink (&head_, &tail_);
    delete node;
    Node[pnid] = NULL;

    // Decrement the physical node count
    pnodeCount_--;
    
    // unlock sync thread
    if ( !Emulate_Down )
    {
        Monitor->ExitSyncCycle();
    }

    if (trace_settings & (TRACE_INIT | TRACE_REQUEST))
    {
        trace_printf( "%s@%d deleted (pnid=%d), pnodesCount=%d, lnodesCount=%d\n"
                     , method_name, __LINE__
                     , pnid
                     , GetPNodesCount()
                     , GetLNodesCount() );
    }

    TRACE_EXIT;
}

void CNodeContainer::DeleteNodeLNodes( CNode *node )
{
    const char method_name[] = "CNodeContainer::DeleteNodeLNode";
    TRACE_ENTRY;


    CLNode *lnode = node->GetFirstLNode();
    while ( lnode )
    {
        int nid = lnode->GetNid();
        LNode[nid] = NULL;
        // Remove logical node from physical node
        node->RemoveLNodeP( lnode );
        // Delete logical node
        Nodes->DeleteLNode( lnode );
        lnode = node->GetFirstLNode();
    }
    
    TRACE_EXIT;
}

int CNodeContainer::GetFirstNid( void )
{
    const char method_name[] = "CNodeContainer::GetFirstNid";
    TRACE_ENTRY;

    int target = -1;
    for (int i = 0; i <  clusterConfig_->GetLNodesConfigMax(); i++ )
    {
        CLNode *lnode = LNode[i];
        if ( lnode )
        {
            target = lnode->GetNid();
            break;
        }
    }

    TRACE_EXIT;
    return( target );
}

int CNodeContainer::GetNextNid( int nid )
{
    const char method_name[] = "CNodeContainer::GetNextNid";
    TRACE_ENTRY;

    int target = -1;
    for (int i = (nid+1); i <  clusterConfig_->GetLNodesConfigMax(); i++ )
    {
        CLNode *lnode = LNode[i];
        if ( lnode )
        {
            target = lnode->GetNid();
            break;
        }
    }

    TRACE_EXIT;
    return( target );
}

CNode *CNodeContainer::GetNode(int pnid)
{
    const char method_name[] = "CNodeContainer::GetNode";
    TRACE_ENTRY;

    CNode *node = head_;
    while (node)
    {
        if ( node->GetPNid() == pnid )
        { 
            break;
        }
        node = node->GetNext();
    }

    TRACE_EXIT;
    return node;
}

CNode *CNodeContainer::GetNodeByMap(int index )
{
    const char method_name[] = "CNodeContainer::GetNodeByMap";
    TRACE_ENTRY;

    CNode *node = NULL;
    
    if( index >= 0 && index < GetPNodesCount() )
    {
        node = Node[indexToPnid_[index]];
    }

    TRACE_EXIT;
    return node;
}

CNode *CNodeContainer::GetNode(char *name )
{
    CNode *node = head_;
    const char method_name[] = "CNodeContainer::GetNode";
    TRACE_ENTRY;

    while (node)
    {
        if ( CPNodeConfigContainer::hostnamecmp( node->GetName(), name) == 0 )
        { 
            break;
        }
        node = node->GetNext();
    }

    TRACE_EXIT;
    return node;
}

CNode *CNodeContainer::GetNode(char *process_name, CProcess ** process,
                               bool checkstate)
{
    CNode *node = head_;
    CProcess *p_process;
    const char method_name[] = "CNodeContainer::GetNode";
    TRACE_ENTRY;

    while (node)
    {
        *process = node->CProcessContainer::GetProcess (process_name,
                                                        checkstate);
        if (*process)
        { 
            p_process = *process;
            if (!p_process->IsBackup())
                break;
        }
        node = node->GetNext ();
    }

    TRACE_EXIT;
    return node;
}

CProcess *CNodeContainer::GetProcess( int nid, int pid, bool checknode )
{
    const char method_name[] = "CNodeContainer::GetProcess";
    TRACE_ENTRY;

    CProcess *process = NULL;
    CLNode   *lnode = GetLNode( nid );

    if ( lnode )
    {
        if ( checknode )
        {
            if ( lnode->GetState() == State_Up ||
                 lnode->GetState() == State_Shutdown )
            {
                process = lnode->GetProcessL( pid );
            }
            else
            {
                if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS_DETAIL))
                   trace_printf("%s@%d" " - Node is not up, nid=%d, state=%d, \n", method_name, __LINE__, lnode->GetNid(), lnode->GetState());
            }
        }
        else
        {
            process = lnode->GetProcessL( pid );
        }
    }

    TRACE_EXIT;
    return( process );
}

CProcess *CNodeContainer::GetProcess( int nid
                                    , int pid
                                    , Verifier_t verifier
                                    , bool checknode
                                    , bool checkprocess
                                    , bool backupOk )
{
    const char method_name[] = "CNodeContainer::GetProcess(nid,pid,verifier)";
    TRACE_ENTRY;

    CProcess *process = NULL;

    if ( nid != -1 )
    {
        CLNode   *lnode = GetLNode( nid );
        if ( lnode )
        {
            if ( checknode )
            {
                if ( lnode->GetState() == State_Up ||
                     lnode->GetState() == State_Shutdown )
                {
                    process = lnode->GetProcessL( pid, verifier, checkprocess );
                    if ( process ) 
                    {
                        if ( nid != process->GetNid() )
                        {
                           if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS_DETAIL))
                           {
                              trace_printf( "%s@%d - Get (%d, %d:%d) failed -- nid mismatch (%d)\n"
                                          , method_name, __LINE__
                                          , nid
                                          , pid
                                          , verifier
                                          , process->GetNid() );
                           }            
                           process = NULL;
                        }
                        else if ( !backupOk && process->IsBackup() )
                        {
                            if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS_DETAIL))
                            {
                                trace_printf( "%s@%d - Get (%d, %d:%d) failed -- backupOk=%d backup=%d\n"
                                            , method_name, __LINE__
                                            , nid
                                            , pid
                                            , verifier
                                            , backupOk
                                            , process->IsBackup() );
                            }
                            process = NULL;
                        }
                    }
                    else
                    {
                        if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS_DETAIL))
                        {
                           trace_printf( "%s@%d - Get (%d, %d:%d) failed\n"
                                       , method_name, __LINE__
                                       , nid
                                       , pid
                                       , verifier );
                        }
                    }
                }
                else
                {
                    if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS_DETAIL))
                       trace_printf("%s@%d" " - Node is not up, nid=%d, state=%d, \n", method_name, __LINE__, lnode->GetNid(), lnode->GetState());
                }
            }
            else
            {
                process = lnode->GetProcessL( pid, verifier, checkprocess );
                if ( process ) 
                {
                    if ( nid != process->GetNid() )
                    {
                       if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS_DETAIL))
                       {
                          trace_printf( "%s@%d - Get (%d, %d:%d) failed -- nid mismatch (%d)\n"
                                      , method_name, __LINE__
                                      , nid
                                      , pid
                                      , verifier
                                      , process->GetNid() );
                       }            
                       process = NULL;
                    }
                    else if ( !backupOk && process->IsBackup() )
                    {
                        if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS_DETAIL))
                        {
                            trace_printf( "%s@%d - Get (%d, %d:%d) failed -- backupOk=%d backup=%d\n"
                                        , method_name, __LINE__
                                        , nid
                                        , pid
                                        , verifier
                                        , backupOk
                                        , process->IsBackup() );
                        }
                        process = NULL;
                    }
                }
                else
                {
                    if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS_DETAIL))
                    {
                       trace_printf( "%s@%d - Get (%d, %d:%d) failed\n"
                                   , method_name, __LINE__
                                   , nid
                                   , pid
                                   , verifier );
                    }            
                }
            }
        }
    }

    TRACE_EXIT;
    return( process );
}

CProcess *CNodeContainer::GetProcess( const char *name
                                    , Verifier_t verifier
                                    , bool checknode
                                    , bool checkprocess
                                    , bool backupOk )
{
    CNode *node = head_;
    CProcess *process = NULL;
    const char method_name[] = "CNodeContainer::GetProcess(name,verifier)";
    TRACE_ENTRY;

    if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS_DETAIL))
    {
        trace_printf( "%s@%d Getting %s:%d, "
                      "checknode=%d, checkprocess=%d, backupOk=%d\n"
                    , method_name, __LINE__
                    , name
                    , verifier
                    , checknode
                    , checkprocess
                    , backupOk );
    }

    while ( node )
    {
        if ( checknode )
        {
            if ( node->GetState() == State_Up ||
                 node->GetState() == State_Shutdown )
            {
                process = node->CProcessContainer::GetProcess( name
                                                             , verifier
                                                             , checkprocess );
                if ( process ) 
                {
                    if ( !backupOk && process->IsBackup() )
                    {
                        if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS_DETAIL))
                        {
                            trace_printf( "%s@%d - Get (%s:%d) failed -- backupOk=%d backup=%d\n"
                                        , method_name, __LINE__
                                        , name
                                        , verifier
                                        , backupOk
                                        , process->IsBackup() );
                        }
                        process = NULL;
                    }
                    else
                    {
                        if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS_DETAIL))
                        {
                            trace_printf( "%s@%d - Found (%s:%d) (%d, %d) backup=%d\n"
                                        , method_name, __LINE__
                                        , name
                                        , verifier
                                        , process->GetNid()
                                        , process->GetPid()
                                        , process->IsBackup() );
                        }
                        break;
                    }
                }
                else
                {
                    if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS_DETAIL))
                    {
                       trace_printf( "%s@%d - Get (%s:%d) in pnid=%d failed\n"
                                   , method_name, __LINE__
                                   , name
                                   , verifier
                                   , node->GetPNid() );
                    }            
                }
            }
            else
            {
                if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS_DETAIL))
                   trace_printf("%s@%d" " - Node is not up, pnid=%d, state=%d, \n", method_name, __LINE__, node->GetPNid(), node->GetState());
            }
        }
        else
        {
            process = node->CProcessContainer::GetProcess( name
                                                         , verifier
                                                         , checkprocess );
            if ( process)
            { 
                if ( !backupOk && process->IsBackup() )
                {
                    if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS_DETAIL))
                    {
                        trace_printf( "%s@%d - Get (%s:%d) failed -- backupOk=%d backup=%d\n"
                                    , method_name, __LINE__
                                    , name
                                    , verifier
                                    , backupOk
                                    , process->IsBackup() );
                    }
                    process = NULL;
                }
                else
                {
                    break;
                }
            }
            else
            {
                if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS_DETAIL))
                {
                   trace_printf( "%s@%d - Get (%s:%d) failed\n"
                               , method_name, __LINE__
                               , name
                               , verifier );
                }            
            }
        }
        node = node->GetNext ();
    }

    TRACE_EXIT;
    return( process );
}

CProcess *CNodeContainer::GetProcessByName( const char *name, bool checkstate )
{
    CNode *node = head_;
    CProcess *process = NULL;
    const char method_name[] = "CNodeContainer::GetProcessByName";
    TRACE_ENTRY;

    while (node)
    {
        process = node->CProcessContainer::GetProcess( name, checkstate );
        if (process)
        { 
            if (!process->IsBackup())
                break;
        }
        node = node->GetNext ();
    }

    TRACE_EXIT;
    return( process );
}

#ifndef NAMESERVER_PROCESS
int CNodeContainer::GetProcessInfoNs( int nid
                                    , int pid
                                    , Verifier_t verifier
                                    , ProcessInfoNs_reply_def *processInfo )
{
    const char method_name[] = "CNodeContainer::GetProcessInfoNs";
    TRACE_ENTRY;

    int rc = MPI_SUCCESS;

    struct message_def msg;
    msg.type = MsgType_Service;
    msg.noreply = false;
    msg.reply_tag = REPLY_TAG;
    msg.u.request.type = ReqType_ProcessInfoNs;

    struct ProcessInfo_def *process_info = &msg.u.request.u.process_info;
    process_info->nid = -1;
    process_info->pid = -1;
    process_info->verifier = -1;
    process_info->process_name[0] = 0;
    process_info->target_nid = nid;
    process_info->target_pid = pid;
    process_info->target_verifier = verifier;
    process_info->target_process_name[0] = 0;
    process_info->target_process_pattern[0] = 0;
    process_info->type = ProcessType_Undefined;
    
    int error = NameServer->ProcessInfoNs(&msg); // in reqQueue thread (CExternalReq)
    if (error == 0)
    {
        if ( (msg.type == MsgType_Service) &&
             (msg.u.reply.type == ReplyType_ProcessInfoNs) )
        {
            if ( msg.u.reply.u.process_info_ns.return_code == MPI_SUCCESS )
            {
                *processInfo = msg.u.reply.u.process_info_ns;
            }
            else
            {
                char buf[MON_STRING_BUF_SIZE];
                snprintf( buf, sizeof(buf),
                          "[%s] ProcessInfo failed, rc=%d\n"
                        , method_name, msg.u.reply.u.process_info_ns.return_code );
                mon_log_write( MON_NODE_GETPROCESSNS_1, SQ_LOG_ERR, buf );
            }
            rc = msg.u.reply.u.process_info_ns.return_code;
        }
        else
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf),
                      "[%s], Invalid MsgType(%d)/ReplyType(%d) for "
                      "ProcessInfoNs\n"
                    , method_name, msg.type, msg.u.reply.type );
            mon_log_write( MON_NODE_GETPROCESSNS_2, SQ_LOG_ERR, buf );
            rc = MPI_ERR_OP;
        }
    }
    else
    {
        char la_buf[MON_STRING_BUF_SIZE];
        snprintf( la_buf, sizeof(la_buf)
                , "[%s] - Process info request to Name Server failed\n"
                , method_name );
        mon_log_write( MON_NODE_GETPROCESSNS_3, SQ_LOG_ERR, la_buf );
        rc = MPI_ERR_OP;
    }

    TRACE_EXIT;
    return( rc );
}

int CNodeContainer::GetProcessInfoNs( const char *name
                                    , Verifier_t verifier
                                    , ProcessInfoNs_reply_def *processInfo )
{
    const char method_name[] = "CNodeContainer::GetProcessInfoNs";
    TRACE_ENTRY;

    int rc = MPI_SUCCESS;

    struct message_def msg;
    msg.type = MsgType_Service;
    msg.noreply = false;
    msg.reply_tag = REPLY_TAG;
    msg.u.request.type = ReqType_ProcessInfoNs;

    struct ProcessInfo_def *process_info = &msg.u.request.u.process_info;
    process_info->nid = -1;
    process_info->pid = -1;
    process_info->verifier = -1;
    process_info->process_name[0] = 0;
    process_info->target_nid = -1;
    process_info->target_pid = -1;
    process_info->target_verifier = verifier;
    STRCPY( process_info->target_process_name, name);
    process_info->target_process_pattern[0] = 0;
    process_info->type = ProcessType_Undefined;

    int error = NameServer->ProcessInfoNs(&msg); // in reqQueue thread (CExternalReq)
    if (error == 0)
    {
        if ( (msg.type == MsgType_Service) &&
             (msg.u.reply.type == ReplyType_ProcessInfoNs) )
        {
            if ( msg.u.reply.u.process_info_ns.return_code == MPI_SUCCESS )
            {
                *processInfo = msg.u.reply.u.process_info_ns;
            }
            else
            {
                char buf[MON_STRING_BUF_SIZE];
                snprintf( buf, sizeof(buf),
                          "[%s] ProcessInfo failed, rc=%d\n"
                        , method_name, msg.u.reply.u.process_info_ns.return_code );
                mon_log_write( MON_NODE_GETPROCESSNS_4, SQ_LOG_ERR, buf );
            }
            rc = msg.u.reply.u.process_info_ns.return_code;
        }
        else
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf),
                      "[%s], Invalid MsgType(%d)/ReplyType(%d) for "
                      "ProcessInfo\n"
                    , method_name, msg.type, msg.u.reply.type );
            mon_log_write( MON_NODE_GETPROCESSNS_5, SQ_LOG_ERR, buf );
            rc = MPI_ERR_OP;
        }
    }
    else
    {
        char la_buf[MON_STRING_BUF_SIZE];
        snprintf( la_buf, sizeof(la_buf)
                , "[%s] - Process info request to Name Server failed\n"
                , method_name );
        mon_log_write( MON_NODE_GETPROCESSNS_6, SQ_LOG_ERR, la_buf );
        rc = MPI_ERR_OP;
    }

    TRACE_EXIT;
    return( rc );
}

CProcess *CNodeContainer::GetProcessLByTypeNs( int nid, PROCESSTYPE type )
{
    const char method_name[] = "CNodeContainer::GetProcessLByTypeNs";
    TRACE_ENTRY;

    CProcess *process = NULL;

    struct message_def msg;
    msg.type = MsgType_Service;
    msg.noreply = false;
    msg.reply_tag = REPLY_TAG;
    msg.u.request.type = ReqType_ProcessInfoNs;

    struct ProcessInfo_def *processInfo = &msg.u.request.u.process_info;
    processInfo->nid = -1;
    processInfo->pid = -1;
    processInfo->verifier = -1;
    processInfo->process_name[0] = 0;
    processInfo->target_nid = nid;
    processInfo->target_pid = -1;
    processInfo->target_verifier = -1;
    processInfo->target_process_name[0] = 0;
    processInfo->target_process_pattern[0] = 0;
    processInfo->type = type;

    if ( trace_settings & ( TRACE_PROCESS | TRACE_REQUEST) )
    {
        trace_printf( "%s@%d - Received monitor request process-info-ns data.\n"
                      "        process_info.nid=%d\n"
                      "        process_info.pid=%d\n"
                      "        process_info.verifier=%d\n"
                      "        process_info.target_nid=%d\n"
                      "        process_info.target_pid=%d\n"
                      "        process_info.target_verifier=%d\n"
                      "        process_info.target_process_name=%s\n"
                      "        process_info.target_process_pattern=%s\n"
                      "        process_info.type=%d\n"
                    , method_name, __LINE__
                    , processInfo->nid
                    , processInfo->pid
                    , processInfo->verifier
                    , processInfo->target_nid
                    , processInfo->target_pid
                    , processInfo->target_verifier
                    , processInfo->target_process_name
                    , processInfo->target_process_pattern
                    , processInfo->type
                    );
    }

    int error = NameServer->ProcessInfoNs(&msg); // in reqQueue thread (CExternalReq)
    if (error == 0)
    {
        if ( (msg.type == MsgType_Service) &&
             (msg.u.reply.type == ReplyType_ProcessInfoNs) )
        {
            if ( msg.u.reply.u.process_info_ns.return_code == MPI_SUCCESS )
            {
                process = AddCloneProcess( &msg.u.reply.u.process_info_ns );
            }
            else
            {
                char buf[MON_STRING_BUF_SIZE];
                snprintf( buf, sizeof(buf),
                          "[%s] ProcessInfo failed, rc=%d\n"
                        , method_name, msg.u.reply.u.process_info_ns.return_code );
                mon_log_write( MON_NODE_GETPROCESSLBYTYPENS_1, SQ_LOG_ERR, buf );
            }
        }
        else
        {
            char buf[MON_STRING_BUF_SIZE];
            snprintf( buf, sizeof(buf),
                      "[%s], Invalid MsgType(%d)/ReplyType(%d) for "
                      "ProcessInfo\n"
                    , method_name, msg.type, msg.u.reply.type );
            mon_log_write( MON_NODE_GETPROCESSLBYTYPENS_2, SQ_LOG_ERR, buf );
        }
    }
    else
    {
        char la_buf[MON_STRING_BUF_SIZE];
        snprintf( la_buf, sizeof(la_buf)
                , "[%s] - Process info request to Name Server failed\n"
                , method_name );
        mon_log_write( MON_NODE_GETPROCESSLBYTYPENS_3, SQ_LOG_ERR, la_buf );
    }

    TRACE_EXIT;
    return( process );
}

SyncState CNodeContainer::GetTmState ( SyncState check_state )
{
    SyncState state = check_state;
    CNode *node = head_;
    const char method_name[] = "CNodeContainer::GetTmState";
    TRACE_ENTRY;
    
    while (node)
    {
        if ( node->GetState() == State_Up && ! node->IsSpareNode() && node->GetPhase() == Phase_Ready)
        {
            if ( check_state == SyncState_Start )
            {
                if ( node->GetPNid() == MyPNID )
                {
                    if ( node->GetTmSyncState() != SyncState_Start )
                    {
                        state = SyncState_Abort;
                        if (trace_settings & TRACE_TMSYNC)
                           trace_printf("%s@%d" " - Node %s, pnid=%d" " no longer in Master Sync Start state" "\n", method_name, __LINE__, node->GetName(), node->GetPNid());
                        break;
                    }
                }
                else
                {
                    if ( node->GetTmSyncState() != SyncState_Continue )
                    {
                        state = SyncState_Abort;
                        if (trace_settings & TRACE_TMSYNC)
                           trace_printf("%s@%d" " - Node %s, pnid=%d" " doesn't agree on Sync Start state, returned state=" "%d" "\n", method_name, __LINE__, node->GetName(), node->GetPNid(), node->GetTmSyncState());
                        break;
                    }
                }
            }
            else
            {
                if ( check_state == SyncState_Suspended )
                {
                    state = node->GetTmSyncState();
                    if ( state == SyncState_Suspended )
                    {
                        if (trace_settings & TRACE_TMSYNC)
                           trace_printf("%s@%d" " - Node %s, pnid=%d" " is in TmSync Suspended state\n", method_name, __LINE__, node->GetName(), node->GetPNid());
                        break;
                    }
                }
                else if ( node->GetTmSyncState() != check_state )
                {
                    state = node->GetTmSyncState();
                    if (trace_settings & TRACE_TMSYNC)
                       trace_printf("%s@%d" " - Node %s, pnid=%d" " doesn't agree on TmState, returned state=" "%d" "\n", method_name, __LINE__, node->GetName(), node->GetPNid(), state);
                    break;
                }
            }
        }
        node = node->GetNext ();
    }
    
    TRACE_EXIT;
    return state;
}
#endif

CNode *CNodeContainer::GetZoneNode(int zid)
{
    CNode *node = NULL;
    const char method_name[] = "CNodeContainer::GetZoneNode";
    TRACE_ENTRY;

    for ( int pnid = 0; pnid < GetPNodesCount(); pnid++ )
    {
        if ( ! Node[indexToPnid_[pnid]]->IsSpareNode() 
            && Node[indexToPnid_[pnid]]->GetZone() == zid )
        {
            node = Node[indexToPnid_[pnid]];
            break;
        }
    }

    TRACE_EXIT;
    return node;
}

struct internal_msg_def *
CNodeContainer::InitSyncBuffer( struct sync_buffer_def *syncBuf
                              , unsigned long long seqNum
                              , upNodes_t upNodes )
{
    const char method_name[] = "CNodeContainer::InitSyncBuffer";
    TRACE_ENTRY;

    struct internal_msg_def *msg;

    syncBuf->nodeInfo.node_state    = MyNode->GetState();
    syncBuf->nodeInfo.sdLevel       = MyNode->GetShutdownLevel();
#ifndef NAMESERVER_PROCESS
    syncBuf->nodeInfo.tmSyncState   = MyNode->GetTmSyncState();
#endif
    syncBuf->nodeInfo.internalState = MyNode->getInternalState();
    syncBuf->nodeInfo.change_nid    = -1;
    syncBuf->nodeInfo.seq_num       = seqNum;
    syncBuf->nodeInfo.nodeMask      = upNodes;
#ifdef NAMESERVER_PROCESS
    syncBuf->nodeInfo.monConnCount  = MyNode->GetMonConnCount();
#else
    syncBuf->nodeInfo.monProcCount  = MyNode->GetNumProcs();
#endif

    for (int i = 0; i < GetPNodesCount(); i++)
    {
        if ( Node[indexToPnid_[i]] && Node[indexToPnid_[i]]->GetChangeState())
        {
            syncBuf->nodeInfo.change_nid = indexToPnid_[i];
            Node[indexToPnid_[i]]->SetChangeState( false );
            break;
        }
    }

    if (trace_settings & (TRACE_SYNC_DETAIL | TRACE_TMSYNC))
    {
#ifdef NAMESERVER_PROCESS
        trace_printf( "%s@%d - Node %s (pnid=%d) node_state=(%d)(%s), internalState=%d, change_nid=%d, seqNum_=%lld, monConnCount=%d\n"
                    , method_name, __LINE__
                    , MyNode->GetName()
                    , MyPNID
                    , syncBuf->nodeInfo.node_state
                    , StateString( MyNode->GetState() )
                    , syncBuf->nodeInfo.internalState
                    , syncBuf->nodeInfo.change_nid
                    , syncBuf->nodeInfo.seq_num
                    , syncBuf->nodeInfo.monConnCount);
#else
        trace_printf( "%s@%d - Node %s (pnid=%d) node_state=(%d)(%s), internalState=%d, TmSyncState=(%d)(%s), change_nid=%d, seqNum_=%lld, monProcCount=%d\n"
                    , method_name, __LINE__
                    , MyNode->GetName()
                    , MyPNID
                    , syncBuf->nodeInfo.node_state
                    , StateString( MyNode->GetState() )
                    , syncBuf->nodeInfo.internalState
                    , syncBuf->nodeInfo.tmSyncState
                    , SyncStateString( syncBuf->nodeInfo.tmSyncState )
                    , syncBuf->nodeInfo.change_nid
                    , syncBuf->nodeInfo.seq_num
                    , syncBuf->nodeInfo.monProcCount);
#endif
    }

    syncBuf->msgInfo.msg_count = 0;
    syncBuf->msgInfo.msg_offset = 0;

    msg = (struct internal_msg_def *) &syncBuf->msg[0];
    msg->type = InternalType_Null;

    syncBufferFreeSpace_ = ( MAX_SYNC_SIZE - 
                             (sizeof(cluster_state_def_t) + sizeof(msgInfo_t)));

    return msg;
}

bool CNodeContainer::IsShutdownActive (void)
{
    bool status = false;
    CNode *node = head_;

    const char method_name[] = "CNodeContainer::IsShutdownActive";
    TRACE_ENTRY;
    
    while (node)
    {
        if (( node->GetState() == State_Shutdown ) ||
            ( node->GetState() == State_Stopped  )   )
        {
            status = true;
        }
        node = node->GetNext ();
    }

    TRACE_EXIT;

    return status;
}


struct internal_msg_def *CNodeContainer::PopMsg( struct sync_buffer_def *recvBuf)
{
    struct internal_msg_def *msg;

    const char method_name[] = "CNodeContainer::PopMsg";
    TRACE_ENTRY;

    if ( recvBuf->msgInfo.msg_count > 0 )
    {
        msg = (struct internal_msg_def *)&recvBuf->msg[recvBuf->msgInfo.msg_offset];
        recvBuf->msgInfo.msg_count --;
        recvBuf->msgInfo.msg_offset += msg->replSize;
    }
    else
    {
        // SyncBuffer is empty --- just return null
        msg = (struct internal_msg_def *)&recvBuf->msg[0];
        msg->type = InternalType_Null;
    }

    TRACE_EXIT;

    return msg;
}

void CNodeContainer::SaveMyLastSyncBuffer( void )
{
    memcpy( (void*)lastSyncBuffer_, (void*)SyncBuffer, sizeof(sync_buffer_def) );
}

bool CNodeContainer::SpaceAvail ( int msgSize )
{
    // temp trace
    const char method_name[] = "CNodeContainer::SpaceAvail";

    // Determine if there is enough space in the sync buffer to 
    // hold a message of "msgSize" bytes plus the final end-of-buffer
    // indicator.
    if (syncBufferFreeSpace_ >= (msgSize + sizeof (InternalType)))
    {   // There is enough space
        if (trace_settings & TRACE_SYNC_DETAIL)
            trace_printf("%s@%d - Sync buffer has space to hold a %d byte message\n", method_name, __LINE__, msgSize);

        return true;
    }
    else
    {   // Not enough space left
        if (trace_settings & TRACE_SYNC)
            trace_printf("%s@%d - Sync buffer does not have enough space "
                         "to hold a %d byte message, sync buffer free=%d\n",
                         method_name, __LINE__, msgSize,
                         (int) syncBufferFreeSpace_);

        return false;
    }
}

void CNodeContainer::AddMsg (struct internal_msg_def *&msg,
                             int msgSize )
{

    // Insert the message size into the message header
    msg->replSize = msgSize;

    // Account for new message that has been inserted
    SyncBuffer->msgInfo.msg_count++;
    SyncBuffer->msgInfo.msg_offset += msgSize;
    syncBufferFreeSpace_ -= msgSize;

    // temp trace
    const char method_name[] = "CNodeContainer::AddMsg";
    if (trace_settings & TRACE_SYNC_DETAIL)
    {
        trace_printf("%s@%d - Added msg of size=%d, msg_count=%d, "
                     "msg_offset=%d, sync buffer free=%d\n", method_name,
                     __LINE__, msgSize, SyncBuffer->msgInfo.msg_count,
                     SyncBuffer->msgInfo.msg_offset, (int)syncBufferFreeSpace_);
    }

    // Set end-of-buffer marker
    msg = (struct internal_msg_def *)&SyncBuffer->msg[SyncBuffer->msgInfo.msg_offset];
    msg->type = InternalType_Null;

    return;
}

#ifndef NAMESERVER_PROCESS
void CNodeContainer::KillAll( CProcess *requester )
{
    CNode *node = head_;

    const char method_name[] = "CNodeContainer::KillAll";
    TRACE_ENTRY;

    while (node)
    {
        node->CProcessContainer::KillAll( node->GetState(), requester );
        node = node->GetNext ();
    }

    TRACE_EXIT;
}
#endif


int CNodeContainer::ProcessCount( void )
{
    int count = 0;
    CNode *node = head_;

    const char method_name[] = "CNodeContainer::ProcessCount";
    TRACE_ENTRY;

    while (node)
    {
#ifdef NAMESERVER_PROCESS // don't check state
        count += node->GetNumProcs();
#else
        if ( node->GetState() == State_Up || node->GetState() == State_Shutdown )
        {
            count += node->GetNumProcs();
        }
#endif
        node = node->GetNext ();
    }

    if (trace_settings & TRACE_ENTRY_EXIT)
       trace_printf("%s@%d" " - Count=" "%d" ", Exit" "\n", method_name, __LINE__, count);
    TRACE_EXIT;

    return (count<=0?0:count);
}

void CNodeContainer::RemoveFromSpareNodesList( CNode *node )
{
    const char method_name[] = "CNodeContainer::RemoveFromSpareNodesList";
    TRACE_ENTRY;

    if (trace_settings & TRACE_INIT)
        trace_printf("%s@%d - removing pnid=%d from spare node list\n", method_name, __LINE__, node->GetPNid());

    spareNodesList_.remove( node );

    if (trace_settings & TRACE_INIT)
    {
        CNode *spareNode;
        NodesList::iterator itSn;
        for ( itSn = spareNodesList_.begin(); itSn != spareNodesList_.end() ; itSn++ ) 
        {
            spareNode = *itSn;
            trace_printf("%s@%d - pnid=%d is in spare node list\n", method_name, __LINE__, spareNode->GetPNid());
        }
    }

    TRACE_EXIT;
}

void CNodeContainer::SetupCluster( CNode ***pnode_list, CLNode ***lnode_list, int **indexToPnid )
{
    const char method_name[] = "CNodeContainer::SetupCluster";
    TRACE_ENTRY;

    *pnode_list = Node;
    *lnode_list = LNode;
    *indexToPnid = indexToPnid_;

    // Build list of spare nodes
    CNode *node = GetFirstNode();
    for ( int i = 0; node && i < GetPNodesCount(); i++, node = node->GetNext() )
    {
        if (node)
        {
            if (trace_settings & TRACE_INIT)
                trace_printf( "%s@%d - Node %s (pnid=%d, zid=%d, state=%s) is Spare=%d\n"
                            , method_name, __LINE__
                            , node->GetName()
                            , node->GetPNid()
                            , node->GetZone()
                            , StateString(node->GetState())
                            , node->IsSpareNode());
            if ( node->GetState() == State_Up && node->IsSpareNode() )
            {
                spareNodesConfigList_.push_back( node );
                if (IAmIntegrating)
                {
                    // do nothing. spareNodesList will get populated in the join phase.
                }
                else
                {
                    spareNodesList_.push_back( node );
                }
            }
        }
    }
    if (trace_settings & TRACE_INIT)
    {
        CNode *spareNode;
        NodesList::iterator itSn;
        for ( itSn = spareNodesList_.begin(); itSn != spareNodesList_.end() ; itSn++ ) 
        {
            spareNode = *itSn;
            trace_printf("%s@%d - pnid=%d is in spare node list\n", method_name, __LINE__, spareNode->GetPNid());
        }
    }

    UpdateCluster();

    TRACE_EXIT;
}

void CNodeContainer::LoadConfig( void )
{
    const char method_name[] = "CNodeContainer::LoadConfig";
    TRACE_ENTRY;

    // The configuration is now global.  To minimize impact for the time being, just set the local
    // pointer to the global configuration
    if ( !clusterConfig_ )
    {
        clusterConfig_ = ClusterConfig;
    }

    if ( !nameServerConfig_ )
    {
        nameServerConfig_ = NameServerConfig;
    }
    if ( nameServerConfig_ )
    {
        if ( ! nameServerConfig_->LoadConfig() )
        {
            char la_buf[MON_STRING_BUF_SIZE];
            sprintf(la_buf, "[%s], Failed to load nameserver configuration.\n", method_name);
            mon_log_write(MON_NODECONT_LOAD_CONFIG_4, SQ_LOG_CRIT, la_buf);
            
            abort();
        }
    }

    TRACE_EXIT;
}

CLNode *CNodeContainer::SelectLNode( CProcess *requester, ZoneType type, int nid, int not_zone, bool considerLoad )
{
    CLNode *lnode = NULL;
    
    const char method_name[] = "CNodeContainer::SelectLNode";
    TRACE_ENTRY;

    if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS))
       trace_printf("%s@%d" " - nid=%d, not_zone=%d\n", method_name, __LINE__, nid, not_zone);

    lnode = (nid == -1) ? NextPossibleLNode(requester, type, nid, not_zone, considerLoad) : GetLNode(nid);

    if ( lnode )
    {
        if (trace_settings & (TRACE_REQUEST | TRACE_PROCESS))
           trace_printf("%s@%d" " - Selected Nid=" "%d" "\n", method_name, __LINE__, lnode->Nid);
        requester->SetLastNid ( lnode->Nid );
    }
    else
    {
        if (trace_settings & (TRACE_REQUEST | TRACE_PROCESS))
           trace_printf("%s@%d" " - No node selected " "\n", method_name, __LINE__);
    }

    TRACE_EXIT;

    return lnode;
}


CLNode *CNodeContainer::NextPossibleLNode( CProcess *requester, ZoneType type, int nid, int not_zone, bool considerLoad )
{
    int avg_pcount = 0;
    unsigned int avg_memory = 0;
#ifdef USE_MEMORY_LOADBALANCING
    int memory = -1;
#endif
    int cNid = nid;
    int selected_nid = -1;
    int firstViableNid = -1;
    CLNode *lnode = NULL;
    
    const char method_name[] = "CNodeContainer::NextPossibleLNode";
    TRACE_ENTRY;

    if ( considerLoad )
    {
        // get scheduling data
        AvgNodeData(type, &avg_pcount, &avg_memory);
    }

    // Initialize candidate nid (cNid) to the specified nid.  If none
    // specified use the nid assigned the previous time that "requester"
    // started a process.  [Note that the first candidate nid will be the 
    // first one after the initialized value.]
    cNid = (cNid == -1) ? requester->GetLastNid() : nid;

    // Beginning with the first nid after the initialized cNid, examine
    // nodes until find one that meets all necessary criteria:
    //   1) node must be up
    //   2) node must be of type specified ZoneType
    //   3) node must not be a "not_zone" node
    //   4) if load is considered, node must meet load criteria
    int nodesConsidered = 0;
    while ( selected_nid == -1  && ++nodesConsidered <= GetLNodesConfigMax() )
    {
        // Advance to next logical node number
        ++cNid;
        if ( cNid >= GetLNodesConfigMax() )
        {   // Wrap around to node 0
            cNid = 0;
        }

        lnode = GetLNode( cNid );
        if ( ! lnode )
        {
            continue;
        }

        if ( lnode->GetState() != State_Up ||
             lnode->IsKillingNode() )
        {
            continue;
        }

        if ( considerLoad )
        {
#ifdef USE_MEMORY_LOADBALANCING
            memory = (lnode->GetNode()->GetFreeCache() + lnode->GetNode()->GetFreeSwap())*0.95; 
            if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS_DETAIL))
                trace_printf("%s@%d - Nid=%d, Memory=%d, AvgMemory=%d\n", method_name, __LINE__, cNid, memory, avg_memory );
#endif
            if (trace_settings & (TRACE_REQUEST_DETAIL | TRACE_PROCESS_DETAIL))
                trace_printf("%s@%d - Nid=%d, NumProcs=%d, AvgProcs=%d\n", method_name, __LINE__, cNid, lnode->GetNumProcs(), avg_pcount );
        }

        // *** round robin based on requester's last used nid skipping overloaded nodes
        if (( lnode->GetZone() != not_zone  ) &&
            ( lnode->NodeZoneType & type    ) )
        {
            if ( !considerLoad
                 || (
#ifdef USE_MEMORY_LOADBALANCING
                  ( memory <= avg_memory             ) && 
#endif
                  ( lnode->GetNumProcs() <= avg_pcount ))   )
            {
                selected_nid = lnode->GetNid();
                if (trace_settings & (TRACE_REQUEST | TRACE_PROCESS))
                    trace_printf("%s@%d - Selecting Nid=%d, ZoneType=%d, "
                                 "requested: ZoneType=%d, not_zone=%d\n",
                                 method_name, __LINE__, selected_nid,
                                 lnode->NodeZoneType, type, not_zone);
            }
            else
            {   // Node has load greater than desired.
                // Keep track of the first node that is of the correct type
                // in case no nodes meet the required load criteria.
                firstViableNid =(firstViableNid == -1) ? cNid : firstViableNid;
            }
        }
    }

    if ( selected_nid == -1 )
    {
        if ( firstViableNid != -1 )
        {
            lnode = GetLNode( firstViableNid );
            assert( lnode );
            if (lnode && trace_settings & (TRACE_REQUEST | TRACE_PROCESS))
            {
                trace_printf("%s@%d - Selecting Nid=%d, ZoneType=%d, "
                             "requested: ZoneType=%d, not_zone=%d\n",
                             method_name, __LINE__, firstViableNid,
                             lnode->NodeZoneType,
                             type, not_zone);
            }
        }
        else if (trace_settings & (TRACE_REQUEST_DETAIL|TRACE_PROCESS_DETAIL))
        {
            trace_printf("%s@%d - no node of requested type is available. "
                         "requested: ZoneType=%d, not_zone=%d\n",
                         method_name, __LINE__, type, not_zone);
        }
    }


    TRACE_EXIT;
    return lnode;
}

void CNodeContainer::UpdateCluster( void )
{
    const char method_name[] = "CNodeContainer::UpdateCluster";
    TRACE_ENTRY;

    CLNode *lnode;
    CNode  *node;

    for (int i = 0; i < clusterConfig_->GetPNodesConfigMax(); i++ )
    {
        indexToPnid_[i] = -1;
    }

    node = GetFirstNode();
    // Refresh the index to pnid map
    for ( int i = 0; node && i < GetPNodesCount(); i++, node = node->GetNext() )
    {
        indexToPnid_[i] = node->GetPNid();
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST))
            trace_printf( "%s@%d - indexToPnid_[%d]=%d\n"
                        , method_name, __LINE__
                        , i
                        , indexToPnid_[i]);
    }

    // Refresh the index to nid map
    lnode = GetFirstLNode();
    for ( int i = 0; lnode && i < GetLNodesCount(); i++, lnode = lnode->GetNext() )
    {
        indexToNid_[i] = lnode->GetNid();
        if (trace_settings & (TRACE_INIT | TRACE_RECOVERY | TRACE_REQUEST))
        {
            trace_printf( "%s@%d - indexToNid_[%d]=%d\n"
                        , method_name, __LINE__
                        , i
                        , indexToNid_[i]);
        }
    }

    TRACE_EXIT;
}