#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
# 
#   http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

"""
hawq checkperf -- Check the hardware for Greenplum Database

Usage: hawq checkperf <options>

    --version  : print version information
    -d segdir  : where segdir is a directory on which to test I/O performance. 
                 Multiple -d flags may be specified for multiple directories
                 on each host. These will typically be your Greenplum primary
                 (and/or mirror) data directories. 
                 e.g. -d /dbfast1 -d /dbfast2
    -?         : print this help text
    -v         : enable verbose
    -V         : enable very verbose
    -D         : print per host disk stats on disk read/write tests
    -r flags   : where flags specifies the tests to run [default=dsn]. 
                 Supported test flags are:
                       d: runs only disk tests
                       s: runs only stream benchmark
                       n: runs only netperf tests (serial)
                       N: runs only netperf tests (parallel)
                       M: runs only netperf tests (full matrix)
    -B size    : where size specifies the block size for disk performance tests
                 [default=32KB] e.g. 1KB, 4MB
    -S size    : where size specifies the file size for disk performance tests
                 [default=2X Memory] e.g. 500MB, 16GB
    -h host    : the host to connect to (multiple -h is okay)
    -f file    : a file listing all hosts to connect to
    --duration : how long to run network test (default 5 seconds)
    --netperf  : use netperf instead of gpnetbenchServer/gpnetbenchClient
"""

import os, sys

sys.path.append(sys.path[0] + '/lib')
try:
    import getopt, math, StringIO, stat, subprocess, signal
    from gppylib.gpparseopts import OptParser
    from gppylib.util import ssh_utils
    from gppylib.gpcoverage import GpCoverage
except ImportError, e:
    sys.exit('Error: unable to import module: ' + str(e))


GPHOME=os.getenv('GPHOME')
if GPHOME is None:
    sys.exit('Please set GPHOME environment variable')

USER=os.getenv('USER')
if USER is None or USER is ' ':
    sys.exit('Please set USER environment variable')

################
def usage(exitarg):
    parser = OptParser()
    try:
        parser.print_help()
    except:
        print __doc__
    sys.exit(exitarg)


class Global():
    script_name = os.path.split(__file__)[-1]
    opt = {}
    opt['-d'] = []
    opt['-D'] = False
    opt['-v'] = False
    opt['-V'] = False
    opt['-r'] = ''
    opt['-B'] = 1024 * 32
    opt['-S'] = 0
    opt['-h'] = []
    opt['-f'] = None
    opt['--duration'] = 15  # default to 15s
    opt['--net'] = None
    opt['--netserver'] = 'gpnetbenchServer'
    opt['--netclient'] = 'gpnetbenchClient'

GV = Global()

#############
def killall(procname):
    return 'F=%s && (pkill $F || pkill -f $F || killall -9 $F) > /dev/null 2>&1 || true' % procname

###############################
def strcmd(cmd):
    return reduce(lambda x, y: x + ' ' + y, map(lambda x: x.find(' ') > 0 and "'" + x + "'" or x, cmd))

#############
def gpssh(cmd):
    c = ['%s/bin/gpssh' % GPHOME]
    if (GV.opt['-V']): c.append('-v')
    if (GV.opt['-f']):
        c.append('-f')
        c.append(GV.opt['-f'])
    else:
        for h in GV.opt['-h']: 
            c.append('-h')
            c.append(h)
    c.append(cmd)

    if GV.opt['-v']: print '[Info]', strcmd(c)
    p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    out = p.stdout.read(-1)
    rc = p.wait()
    return (not rc, out)

#############
def gpscp(src, dst):
    c = ['%s/bin/gpscp' % GPHOME]
    if (GV.opt['-V']): c.append('-v')
    if (GV.opt['-f']):
        c.append('-f')
        c.append(GV.opt['-f'])
    else:
	for h in GV.opt['-h']: 
            c.append('-h')
            c.append(h)
    c.append(src)
    c.append(dst)
    if GV.opt['-v']: print '[Info]', strcmd(c)
    p = subprocess.Popen(c, stdout=subprocess.PIPE)
    out = p.stdout.read(-1)
    rc = p.wait()
    return (not rc, out)

################
def getPlatform():
    if sys.platform.find('linux') >= 0: return 'linux'
    if sys.platform.find('darwin') >= 0: return 'darwin'
    if sys.platform.find('sunos5') >= 0: return 'sunos5'
    return '?'

################
def getMemory():
    if getPlatform() == 'linux':
        ok, out = run("sh -c 'cat /proc/meminfo | grep MemTotal'")
        if not ok: return 0
        list = out.strip().split(' ')
        val = int(list[len(list) - 2])
        factor = list[len(list) - 1]
        if factor == 'kB': 
            return val * 1024
        return 0

    if getPlatform() == 'darwin':
        ok, out = run("/usr/sbin/sysctl hw.physmem")
        if not ok: return 0
        list = out.strip().split(' ')
        val = int(list[1])
        return val

    if getPlatform() == 'sunos5':
        ok, out = run("sh -c \"/usr/sbin/prtconf | awk '/^Memory/{print}'\"")
        if not ok: return 0
        list = out.strip().split(' ')
        val = int(list[2])
        factor = list[3]
        if factor == 'Megabytes': 
            return val * 1024 * 1024
        return 0

    return 0

################
def parseMemorySize(line):
    factor = 1
    try:
	line = line.strip().upper()
	if line.endswith('B'): line = line[:-1]
	if line.endswith('G'): factor = 1024 * 1024 * 1024
	elif line.endswith('M'): factor = 1024 * 1024
	elif line.endswith('K'): factor = 1024
	if factor > 1: line = line[:-1]
	return int(line) * factor
    except Exception, err: 
        print 'Error: ' + err
        sys.exit(-1)
        return 0

#############
def parseDuration(val):
    factor = 1
    try:
        val = val.strip().upper()
        if val.endswith('S'): val = val[:-1]
        if val.endswith('M'): factor = 60
        if val.endswith('H'): factor = 3600
        if val.endswith('D'): factor = 86400
        if factor > 1: val = val[:-1]
        return int(val) * factor
    except Exception, err:
        print "exception: %s" % err
        sys.exit(-1)


#############
def print_version():
    print '%s version $Revision$' % GV.script_name
    sys.exit(0)

#############
def parseCommandLine():
    global opt
    try:
	(options, args) = getopt.getopt(sys.argv[1:], '?vVDd:r:B:S:p:h:f:', ['duration=', 'version', 'netperf'])
    except Exception, e:
	usage('Error: ' + str(e))

    for (switch, val) in options:
        if (switch == '-?'): usage(0)
        elif (switch[1] in 'vVD'): GV.opt[switch] = True
        elif (switch[1] in 'dh'): GV.opt[switch].append(val)
        elif (switch[1] in 'rf'): GV.opt[switch] = val
        elif (switch[1] in 'BS'): GV.opt[switch] = parseMemorySize(val)
        elif (switch == '--duration'): GV.opt[switch] = parseDuration(val)
        elif (switch == '--version'): print_version()
        elif (switch == '--netperf'): 
            GV.opt['--netserver'] = 'netserver'
            GV.opt['--netclient'] = 'netperf'

    # run default tests (if not specified)
    if GV.opt['-r'] == '':
        GV.opt['-r'] = 'dsN'

    # parse netperf test type, error if more than one specified
    netcount = 0
    if GV.opt['-r'].find('n') >= 0: 
        GV.opt['--net'] = 'netperf'
        netcount += 1
    if GV.opt['-r'].find('N') >= 0:
        GV.opt['--net'] = 'parallel'
        netcount += 1
    if GV.opt['-r'].find('M') >= 0:
        GV.opt['--net'] = 'matrix'
        netcount += 1
    if netcount > 1:
        usage('Error: please only specify one network test at a time (-r [n|M|N])')
        
    if GV.opt['-V']: GV.opt['-v'] = True

    if (len(GV.opt['-d']) == 0): 
	usage('Error: please specify at least one segdir (-d switch)')

    if not (0 <= GV.opt['-B'] and 0 <= GV.opt['-S']):
	usage('Error: please enter valid memory sizes for -B or -S switches')

    hf = (len(GV.opt['-h']) and 1 or 0) + (GV.opt['-f'] and 1 or 0)
    if hf != 1:
	usage('Error: please specify at least one of -h or -f args, but not both')

    if (GV.opt['-B']) > 1024*1024:
	usage('Error: maximum size for -B parameter is 1MB')

    if GV.opt['-S'] == 0:
        GV.opt['-S'] = 2 * getMemory() / len(GV.opt['-d'])
    else:
        GV.opt['-S'] /= len(GV.opt['-d'])

    if GV.opt['--duration'] <= 0:
        GV.opt['--duration'] = 15
        print '[INFO] Invalid network duration specified.  Using default (15 seconds)'

    # strip the last '/' from the dir
    dd = []
    for d in GV.opt['-d']:
	if d and d[-1] == '/': d = d[:-1]
	dd.append(d)
    GV.opt['-d'] = dd

    def strcmd(cmd):
        return reduce(lambda x, y: x + ' ' + y, map(lambda x: x.find(' ') > 0 and "'" + x + "'" or x, cmd))

    print strcmd(sys.argv)


#############
def run(cmd):
    f = None
    ok = False
    out = ''
    try:
	try:
	    if GV.opt['-v']: print '[Info]', cmd
	    f = os.popen(cmd)
	    out = f.read()
	    ok = not f.close()
	    f = None
	except KeyboardInterrupt:
	    raise
    finally:
	if f: f.close()

    if GV.opt['-v'] and out and out != '': print out

    if out and out.find('Permission denied') >= 0 and out.find('publickey') >= 0 : 
	if not GV.opt['-v']: print out
	print 'Please use gpssh-exkeys to exchange keys ...'
	sys.exit(1)

    return (ok, out)

#############
runTeardown_torndown = False
def runTeardown():
    global runTeardown_torndown
    if runTeardown_torndown: return
    if GV.opt['-v']:
	print '--------------------'
	print '  TEARDOWN'
	print '--------------------'
    dirs = ''
    for d in GV.opt['-d']: dirs = '%s %s/gpcheckperf_$USER' % (dirs, d)
    try:
        (ok, out) = gpssh('rm -rf ' + dirs)
    except:
	pass

    try:
        if GV.opt['--net']:
            (ok, out) = gpssh(killall(GV.opt['--netserver']))
    except:
        pass

    runTeardown_torndown = True

#############
def runSetup():
    if GV.opt['-v']:
	print '--------------------'
	print '  SETUP'
	print '--------------------'
    okCount = 0
    try:
        #check hostfile if have -f opt
        if GV.opt['-f']:
            if not os.path.exists(GV.opt['-f']):
                sys.exit("[Error] unable to find hostfile: %s" % GV.opt['-f'])
	#check python reachable
	if GV.opt['-v']: print '[Info] verify python interpreter exists'
	(ok, out) = gpssh('python -c print')
	if not ok:
	    if not GV.opt['-v']: print out
	    sys.exit("[Error] unable to find python interpreter on some hosts\n"
		     +"        verify PATH variables on the hosts")

	#mkdir cperf
	if GV.opt['-v']: print '[Info] making hawq checkperf directory on all hosts ... '
	dirs = ''
	for d in GV.opt['-d']: dirs = '%s %s/gpcheckperf_$USER' % (dirs, d)

        cmd = 'rm -rf %s ; mkdir -p %s' % (dirs, dirs)
	(ok, out) = gpssh(cmd)
	if not ok: 
	    sys.exit("[Error] unable to make hawq checkperf directory. \n"
		   + "        command failed: " + cmd)

    except:
	runTeardown()
	raise

################
def copyExecOver(fname):

    #copy to this dir
    d = GV.opt['-d'][0]
    target = '%s/gpcheckperf_$USER/%s' % (d, fname)
    path = '%s/lib/%s' % (sys.path[0], fname)

    if not os.path.isfile(path):
        print "binary file not found: %s" % path
        sys.exit(1)

    if GV.opt['-v']: print '[Info] copy local %s to remote %s' % (path, target)

    try: pathStat = os.stat(path)
    except: sys.exit('[Error] cannot stat file ' + path)
    if not stat.S_ISREG(pathStat.st_mode): sys.exit('[Error] invalid file ' + path)
    if not os.access(path, os.X_OK): sys.exit('[Exit] file not executable: ' + path)

    (ok, out) = gpscp(path, '=:%s' % target)
    if not ok: sys.exit('[Error] command failed: gpscp %s =:%s' % (path, target))
    
    #chmod +x file
    (ok, out) = gpssh('chmod a+rx %s' % target)
    if not ok: sys.exit('[Error] command failed: chmod a+rx %s' % target)
    
    return target


#############
def parseMultiDDResult(out):
    # parse output of "time -p"
    out = StringIO.StringIO(out)
    result = {}
    bytes = 0
    for line in out:
	i = line.find(']')
	if i == -1: continue
	o = line[i+2:]

	if o.startswith('multidd total bytes '):
	    h = line[1:i]
	    o = o.split()
	    bytes = int(o[-1])
	    continue
	    
	if o.startswith('real'):
	    h = line[1:i]
	    o = o.split()
	    time = float(o[1])
	    if time < 0.1: time = 0.1 		# avoid division by zero
	    mbps = bytes / time / 1024 / 1024
	    result[h] = (mbps, time, bytes)
	    continue

    return result


#############
def runDiskWriteTest(multidd):
    print
    print '--------------------'
    print '--  DISK WRITE TEST'
    print '--------------------'
    
    cmd = 'time -p ' + multidd
    for d in GV.opt['-d']:
	cmd = cmd + (' -i /dev/zero -o %s/gpcheckperf_$USER/ddfile' % d)
    if GV.opt['-B']: cmd = cmd + (' -B %d' % GV.opt['-B'])
    if GV.opt['-S']: cmd = cmd + (' -S %d' % GV.opt['-S'])
    (ok, out) = gpssh(cmd)
    if not ok:
	sys.exit('[Error] command failed: ' + cmd)
    return parseMultiDDResult(out)


#############
def runDiskReadTest(multidd):
    print
    print '--------------------'
    print '--  DISK READ TEST'
    print '--------------------'

    cmd = 'time -p ' + multidd
    for d in GV.opt['-d']:
	cmd = cmd + (' -o /dev/null -i %s/gpcheckperf_$USER/ddfile' % d)
    if GV.opt['-B']: cmd = cmd + (' -B %d' % GV.opt['-B'])
    if GV.opt['-S']: cmd = cmd + (' -S %d' % GV.opt['-S'])
    (ok, out) = gpssh(cmd)
    if not ok:
	sys.exit('[Error] command failed: ' + cmd)
    return parseMultiDDResult(out)


#############
def runStreamTest():
    print
    print '--------------------'
    print '--  STREAM TEST'
    print '--------------------'

    rmtPath = copyExecOver('stream')
    (ok, out) = gpssh(rmtPath)
    if not ok: sys.exit('[Error] command failed: ' + rmtPath)
    out = StringIO.StringIO(out)
    result = {}
    for line in out:
	i = line.find(']')
	o = line[i+2:]
	if o.startswith('Copy:'): 
	    h = line[1:i]
	    o = o.split()
	    result[h] = (float(o[1]), 0, 0)
    return result


#############
def startNetServer():
    port = 23000
    rmtPath = copyExecOver(GV.opt['--netserver'])

    for i in xrange(5):
        if i > 0: print '[Warning] retrying with port %d' % port
        (ok, out) = gpssh(killall(GV.opt['--netserver']))

        (ok, out) = gpssh('%s -p %d > /dev/null 2>&1' % (rmtPath, port))
        if ok: return port

        if GV.opt['-v']: print '[Warning] start netserver with port %d failed' % port
        port += 12

    return 0

#############
def killProc(proc):
    if proc: 
        try:
            os.kill(proc.pid, signal.SIGKILL)
        except OSError:
            pass
        return proc.wait()
    return 0


#############
def spawnNetperfTestBetween(x, y, netperf_path, netserver_port, sec=5):
    cmd = ('%s -H %s -p %d -t TCP_STREAM -l %s -f M -P 0 '
           % (netperf_path, y, netserver_port, sec))
    c = ['ssh', '-o', 'BatchMode yes', 
         '-o', 'StrictHostKeyChecking no',
         x, cmd]
    proc = None
    try: 
        if GV.opt['-v']: print '[Info]', strcmd(c)
        proc = subprocess.Popen(c, stdout=subprocess.PIPE)
    except KeyboardInterrupt:
        killProc(proc)
        raise
    if not proc:
        print '[Warning] netperf failed on %s -> %s' % (x, y)
        return (None, None, None)

    return (proc, x, y)


#############
def reapNetperfTest(proc, x, y):
    ok = True
    out = None
    if proc:
        try:
            rc = proc.wait()
            out = proc.stdout.read(-1)
            ok = not killProc(proc)
            proc = None
        except KeyboardInterrupt, ki:
            raise
        except:
            pass
        finally:
            proc = None
    
    if GV.opt['-v'] and out and out != '': 
        print '[Info]',out
    if out and out.find('Permission denied') >= 0 and out.find('publickey') >= 0 : 
        if not GV.opt['-v']: 
            print out
        print '[Error] Please use gpssh-exkeys to exchange keys ...'
        sys.exit(1)

    if not ok: 
        print '[Warning] netperf failed on %s -> %s' % (x, y)
        return []

    for line in StringIO.StringIO(out):
        line = line.split()
        if len(line) != 5: continue
	l = [x, y]
	l.extend(map(lambda x: float(x), line))
        if GV.opt['-v']: 
            print '[Info] %s -> %s : %s' % (x, y, str(line))
	return l

    print '[Warning] netperf failed on %s -> %s (invalid result)' % (x, y)
    print '[Info]', out
    return []


#############
def runNetperfTestBetween(x, y, netperf_path, netserver_port):
    (proc, x, y) = spawnNetperfTestBetween(x, y, netperf_path, netserver_port, GV.opt['--duration'])
    if not proc: 
        return []

    return reapNetperfTest(proc, x, y)


#############
def setupNetPerfTest():
    print
    print '-------------------'
    print '--  NETPERF TEST'
    print '-------------------'
    
    hostlist = ssh_utils.HostList()
    for h in GV.opt['-h']: hostlist.add(h)
    if GV.opt['-f']: hostlist.parseFile(GV.opt['-f'])

    h = hostlist.get()
    if len(h) == 0:
        usage('Error: missing hosts in -h and/or -f arguments')
    
    if len(h) == 1:
        print '[Warning] single host only - abandon netperf test'
        return (None, None, None)

    # start netserver
    netserver_port = startNetServer()
    if netserver_port == 0:
        print '[Error] unable to start netserver ... abort netperf test'
        return (None, None, None)
    
    netperf = copyExecOver(GV.opt['--netclient'])
    return (netperf, h, netserver_port)
    

#############
def runNetPerfTest():
    (netperf, hostlist, netserver_port) = setupNetPerfTest()
    if not netperf:
        return None

    # make sure we have even number of hosts
    odd = None
    h = hostlist[:]
    if len(h) & 1 == 1:
        odd = h.pop()           #  take one out

    # run netperf test between host[0] & host[1], host[2] & host[3], ...
    result = []
    for i in xrange(len(h)):
        if i & 1 == 0: continue
        x = h[i-1]
        y = h[i]
        res = runNetperfTestBetween(x, y, netperf, netserver_port)
	if res and len(res) >= 6: result.append(res)
        res = runNetperfTestBetween(y, x, netperf, netserver_port)
	if res and len(res) >= 6: result.append(res)

    # run netperf test between the odd host and host 0
    if odd:
        x = odd 
        y = h[0]
        res = runNetperfTestBetween(x, y, netperf, netserver_port)
        if res and len(res) >= 6: result.append(res)
        res = runNetperfTestBetween(y, x, netperf, netserver_port)
        if res and len(res) >= 6: result.append(res)

    return result


#############
def runNetPerfTestParallel():
    (netperf, hostlist, netserver_port) = setupNetPerfTest()
    if not netperf:
        return None

    # make sure we have even number of hosts
    h = hostlist[:]
    if len(h) & 1 == 1:
        h.append(h[0])

    # run netperf test between host[0] & host[1], host[2] & host[3], ...
    result = []
    procList = []
    for i in xrange(len(h)):
        if i & 1 == 0: continue
        x = h[i-1]
        y = h[i]
        p = spawnNetperfTestBetween(x, y, netperf, netserver_port, GV.opt['--duration'])
        if p[0]: 
            procList.append(p)
    
    for (proc, x, y) in procList:
        res = reapNetperfTest(proc, x, y)
	if res and len(res) >= 6: result.append(res)

    # second round: test between host[1] & host[0], host[3] & host[2], ...
    procList = []
    for i in xrange(len(h)):
        if i & 1 == 0: continue
        x = h[i-1]
        y = h[i]
        p = spawnNetperfTestBetween(y, x, netperf, netserver_port, GV.opt['--duration'])
        if p[0]: 
            procList.append(p)
    
    for (proc, x, y) in procList:
        res = reapNetperfTest(proc, x, y)
	if res and len(res) >= 6: result.append(res)


    return result

#############
def get_host_map(hostlist):
    '''
        Returns a seglist dictionary that maps segment name to hostname and
        a uniqhosts dictionary that maps each hostname to one segment name.

        For example:
        seglist = 
            seglist['sdw1-1'] = apollo1
            seglist['sdw1-2'] = apollo1
            seglist['sdw2-1'] = apollo2
            seglist['sdw2-2'] = apollo2
        uniqhosts =
            uniqhosts['apollo1'] = 'sdw1-1'
            uniqhosts['apollo2'] = 'sdw2-1'
            
    '''
    seglist   = dict()   # segment list
    uniqhosts = dict()   # unique host list

    # get list of hostnames
    rc, out = gpssh('hostname')
    if not rc:
        raise Exception('Encountered error running hostname')
        sys.exit(-1)

    # get unique hostname list
    for line in out.splitlines():
        seg, host = line.translate(None,'[]').split()
        uniqhosts[host] = seg

    # get list of segments associated with each host (can't use gpssh since it de-dupes hosts)
    for host in hostlist:
        cmd = ['ssh', '-o', 'BatchMode yes', '-o', 'StrictHostKeyChecking no', host, 'hostname' ]

        try:
            if GV.opt['-v']: print '[Info]', strcmd(cmd)
            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
            rc = proc.wait()
            if rc:
                raise Exception('ssh error with the following command:\n%s' % cmd)
            out = proc.stdout.read(-1)
            seglist[host] = out.strip()
        except KeyboardInterrupt:
            killProc(proc)
            raise
        except Exception, err:
            print err
            sys.exit(-1)

    return seglist, uniqhosts

#############
def runNetPerfTestMatrix():
    '''
        Runs full matrix network test where each host connnects with
            every link on every other host.  
        Returns performance results and a dictionary mapping segment_name
            to hostname
    '''
    (netperf, hostlist, netserver_port) = setupNetPerfTest()
    if not netperf:
        return None

    # dict() of seglist[segname] = hostname, uniqhosts[hostname] = 1 segment name
    seglist, uniqhosts = get_host_map(hostlist)

    # spawn netperf between all hosts 
    result = []
    procList = []
    for host in uniqhosts:
        for segment in seglist:
            if seglist[segment] == host:
                pass
            else:
                p = spawnNetperfTestBetween( host, segment, netperf, netserver_port, GV.opt['--duration'])
                if p[0]:
                    procList.append(p)

    for (proc, srcHost, dstHost) in procList:
        res = reapNetperfTest(proc, srcHost, dstHost)
	if res and len(res) >= 6: result.append(res)

    return result, seglist


#############
def printMatrixResult(result, seglist):

    print 'Full matrix netperf bandwidth test'

    # sum up Rx/Tx rate for each host
    netTx = dict()
    netRx = dict()
    for h in result:
        if netTx.has_key( h[0] ):
            netTx[ h[0] ] += float( h[6] )
        else:
            netTx[ h[0] ] = float( h[6] )

        # netRx requires that we lookup the hostname for a given segment name
        if netRx.has_key( seglist[ h[1] ] ):
            netRx[ seglist[ h[1] ] ] += float( h[6] )
        else:
            netRx[ seglist[ h[1] ] ] = float( h[6] )

    # print Tx rates 
    print "\nPer host transfer rates"
    for host in netTx:
        print '%s Tx rate: %.2f' % (host, netTx[host])
    print

    # print Rx rates
    print "Per host receive rates"
    for host in netRx:
        print '%s Rx rate: %.2f' % (host, netRx[host])
    print

    # print per link results in verbose mode
    if GV.opt['-v']: 
        for h in result:
            print '%s -> %s = %f' % (h[0], h[1], h[6])

    sum = min = max = avg = 0
    n = map(lambda x: float(x[6]), result)
    sum = reduce(lambda x, y: x + y, n)
    min = reduce(lambda x, y: x > y and y or x, n, 9999999999)
    max = reduce(lambda x, y: x > y and x or y, n)
    avg = float(sum) / len(result)
    
    copy = n[:]
    copy.sort()
    median = copy[len(copy) / 2]

    print ''
    print 'Summary:'
    print 'sum = %.2f MB/sec' % sum
    print 'min = %.2f MB/sec' % min
    print 'max = %.2f MB/sec' % max
    print 'avg = %.2f MB/sec' % avg
    print 'median = %.2f MB/sec' % median
    print ''


#############
def printNetResult(result):

    print 'Netperf bisection bandwidth test'
    for h in result:
        print '%s -> %s = %f' % (h[0], h[1], h[6])

    sum = min = max = avg = 0
    n = map(lambda x: float(x[6]), result)
    sum = reduce(lambda x, y: x + y, n)
    min = reduce(lambda x, y: x > y and y or x, n, 9999999999)
    max = reduce(lambda x, y: x > y and x or y, n)
    avg = float(sum) / len(result)
    
    copy = n[:]
    copy.sort()
    median = copy[len(copy) / 2]

    print ''
    print 'Summary:'
    print 'sum = %.2f MB/sec' % sum
    print 'min = %.2f MB/sec' % min
    print 'max = %.2f MB/sec' % max
    print 'avg = %.2f MB/sec' % avg
    print 'median = %.2f MB/sec' % median
    print ''

    limit = max * 0.9
    for r in result:
        if r[6] < limit:
            print '[Warning] connection between %s and %s is no good' % (r[0], r[1])


################
def printResult(title, result):
    totTime = 0
    totBytes = 0
    totMBPS = 0
    min = 99999999
    max = 0
    minHost = maxHost = None
    for h in result:
	(mbps, time, bytes) = result[h]
	totTime += time
	totBytes += bytes
	totMBPS += mbps
	if min > mbps: 
	    min = mbps
	    minHost = h
	if max < mbps: 
	    max = mbps
	    maxHost = h

    avgTime = totTime / len(result)

    print
    if (totTime > 0): print ' %s avg time (sec): %3.2f' % (title, avgTime)
    if (totBytes > 0): print ' %s tot bytes: %d' % (title, totBytes)
    print ' %s tot bandwidth (MB/s): %3.2f' % (title, totMBPS)
    print ' %s min bandwidth (MB/s): %3.2f [%s]' % (title, min, minHost)
    print ' %s max bandwidth (MB/s): %3.2f [%s]' % (title, max, maxHost)
    if GV.opt['-D']:
	print ' -- per host bandwidth --'
	for h in result:
	    (mbps, time, bytes) = result[h]
	    print '    %s bandwidth (MB/s): %3.2f [%s]' % (title, mbps, h)
    print


################

coverage = GpCoverage()
coverage.start()

try:
    parseCommandLine()
    runSetup()
    diskWriteResult = diskReadResult = streamResult = netResult = None
    tornDown = False
    try:
        if GV.opt['-r'].find('d') >= 0:
            multidd = copyExecOver('multidd')
            diskWriteResult = runDiskWriteTest(multidd)
            diskReadResult = runDiskReadTest(multidd)

        if GV.opt['-r'].find('s') >= 0:
            streamResult = runStreamTest()

        if GV.opt['--net'] == 'netperf':
            netResult = runNetPerfTest()
        elif GV.opt['--net'] == 'parallel':
            netResult = runNetPerfTestParallel()
        elif GV.opt['--net'] == 'matrix':
            netResult, seglist = runNetPerfTestMatrix()

        runTeardown()

    finally:
        print
        print '===================='
        print '==  RESULT'
        print '===================='

        if diskWriteResult:
            printResult('disk write', diskWriteResult)

        if diskReadResult:
            printResult('disk read', diskReadResult)

        if streamResult:
            printResult('stream', streamResult)

        if netResult and GV.opt['--net'] == 'matrix':
            printMatrixResult(netResult, seglist)
        elif netResult and GV.opt['--net']:
            printNetResult(netResult)

        runTeardown()

except KeyboardInterrupt:
    print '[Abort] Keyboard Interrupt ...'
finally:
    coverage.stop()
    coverage.generate_report()