blob: 1913a9829b94f81f6797825666ee34fad21a4d86 [file] [log] [blame]
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Text.RegularExpressions;
using System.Threading;
namespace Apache.Geode.Client.FwkLib
{
using Apache.Geode.DUnitFramework;
using Apache.Geode.Client.Tests;
using Apache.Geode.Client;
using NEWAPI = Apache.Geode.Client.Tests;
[Serializable]
struct HostInfo
{
public bool Started;
public string HostType;
public string HostName;
public string ExtraServerArgs;
public HostInfo(bool started, string hostType, string hostName,
string extraServerArgs)
{
Started = started;
HostType = hostType;
HostName = hostName;
ExtraServerArgs = extraServerArgs;
}
}
public class Utils<TKey, TVal> : FwkTest<TKey, TVal>
{
private const char PathSep = '/';
private const int MaxWaitMillis = 1800000;
private const string SetupJSName = "setupJavaServers";
private const string StartJSName = "startJavaServers";
private const string StopJSName = "stopJavaServers";
private const string KillJSName = "killJavaServers";
private const string SleepTime = "sleepTime";
private const string MinServers = "minServers";
private const string JavaServerCountKey = "ServerCount";
private const string JavaServerMapKey = "ServerMap";
private const string JavaServerName = "cacheserver.bat";
private const string JavaServerOtherArgs =
" statistic-sampling-enabled=true" +
" statistic-archive-file=statArchive.gfs mcast-port=0";
private const string JavaServerJavaArgs =
" -J-Xmx1280m -J-Xms512m -J-DCacheClientProxy.MESSAGE_QUEUE_SIZE=100000";
private const string JavaServerJavaArgsUnix =
" -J-Xmx2048m -J-Xms1024m -J-XX:+HeapDumpOnOutOfMemoryError " +
"-J-DCacheClientProxy.MESSAGE_QUEUE_SIZE=100000";
private static string TempDir = Util.GetEnvironmentVariable("TMP");
// Constants for status
private const int GFNoError = 0;
private const int GFError = 1;
private const int GFTimeout = 2;
private static Dictionary<string, HostInfo> m_javaServerHostMap =
new Dictionary<string, HostInfo>();
private static int m_numServers = 1;
private static int locCount = 0;
private static volatile bool m_exiting = false;
private static bool m_firstRun = true;
private static string m_locatorHost = null;
private static string m_locatorType = null;
#region Private methods
private string[] ParseJavaServerArgs(string argStr, ref int numServers,
out int argIndx, out string extraServerArgs)
{
argIndx = 0;
extraServerArgs = string.Empty;
if (argStr != null && argStr.Length > 0)
{
string[] args = argStr.Split(' ');
while (args.Length > argIndx && args[argIndx][0] == '-')
{
FwkAssert(args.Length > (argIndx + 1),
"JavaServer() value not provided after option '{0}'.",
args[argIndx]);
string argName = args[argIndx];
string argValue = args[argIndx + 1];
switch (argName)
{
case "-t":
// Ignore the tagname; we now use the hostGroup name as the tag
break;
case "-N":
break;
case "-M":
break;
case "-X":
break;
case "-c":
numServers = int.Parse(argValue);
break;
case "-e":
// @TODO: this is C++ specific environment variables -- ignored for now
break;
case "-p":
extraServerArgs += (" " + argValue.Replace("\\", "").
Replace("\"", "").Replace("'", ""));
break;
default:
FwkException("JavaServer() Unknown option '{0}'", argName);
break;
}
argIndx += 2;
}
extraServerArgs += " ";
return args;
}
return new string[0];
}
private string GetHostGroup()
{
string hostGroup;
try
{
hostGroup = Util.BBGet(Util.ClientId,
FwkReadData.HostGroupKey) as string;
}
catch
{
hostGroup = null;
}
return (hostGroup == null ? string.Empty : hostGroup);
}
private static HostInfo GetHostInfo(string serverId)
{
HostInfo hostInfo = new HostInfo();
lock (((ICollection)m_javaServerHostMap).SyncRoot)
{
if (m_javaServerHostMap.Count == 0)
{
try
{
m_javaServerHostMap = Util.BBGet(JavaServerBB, JavaServerMapKey)
as Dictionary<string, HostInfo>;
}
catch
{
}
}
if (m_javaServerHostMap.ContainsKey(serverId))
{
hostInfo = m_javaServerHostMap[serverId];
}
}
return hostInfo;
}
private static string GetJavaStartDir(string serverId, string hostType)
{
return Util.GetLogBaseDir(hostType) + PathSep + "GFECS_" + serverId;
}
private static string GetLocatorStartDir(string locatorId, string hostType)
{
return Util.GetLogBaseDir(hostType) + PathSep + "GFELOC_" + locatorId;
}
private string GetJavaLocator()
{
try
{
return (string)Util.BBGet(string.Empty, "LOCATOR_ADDRESS");
}
catch
{
return null;
}
}
private string GetJavaLocatorForPoolAttri()
{
try
{
return (string)Util.BBGet(string.Empty, "LOCATOR_ADDRESS_POOL");
}
catch
{
return null;
}
}
private bool GetSetJavaLocator(string locatorHost,
out int locatorPort)
{
locatorPort = 0;
string locatorAddress = GetJavaLocator();
string locatorAddressForPool = GetJavaLocatorForPoolAttri();
if (locatorAddress == null || locatorAddress.Length == 0)
{
locatorPort = Util.Rand(31124, 54343);
locatorAddress = locatorHost + '[' + locatorPort + ']';
locatorAddressForPool = locatorHost + ':' + locatorPort;
Util.BBSet(string.Empty, "LOCATOR_ADDRESS", locatorAddress);
Util.BBSet(string.Empty, "LOCATOR_ADDRESS_POOL", locatorAddressForPool);
locCount++;
return true;
}
if (locatorAddress != null && locatorAddress.Length > 0 && locCount > 0)
{
locatorPort = Util.Rand(31124, 54343);
locatorAddress += ',' + locatorHost + '[' + locatorPort + ']';
locatorAddressForPool += ',' + locatorHost + ':' + locatorPort;
Util.BBSet(string.Empty, "LOCATOR_ADDRESS", locatorAddress);
Util.BBSet(string.Empty, "LOCATOR_ADDRESS_POOL", locatorAddressForPool);
locCount++;
return true;
}
return false;
}
private static int StartLocalGFExe(string exeName, string gfeDir,
string exeArgs, out string outStr)
{
int status = GFNoError;
Process javaProc;
string javaExePath = gfeDir + PathSep + "bin" + PathSep + exeName;
Util.ServerLog("Executing local Geode command {0} {1}", javaExePath,
exeArgs);
if (!Util.StartProcess(javaExePath, exeArgs, false, TempDir,
true, false, false, out javaProc))
{
outStr = null;
return GFError;
}
StreamReader outSr = javaProc.StandardOutput;
// Wait for process to start
bool started = javaProc.WaitForExit(MaxWaitMillis);
outStr = outSr.ReadLine();
outSr.Close();
if (!started)
{
try
{
javaProc.Kill();
}
catch
{
}
status = GFTimeout;
}
else if (javaProc.ExitCode != 0)
{
status = GFError;
}
return status;
}
private static string StartRemoteGFExe(string host, string hostType,
string exeName, string exeArgs)
{
string gfJavaDir = Util.GetEnvironmentVariable("GFE_DIR", hostType);
string gfClassPath = Util.GetEnvironmentVariable("CLASSPATH", hostType);
string gfJava = Util.GetEnvironmentVariable("GF_JAVA", hostType);
string javaCmd = gfJavaDir + "/bin/" + exeName + ' ' + exeArgs;
Dictionary<string, string> envVars = new Dictionary<string, string>();
if (gfClassPath != null && gfClassPath.Length > 0)
{
envVars["CLASSPATH"] = gfClassPath;
}
if (gfJava != null && gfJava.Length > 0)
{
envVars["GF_JAVA"] = gfJava;
}
return Util.RunClientShellTask(Util.ClientId, host, javaCmd, envVars);
}
private string GetSlaveId(string serverNum)
{
return "slave." + serverNum;
}
private string CreateSlaveTaskSpecification(string progName,
string serverNum, string extraArgs)
{
extraArgs = (extraArgs == null ? string.Empty : ' ' + extraArgs);
return string.Format("<task name=\"Deleg{1}\" action=\"doRunProcess\" " +
"container=\"utils\" waitTime=\"25m\">{0}<data name=\"program\">" +
"{1}</data>{0}" + "<data name=\"arguments\">{2}{3}</data>{0}" +
"<client-set name=\"{4}\">{0}<client name=\"{5}\"/>{0}" +
"</client-set>{0}</task>", Environment.NewLine, progName, serverNum,
extraArgs, Util.ClientId, GetSlaveId(serverNum));
}
private string GetSslProperty(string hostType, bool forServer,string startDir)
{
string sslCmdStr = null;
//if (!File.Exists(startDir + PathSep + "geode.properties"))
//{
TextWriter sw = new StreamWriter(startDir + PathSep + "geode.properties", false);
String locatorAddress = GetJavaLocator();
sw.WriteLine("locators={0}", locatorAddress);
ResetKey("sslEnable");
bool isSslEnable = GetBoolValue("sslEnable");
if (!isSslEnable)
{
sw.Close();
return string.Empty;
}
FwkInfo("ssl is enable");
if (!forServer)
{
//StreamWriter sw = new StreamWriter("geode.properties",false);
sw.WriteLine("ssl-enabled=true");
sw.WriteLine("ssl-require-authentication=true");
sw.WriteLine("mcast-port=0");
sw.WriteLine("ssl-ciphers=SSL_RSA_WITH_NULL_SHA");
}
sw.Close();
string keyStorePath = Util.GetFwkLogDir(hostType) + "/data/keystore/sslcommon.jks";
string trustStorePath = Util.GetFwkLogDir(hostType) + "/data/keystore/server_truststore.jks";
sslCmdStr = " -Djavax.net.ssl.keyStore=" + keyStorePath +
" -Djavax.net.ssl.keyStorePassword=gemstone -Djavax.net.ssl.trustStore=" + trustStorePath +
" -Djavax.net.ssl.trustStorePassword=gemstone";
//string securityParams = GetStringValue(SecurityParams);
if (forServer)
{
sslCmdStr = " -J-Djavax.net.ssl.keyStore=" + keyStorePath +
" -J-Djavax.net.ssl.keyStorePassword=gemstone -J-Djavax.net.ssl.trustStore=" + trustStorePath +
" -J-Djavax.net.ssl.trustStorePassword=gemstone " +
" ssl-enabled=true ssl-require-authentication=true ssl-ciphers=SSL_RSA_WITH_NULL_SHA";
}
//}
return sslCmdStr;
}
private string GetServerSecurityArgs(string hostType)
{
string securityParams = GetStringValue(SecurityParams);
CredentialGenerator gen;
// no security means security params is not applicable
if (securityParams == null || securityParams.Length == 0 ||
(gen = GetCredentialGenerator()) == null)
{
FwkInfo("Security is DISABLED.");
return string.Empty;
}
string logDir = Util.GetFwkLogDir(hostType);
if (logDir == null || logDir.Length == 0)
{
logDir = Util.GetLogBaseDir(hostType);
logDir = logDir.Substring(0, logDir.LastIndexOf("/"));
logDir = logDir.Substring(0, logDir.LastIndexOf("/"));
}
string dataDir = logDir + "/data";
gen.Init(dataDir, dataDir);
Properties<string,string> extraProps = new Properties<string,string>();
if (gen.SystemProperties != null)
{
extraProps.AddAll(gen.SystemProperties);
}
// For now only XML based authorization is supported
AuthzCredentialGenerator authzGen = new XmlAuthzCredentialGenerator();
authzGen.Init(gen);
if (authzGen.SystemProperties != null)
{
extraProps.AddAll(authzGen.SystemProperties);
}
return Utility.GetServerArgs(gen.Authenticator,
authzGen.AccessControl, null, extraProps, gen.JavaProperties);
}
private void SetupJavaServers(string argStr)
{
int argIndx;
string endpoints = string.Empty;
m_numServers = 1;
string gfeDir = Util.GetEnvironmentVariable("GFE_DIR");
FwkAssert(gfeDir != null && gfeDir.Length != 0,
"SetupJavaServers() GFE_DIR is not set.");
string hostGroup = GetHostGroup();
Match mt;
if (argStr == "CPP")
{
// special string to denote that endpoints have to be obtained
// from C++ framework
string fwkBBPath = Util.AssemblyDir + "/../../bin/FwkBB";
string fwkBBArgs = "getInt GFE_BB EP_COUNT";
Process fwkBBProc;
if (!Util.StartProcess(fwkBBPath, fwkBBArgs, false, null,
true, false, false, out fwkBBProc))
{
FwkException("SetupJavaServers() Cannot start C++ FwkBB [{0}].",
fwkBBPath);
}
int numEndpoints = int.Parse(fwkBBProc.StandardOutput.ReadToEnd());
fwkBBProc.WaitForExit();
fwkBBProc.StandardOutput.Close();
for (int index = 1; index <= numEndpoints; index++)
{
fwkBBArgs = "get GFE_BB EndPoints_" + index;
if (!Util.StartProcess(fwkBBPath, fwkBBArgs, false, null,
true, false, false, out fwkBBProc))
{
FwkException("SetupJavaServers() Cannot start C++ FwkBB [{0}].",
fwkBBPath);
}
string endpoint = fwkBBProc.StandardOutput.ReadToEnd();
fwkBBProc.WaitForExit();
fwkBBProc.StandardOutput.Close();
if (endpoints.Length > 0)
{
endpoints += ',' + endpoint;
}
else
{
endpoints = endpoint;
}
}
}
else if ((mt = Regex.Match(gfeDir, "^[^:]+:[0-9]+(,[^:]+:[0-9]+)*$"))
!= null && mt.Length > 0)
{
// The GFE_DIR is for a remote server; contains an end-point list
endpoints = gfeDir;
}
else
{
string extraServerArgs;
string[] args = ParseJavaServerArgs(argStr, ref m_numServers,
out argIndx, out extraServerArgs);
Util.BBSet(JavaServerBB, FwkTest<TKey,TVal>.JavaServerEPCountKey, m_numServers);
FwkAssert(args.Length == argIndx + 1, "SetupJavaServers() cache XML argument not correct");
string cacheXml = args[argIndx];
FwkAssert(cacheXml.Length > 0, "SetupJavaServers() cacheXml argument is empty.");
string xmlDir = Util.GetEnvironmentVariable("GFBASE") + "/framework/xml";
if (xmlDir != null && xmlDir.Length > 0)
{
cacheXml = Util.NormalizePath(xmlDir + PathSep + cacheXml);
}
int javaServerPort = Util.RandPort(21321, 29789);
List<string> targetHosts = Util.BBGet(FwkReadData.HostGroupKey,
hostGroup) as List<string>;
for (int serverNum = 1; serverNum <= m_numServers; serverNum++)
{
if (m_exiting)
{
return;
}
string serverId = hostGroup + '_' + serverNum;
string startDir = GetJavaStartDir(serverId, Util.SystemType);
if (!Directory.Exists(startDir))
{
Directory.CreateDirectory(startDir);
}
string targetHost;
int numHosts = (targetHosts == null ? 0 : targetHosts.Count);
int lruMemSizeMb = 700;
if (numHosts == 0)
{
targetHost = Util.HostName;
lock (((IDictionary)m_javaServerHostMap).SyncRoot)
{
m_javaServerHostMap[serverId] = new HostInfo();
}
}
else
{
int hostIndex;
if (numHosts > 1)
{
hostIndex = ((serverNum - 1) % (numHosts - 1)) + 1;
}
else
{
hostIndex = 0;
}
targetHost = targetHosts[hostIndex];
FwkInfo("Checking the type of host {0}.", targetHost);
string hostType = Util.RunClientShellTask(Util.ClientId,
targetHost, "uname", null);
hostType = Util.GetHostType(hostType);
FwkInfo("The host {0} is: {1}", targetHost, hostType);
if (hostType != "WIN")
{
lruMemSizeMb = 1400;
}
lock (((IDictionary)m_javaServerHostMap).SyncRoot)
{
m_javaServerHostMap[serverId] = new HostInfo(false,
hostType, targetHost, extraServerArgs);
}
}
// Create the cache.xml with correct port
javaServerPort++;
StreamReader cacheXmlReader = new StreamReader(cacheXml);
string cacheXmlContent = cacheXmlReader.ReadToEnd();
cacheXmlReader.Close();
cacheXmlContent = cacheXmlContent.Replace("$PORT_NUM",
javaServerPort.ToString()).Replace("$LRU_MEM",
lruMemSizeMb.ToString());
StreamWriter cacheXmlWriter = new StreamWriter(startDir +
PathSep + "cache.xml");
cacheXmlWriter.Write(cacheXmlContent);
cacheXmlWriter.Close();
Util.ServerLog("SetupJavaServers() added '{0}' for host '{1}'",
serverId, targetHost);
FwkInfo("SetupJavaServers() added '{0}' for host '{1}'",
serverId, targetHost);
if (serverNum == 1)
{
endpoints = targetHost + ':' + javaServerPort;
}
else
{
endpoints += ',' + targetHost + ':' + javaServerPort;
}
Util.BBSet(JavaServerBB, FwkTest<TKey,TVal>.EndPoints + "_" + serverNum.ToString(), targetHost + ':' + javaServerPort);
}
int[] locatorPort = new int[2];
int locatorPort1;
HostInfo locatorInfo = GetHostInfo(hostGroup + "_1");
string locatorHost = locatorInfo.HostName;
string locatorType = locatorInfo.HostType;
if (locatorHost == null)
{
locatorHost = Util.HostName;
locatorType = Util.SystemType;
}
if (locatorType == Util.SystemType)
{
locatorHost = Util.HostName;
}
//ResetKey("multiLocator");
//bool isMultiLocator = GetBoolValue("multiLocator");
//if (isMultiLocator)
//{
for (int i = 0; i < 2; i++)
{
GetSetJavaLocator(locatorHost, out locatorPort1);
locatorPort[i] = locatorPort1;
}
for (int i = 0; i < 2; i++)
{
//if (GetSetJavaLocator(locatorHost, out locatorPort))
//{
FwkInfo("SetupJavaServers() starting locator on host {0}",
locatorHost);
string locatorDir = GetLocatorStartDir((i + 1).ToString(), Util.SystemType);
if (!Directory.Exists(locatorDir))
{
Directory.CreateDirectory(locatorDir);
}
ResetKey("sslEnable");
bool isSslEnable = GetBoolValue("sslEnable");
/*
if (isSslEnable)
{
locatorDir = locatorDir + "/..";
}
*/
string sslArg = GetSslProperty(locatorType, false, locatorDir);
locatorDir = GetLocatorStartDir((i + 1).ToString(), locatorType);
FwkInfo("ssl arguments: {0} {1}", sslArg, locatorDir);
string locatorArgs = "start-locator -port=" + locatorPort[i] +
" -dir=" + locatorDir + sslArg;
if (locatorType != Util.SystemType)
{
FwkInfo(StartRemoteGFExe(locatorHost, locatorType,
"geode", locatorArgs));
}
else
{
string outStr;
int status = StartLocalGFExe("geode.bat", gfeDir,
locatorArgs, out outStr);
if (status == GFTimeout)
{
FwkException("SetupJavaServers() Timed out while starting " +
"locator. Please check the logs in {0}", locatorDir);
}
else if (status != GFNoError)
{
FwkException("SetupJavaServers() Error while starting " +
"locator. Please check the logs in {0}", locatorDir);
}
FwkInfo(outStr);
}
if (isSslEnable)
{
Util.RegisterTestCompleteDelegate(TestCompleteWithSSL);
}
else
{
Util.RegisterTestCompleteDelegate(TestCompleteWithoutSSL);
}
m_locatorHost = locatorHost;
m_locatorType = locatorType;
FwkInfo("SetupJavaServers() started locator on host {0}.",
locatorHost);
//}
}
//}
}
FwkInfo("SetupJavaServers() endpoints: {0}", endpoints);
// Write the endpoints for both the tag and the cumulative one
string globalEndPoints;
try
{
globalEndPoints = Util.BBGet(string.Empty,
FwkTest<TKey, TVal>.EndPointTag) as string;
}
catch (Apache.Geode.DUnitFramework.KeyNotFoundException)
{
globalEndPoints = null;
}
if (globalEndPoints != null && globalEndPoints.Length > 0 &&
endpoints != null && endpoints.Length > 0)
{
globalEndPoints += ',' + endpoints;
}
else
{
globalEndPoints = endpoints;
}
Util.BBSet(JavaServerBB, FwkTest<TKey, TVal>.EndPointTag, globalEndPoints);
Util.BBSet(JavaServerBB, FwkTest<TKey, TVal>.EndPointTag + hostGroup, endpoints);
lock (((IDictionary)m_javaServerHostMap).SyncRoot)
{
Util.BBSet(JavaServerBB, JavaServerMapKey, m_javaServerHostMap);
}
FwkInfo("SetupJavaServers() completed.");
}
private void StartJavaServers(string argStr)
{
int numServers = -1;
int argIndx;
string extraServerArgs;
string[] args = ParseJavaServerArgs(argStr, ref numServers,
out argIndx, out extraServerArgs);
string gfeDir = Util.GetEnvironmentVariable("GFE_DIR");
string endpoints = string.Empty;
string locatorAddress = GetJavaLocator();
FwkAssert(gfeDir != null && gfeDir.Length != 0,
"StartJavaServers() GFE_DIR is not set.");
FwkAssert(locatorAddress != null && locatorAddress.Length > 0,
"StartJavaServers() LOCATOR_ADDRESS is not set.");
string hostGroup = GetHostGroup();
Match mt = Regex.Match(gfeDir, "^[^:]+:[0-9]+(,[^:]+:[0-9]+)*$");
if (mt == null || mt.Length == 0)
{
int startServer = 1;
int endServer;
if (args.Length == (argIndx + 1))
{
startServer = int.Parse(args[argIndx]);
endServer = (numServers == -1 ? startServer :
(startServer + numServers - 1));
}
else
{
endServer = (numServers == -1 ? m_numServers : numServers);
}
for (int serverNum = startServer; serverNum <= endServer; serverNum++)
{
if (m_exiting)
{
break;
}
string serverId = hostGroup + '_' + serverNum;
HostInfo hostInfo = GetHostInfo(serverId);
string targetHost = hostInfo.HostName;
string hostType = hostInfo.HostType;
string startDir = GetJavaStartDir(serverId, hostType);
extraServerArgs = hostInfo.ExtraServerArgs + ' ' + extraServerArgs;
string sslArg = GetSslProperty(hostType, true, startDir);
FwkInfo("ssl arguments for server: {0} ",sslArg);
string javaServerOtherArgs = GetServerSecurityArgs(hostType) + ' ' + sslArg;
if (javaServerOtherArgs != null && javaServerOtherArgs.Length > 0)
{
FwkInfo("StartJavaServers() Using security server args: {0}",
javaServerOtherArgs);
}
javaServerOtherArgs = JavaServerOtherArgs + ' ' + javaServerOtherArgs;
if (targetHost == null || targetHost.Length == 0 ||
Util.IsHostMyself(targetHost))
{
string cacheXml = startDir + PathSep + "cache.xml";
string javaServerArgs = "start" + JavaServerJavaArgs + " -dir=" +
startDir + " cache-xml-file=" + cacheXml + " locators=" +
locatorAddress + extraServerArgs + javaServerOtherArgs ;
// Assume the GFE_DIR is for starting a local server
FwkInfo("StartJavaServers() starting {0} with GFE_DIR {1} " +
"and arguments: {2}", JavaServerName, gfeDir, javaServerArgs);
string outStr;
int status = StartLocalGFExe(JavaServerName, gfeDir,
javaServerArgs, out outStr);
if (status == GFTimeout)
{
FwkException("StartJavaServers() Timed out waiting for Java " +
"cacheserver to start. Please check the server log in {0}.",
startDir);
}
else if (status != GFNoError)
{
FwkSevere("StartJavaServers() Error in starting Java " +
"cacheserver. Please check the server log in {0}.", startDir);
Thread.Sleep(60000);
}
FwkInfo("StartJavaServers() output from start script: {0}",
outStr);
}
else if (hostType != Util.SystemType)
{
FwkInfo("StartJavaServers() starting '{0}' on remote host " +
"'{1}' of type {2}", serverId, targetHost, hostType);
string javaCSArgs = "start" +
JavaServerJavaArgsUnix + " -dir=" + startDir +
" cache-xml-file=cache.xml locators=" + locatorAddress +
extraServerArgs + javaServerOtherArgs;
FwkInfo(StartRemoteGFExe(targetHost, hostType,
"cacheserver", javaCSArgs));
}
else
{
string taskSpec = CreateSlaveTaskSpecification(
"startJavaServers", serverNum.ToString(), null);
FwkInfo("StartJavaServers() starting '{0}' on host '{1}'",
serverId, targetHost);
Util.BBSet(Util.ClientId + '.' + GetSlaveId(
serverNum.ToString()), FwkReadData.HostGroupKey,
hostGroup);
if (!Util.RunClientWinTask(Util.ClientId, targetHost, taskSpec))
{
FwkException("StartJavaServers() failed to start '{0}' on host '{1}'",
serverId, targetHost);
}
}
lock (((IDictionary)m_javaServerHostMap).SyncRoot)
{
hostInfo.Started = true;
m_javaServerHostMap[serverId] = hostInfo;
}
Util.ServerLog("StartJavaServers() started '{0}' on host '{1}'",
serverId, targetHost);
}
}
ResetKey("sslEnable");
bool isSslEnabled = GetBoolValue("sslEnable");
if (isSslEnabled)
{
Util.RegisterTestCompleteDelegate(TestCompleteWithSSL);
}
else
{
Util.RegisterTestCompleteDelegate(TestCompleteWithoutSSL);
}
FwkInfo("StartJavaServers() completed.");
}
private void StopJavaServers(string argStr)
{
string gfeDir = Util.GetEnvironmentVariable("GFE_DIR");
FwkAssert(gfeDir != null && gfeDir.Length != 0,
"StopJavaServers() GFE_DIR is not set.");
Match mt = Regex.Match(gfeDir, "^[^:]+:[0-9]+(,[^:]+:[0-9]+)*$");
if (mt == null || mt.Length == 0)
{
int numServers = -1;
int argIndx;
string extraServerArgs;
string[] args = ParseJavaServerArgs(argStr, ref numServers,
out argIndx, out extraServerArgs);
string hostGroup = GetHostGroup();
string javaServerPath = gfeDir + PathSep + "bin" +
PathSep + JavaServerName;
// Assume the GFE_DIR is for stopping a local server
int startServer = 1;
int endServer;
if (args.Length == (argIndx + 1))
{
startServer = int.Parse(args[argIndx]);
endServer = (numServers == -1 ? startServer :
(startServer + numServers - 1));
}
else
{
endServer = (numServers == -1 ? m_numServers : numServers);
}
for (int serverNum = startServer; serverNum <= endServer; serverNum++)
{
if (m_exiting)
{
break;
}
string serverId = hostGroup + '_' + serverNum;
HostInfo hostInfo = GetHostInfo(serverId);
string targetHost = hostInfo.HostName;
string hostType = hostInfo.HostType;
string startDir = GetJavaStartDir(serverId, hostType);
string javaServerArgs = "stop -dir=" + startDir;
if (targetHost == null || targetHost.Length == 0 ||
Util.IsHostMyself(targetHost))
{
FwkInfo("StopJavaServers() stopping {0} with GFE_DIR {1} " +
"and arguments: {2}", JavaServerName, gfeDir, javaServerArgs);
string outStr;
int status = StartLocalGFExe(JavaServerName, gfeDir,
javaServerArgs, out outStr);
if (status != GFNoError)
{
if (status == GFTimeout)
{
FwkSevere("StopJavaServers() Timed out waiting for Java " +
"cacheserver to stop. Please check the server log in {0}.",
startDir);
}
else
{
Thread.Sleep(20000);
}
KillLocalJavaServer(serverId, "-9");
}
}
else if (hostType != Util.SystemType)
{
FwkInfo("StopJavaServers() stopping '{0}' on remote host " +
"'{1}' of type {2}", serverId, targetHost, hostType);
FwkInfo(StartRemoteGFExe(targetHost, hostType,
"cacheserver", javaServerArgs));
}
else
{
string taskSpec = CreateSlaveTaskSpecification(
"stopJavaServers", serverNum.ToString(), null);
FwkInfo("StopJavaServers() stopping '{0}' on host '{1}'",
serverId, targetHost);
if (!Util.RunClientWinTask(Util.ClientId, targetHost, taskSpec))
{
FwkSevere("StopJavaServers() failed to stop '{0}' on host '{1}'",
serverId, targetHost);
}
FwkInfo("StopJavaServers() stopped '{0}' on host '{1}'",
serverId, targetHost);
}
lock (((ICollection)m_javaServerHostMap).SyncRoot)
{
hostInfo.Started = false;
m_javaServerHostMap[serverId] = hostInfo;
}
Util.ServerLog("StopJavaServers() stopped '{0}' on host '{1}'",
serverId, targetHost);
}
}
FwkInfo("StopJavaServers() completed.");
}
private void KillJavaServers(string argStr)
{
string gfeDir = Util.GetEnvironmentVariable("GFE_DIR");
FwkAssert(gfeDir != null && gfeDir.Length != 0,
"KillJavaServers() GFE_DIR is not set.");
Match mt = Regex.Match(gfeDir, "^[^:]+:[0-9]+(,[^:]+:[0-9]+)*$");
if (mt == null || mt.Length == 0)
{
int numServers = -1;
int argIndx;
string extraServerArgs;
string[] args = ParseJavaServerArgs(argStr, ref numServers,
out argIndx, out extraServerArgs);
string hostGroup = GetHostGroup();
string javaServerPath = gfeDir + PathSep + "bin" +
PathSep + JavaServerName;
// Assume the GFE_DIR is for stopping a local server
int startServer = 1;
int endServer;
string signal = "15";
if (args.Length >= (argIndx + 1))
{
startServer = int.Parse(args[argIndx++]);
endServer = (numServers == -1 ? startServer :
(startServer + numServers - 1));
if (args.Length >= (argIndx + 1))
{
signal = args[argIndx];
}
}
else
{
endServer = (numServers == -1 ? m_numServers : numServers);
}
for (int serverNum = startServer; serverNum <= endServer; serverNum++)
{
if (m_exiting)
{
break;
}
string serverId = hostGroup + '_' + serverNum;
HostInfo hostInfo = GetHostInfo(serverId);
string targetHost = hostInfo.HostName;
string hostType = hostInfo.HostType;
if (targetHost == null || targetHost.Length == 0 ||
Util.IsHostMyself(targetHost))
{
FwkInfo("KillJavaServers() killing '{0}' on localhost", serverId);
KillLocalJavaServer(serverId, '-' + signal);
}
else if (hostType != Util.SystemType)
{
FwkInfo("KillJavaServers() killing '{0}' on remote host " +
"'{1}' of type {2}", serverId, targetHost, hostType);
string startDir = GetJavaStartDir(serverId, hostType);
string killCmd = '"' + Util.GetFwkLogDir(hostType) +
"/data/killJavaServer.sh\" " + signal + " \"" + startDir + '"';
FwkInfo(Util.RunClientShellTask(Util.ClientId, targetHost,
killCmd, null));
}
else
{
string taskSpec = CreateSlaveTaskSpecification(
"killJavaServers", serverNum.ToString(), signal);
FwkInfo("KillJavaServers() killing '{0}' on host '{1}'",
serverId, targetHost);
if (!Util.RunClientWinTask(Util.ClientId, targetHost, taskSpec))
{
FwkSevere("KillJavaServers() failed to kill '{0}' on host '{1}'",
serverId, targetHost);
}
}
lock (((ICollection)m_javaServerHostMap).SyncRoot)
{
hostInfo.Started = false;
m_javaServerHostMap[serverId] = hostInfo;
}
Util.ServerLog("KillJavaServers() killed '{0}' on host '{1}'",
serverId, targetHost);
}
}
FwkInfo("KillJavaServers() completed.");
}
private static string GetGFJavaPID(string javaLog)
{
string javaPID = null;
bool islocator = javaLog.Contains("locator.log");
using (FileStream fs = new FileStream(javaLog, FileMode.Open,
FileAccess.Read, FileShare.ReadWrite))
{
StreamReader sr = new StreamReader(fs);
Regex pidRE = new Regex(@"Process ID: [\s]*(?<PID>[^\s]*)",
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Multiline);
while (!sr.EndOfStream)
{
Match mt = pidRE.Match(sr.ReadLine());
if (mt != null && mt.Length > 0)
{
javaPID = mt.Groups["PID"].Value;
if(!islocator)
break;
}
}
sr.Close();
fs.Close();
}
return javaPID;
}
private static string GetJavaServerPID(string serverId)
{
string startDir = GetJavaStartDir(serverId, Util.SystemType);
string javaLog = startDir + "/cacheserver.log";
return GetGFJavaPID(javaLog);
}
private static bool KillLocalGFJava(string javaPID, string signal)
{
try
{
Process.GetProcessById(int.Parse(javaPID)).Kill();
return true;
}
catch (Exception excp)
{
LogException("KillLocalGFJava: {0}: {1}", excp.GetType().Name, excp.Message);
return false;
}
}
private static bool KillLocalGFJava_disabled(string javaPID, string signal)
{
bool success = false;
if (javaPID != null)
{
Process javaProc;
if (!Util.StartProcess("/bin/kill", "-f " + signal + ' ' + javaPID,
false, TempDir, false, false, false, out javaProc))
{
LogException("KillLocalGFJava(): unable to run 'kill'");
}
// Wait for java process to stop
bool stopped = javaProc.WaitForExit(MaxWaitMillis);
if (!stopped)
{
try
{
javaProc.Kill();
}
catch
{
}
LogSevere("KillLocalGFJava() Could not execute " +
"kill successfully.");
}
int numTries = 30;
while (numTries-- > 0)
{
if (!Util.StartProcess("/bin/kill", "-f -0 " + javaPID,
false, TempDir, false, false, false, out javaProc))
{
LogException("KillLocalGFJava(): unable to run 'kill'");
}
stopped = javaProc.WaitForExit(MaxWaitMillis);
if (stopped && javaProc.ExitCode == 0)
{
success = true;
break;
}
Thread.Sleep(10000);
}
}
return success;
}
private static void KillLocalJavaServer(string serverId, string signal)
{
string javaPID = GetJavaServerPID(serverId);
LogInfo("KillLocalJavaServer() killing '{0}' with PID '{1}' using " +
"signal '{2}'", serverId, javaPID, signal);
if (KillLocalGFJava(javaPID, signal))
{
string startDir = GetJavaStartDir(serverId, Util.SystemType);
File.Delete(startDir + "/.cacheserver.ser");
}
else
{
LogException("KillLocalJavaServer() Timed out waiting for " +
"Java cacheserver to stop.");
}
}
#endregion
#region Public methods loaded by XMLs
/// <summary>
/// Will run a process using Cygwin bash
/// </summary>
public void DoRunProcess()
{
string progName = GetStringValue("program");
//bool driverUsingpsexec = false;
bool hostIsWindows = false;
try
{
//driverUsingpsexec = (bool)Util.BBGet(string.Empty, "UsePsexec");
hostIsWindows = (bool)Util.BBGet(string.Empty, Util.HostName + ".IsWindows");
}
catch
{
}
if (progName == null)
{
FwkException("DoRunProcess(): program not specified for task {0}.",
TaskName);
}
string args = GetStringValue("arguments");
if (progName == "cp") // for smpke perf xml
{
if (hostIsWindows )//&& driverUsingpsexec)
progName = "copy";
string[] argStr = args.Split(' ');
int i = argStr[0].IndexOf("/framework/xml/");
string gfBaseDir = Util.GetEnvironmentVariable("GFBASE");
string sharePath = Util.NormalizePath(gfBaseDir);
string specName = argStr[0].Substring(i);
string perfStatictic = sharePath + specName;
args = perfStatictic + ' ' + (string)Util.BBGet(string.Empty,"XMLLOGDIR") + '/' + argStr[1];
}
string fullProg = progName + ' ' + args;
fullProg = fullProg.Trim();
// Special treatment for java server scripts since they are C++ specific
// (e.g. the environment variables they require, the FwkBBClient program,
// the auto-ssh ...)
string[] progs = fullProg.Split(';');
for (int index = 0; index < progs.Length; index++)
{
if (m_exiting)
{
break;
}
string prog = progs[index].Trim();
int javaIndx;
if ((javaIndx = prog.IndexOf(SetupJSName)) >= 0)
{
args = prog.Substring(javaIndx + SetupJSName.Length).Trim();
SetupJavaServers(args);
}
else if ((javaIndx = prog.IndexOf(StartJSName)) >= 0)
{
args = prog.Substring(javaIndx + StartJSName.Length).Trim();
StartJavaServers(args);
}
else if ((javaIndx = prog.IndexOf(StopJSName)) >= 0)
{
args = prog.Substring(javaIndx + StopJSName.Length).Trim();
StopJavaServers(args);
}
else if ((javaIndx = prog.IndexOf(KillJSName)) >= 0)
{
args = prog.Substring(javaIndx + KillJSName.Length).Trim();
KillJavaServers(args);
}
else
{
FwkInfo("DoRunProcess() starting '{0}' using bash", prog);
Process runProc = new Process();
ProcessStartInfo startInfo;
if (hostIsWindows)// && driverUsingpsexec)
{
prog = prog.Replace('/', '\\');
startInfo = new ProcessStartInfo("cmd",
string.Format("/C \"{0}\"", prog));
}
else
startInfo = new ProcessStartInfo("bash",
string.Format("-c \"{0}\"", prog));
startInfo.CreateNoWindow = true;
startInfo.UseShellExecute = false;
startInfo.RedirectStandardOutput = true;
startInfo.RedirectStandardError = true;
runProc.StartInfo = startInfo;
if (!runProc.Start())
{
FwkException("DoRunProcess() unable to run '{0}' using bash", prog);
}
StreamReader outSr = runProc.StandardOutput;
StreamReader errSr = runProc.StandardError;
string outStr = outSr.ReadToEnd();
string errStr = errSr.ReadToEnd();
runProc.WaitForExit();
errSr.Close();
outSr.Close();
if (outStr != null && outStr.Length > 0)
{
FwkInfo("DoRunProcess() Output from executing '{0}':" +
"{1}{2}{3}{4}", prog, Environment.NewLine, outStr,
Environment.NewLine, Util.MarkerString);
}
if (errStr != null && errStr.Length > 0)
{
FwkSevere("DoRunProcess() Error output from executing '{0}':" +
"{1}{2}{3}{4}", prog, Environment.NewLine, errStr,
Environment.NewLine, Util.MarkerString);
}
FwkInfo("DoRunProcess() completed '{0}'.", prog);
}
}
}
public void DoSleep()
{
int secs = GetUIntValue(SleepTime);
if (secs < 1)
{
secs = 30;
}
FwkInfo("DoSleep() called for task: '{0}', sleeping for {1} seconds.",
TaskName, secs);
Thread.Sleep(secs * 1000);
}
public void DoRunProcessAndSleep()
{
DoRunProcess();
if (!m_exiting)
{
int secs = GetUIntValue(SleepTime);
if (secs > 0)
{
Thread.Sleep(secs * 1000);
}
}
}
public void DoStopStartServer()
{
if (m_firstRun)
{
m_firstRun = false;
Util.BBIncrement(JavaServerBB, JavaServerCountKey);
}
string op = GetStringValue("operation");
if (op == null || op.Length == 0)
{
FwkException("DoStopStartServer(): operation not specified.");
}
string serverId = GetStringValue("ServerId");
if (serverId == null || serverId.Length == 0)
{
FwkException("DoStopStartServer(): server id not specified.");
}
FwkInfo("DoStopStartServer(): stopping and starting server {0}.",
serverId);
UnitFnMethod<string> stopDeleg = null;
string stopArg = null;
if (op == "stop")
{
stopDeleg = StopJavaServers;
stopArg = serverId;
}
else if (op == "term")
{
stopDeleg = KillJavaServers;
stopArg = serverId;
}
else if (op == "kill")
{
stopDeleg = KillJavaServers;
stopArg = serverId + " 9";
}
else
{
FwkException("DoStopStartServer(): unknown operation {0}.", op);
}
int secs = GetUIntValue(SleepTime);
int minServers = GetUIntValue(MinServers);
minServers = (minServers <= 0) ? 1 : minServers;
FwkInfo("DoStopStartServer(): using minServers: {0}", minServers);
bool done = false;
while (!done)
{
int numServers = Util.BBDecrement(JavaServerBB, JavaServerCountKey);
if (numServers >= minServers)
{
stopDeleg(stopArg);
Thread.Sleep(60000);
StartJavaServers(serverId);
Thread.Sleep(60000);
Util.BBIncrement(JavaServerBB, JavaServerCountKey);
done = true;
}
else
{
Util.BBIncrement(JavaServerBB, JavaServerCountKey);
Thread.Sleep(1000);
}
}
if (secs > 0)
{
Thread.Sleep(secs * 1000);
}
}
#endregion
public static void TestCompleteWithSSL()
{
TestComplete(true);
}
public static void TestCompleteWithoutSSL()
{
TestComplete(false);
}
public static void TestComplete(bool ssl)
{
m_exiting = true;
lock (((ICollection)m_javaServerHostMap).SyncRoot)
{
if (m_javaServerHostMap.Count == 0)
{
try
{
m_javaServerHostMap = Util.BBGet(JavaServerBB, JavaServerMapKey)
as Dictionary<string, HostInfo>;
}
catch
{
}
}
if (m_javaServerHostMap.Count > 0)
{
// Stop all the remaining java servers here.
string gfeDir = Util.GetEnvironmentVariable("GFE_DIR");
LogAssert(gfeDir != null, "ClientExit() GFE_DIR is not set.");
LogAssert(gfeDir.Length != 0, "ClientExit() GFE_DIR is not set.");
Match mt = Regex.Match(gfeDir, "^[^:]+:[0-9]+(,[^:]+:[0-9]+)*$");
if (mt == null || mt.Length == 0)
{
foreach (KeyValuePair<string, HostInfo> serverHostPair
in m_javaServerHostMap)
{
string serverId = serverHostPair.Key;
string targetHost = serverHostPair.Value.HostName;
string hostType = serverHostPair.Value.HostType;
string startDir = GetJavaStartDir(serverId, hostType);
string javaServerArgs = "stop -dir=" + startDir;
if (serverHostPair.Value.Started)
{
if (targetHost == null || targetHost.Length == 0 ||
Util.IsHostMyself(targetHost))
{
LogInfo("ClientExit() stopping {0} with GFE_DIR {1} and " +
"arguments: {2}", JavaServerName, gfeDir, javaServerArgs);
string outStr;
int status = StartLocalGFExe(JavaServerName, gfeDir,
javaServerArgs, out outStr);
if (status != GFNoError)
{
if (status == GFTimeout)
{
LogSevere("ClientExit() Timed out waiting for Java " +
"cacheserver to stop. Please check the server log " +
"in {0}.", startDir);
}
KillLocalJavaServer(serverId, "-9");
}
}
else if (hostType != Util.SystemType)
{
// Stop the remote host
LogInfo("ClientExit() stopping '{0}' on remote host " +
"'{1}' of type {2}", serverId, targetHost, hostType);
LogInfo(StartRemoteGFExe(targetHost, hostType,
"cacheserver", javaServerArgs));
}
}
}
}
m_javaServerHostMap.Clear();
}
// Stop the locator here.
if (m_locatorHost != null && m_locatorType != null)
{
LogInfo("ClientExit() stopping locator on host {0}", m_locatorHost);
for (int i = 0; i < locCount; i++)
{
string locatorDir = GetLocatorStartDir((i + 1).ToString(), Util.SystemType);
/* if (ssl)
{
locatorDir = locatorDir + "/..";
}
*/
string locatorPID = GetGFJavaPID(locatorDir +
PathSep + "locator.log");
if (locatorPID != null && locatorPID.Length > 0)
{
if (m_locatorType != Util.SystemType)
{
string killCmd = "kill -15 " + locatorPID;
LogInfo(Util.RunClientShellTask(Util.ClientId, m_locatorHost,
killCmd, null));
Thread.Sleep(3000);
killCmd = "kill -9 " + locatorPID;
LogInfo(Util.RunClientShellTask(Util.ClientId, m_locatorHost,
killCmd, null));
LogInfo("ClientExit() successfully stopped locator PID {0} on host {1}",
locatorPID, m_locatorHost);
}
else
{
if (!KillLocalGFJava(locatorPID, "-15") &&
!KillLocalGFJava(locatorPID, "-9"))
{
LogSevere("ClientExit() Error while stopping " +
"locator. Please check the logs in {0}", locatorDir);
}
else
{
LogInfo("ClientExit() successfully stopped locator on host {0}",
m_locatorHost);
}
}
}
}
}
}
locCount = 0;
m_locatorHost = null;
m_locatorType = null;
Util.BBRemove(string.Empty, "LOCATOR_ADDRESS");
Util.BBRemove(string.Empty, "LOCATOR_ADDRESS_POOL");
m_exiting = false;
}
}
}