| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.hadoop.yarn.client.cli; |
| |
| import java.io.File; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.io.PrintStream; |
| import java.net.ConnectException; |
| import java.net.SocketException; |
| import java.net.SocketTimeoutException; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.Collection; |
| import java.util.Collections; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.Iterator; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Map.Entry; |
| import java.util.Set; |
| import java.util.TreeMap; |
| import java.util.regex.Pattern; |
| |
| import javax.ws.rs.core.MediaType; |
| |
| import org.apache.commons.cli.CommandLine; |
| import org.apache.commons.cli.CommandLineParser; |
| import org.apache.commons.cli.GnuParser; |
| import org.apache.commons.cli.HelpFormatter; |
| import org.apache.commons.cli.Option; |
| import org.apache.commons.cli.Options; |
| import org.apache.commons.cli.ParseException; |
| import org.apache.commons.io.IOUtils; |
| import org.apache.commons.lang.StringUtils; |
| import org.apache.commons.math3.util.Pair; |
| import org.apache.hadoop.classification.InterfaceAudience.Private; |
| import org.apache.hadoop.classification.InterfaceAudience.Public; |
| import org.apache.hadoop.classification.InterfaceStability.Evolving; |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.conf.Configured; |
| import org.apache.hadoop.security.UserGroupInformation; |
| import org.apache.hadoop.util.Tool; |
| import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; |
| import org.apache.hadoop.yarn.api.records.ApplicationId; |
| import org.apache.hadoop.yarn.api.records.ApplicationReport; |
| import org.apache.hadoop.yarn.api.records.ContainerId; |
| import org.apache.hadoop.yarn.api.records.ContainerReport; |
| import org.apache.hadoop.yarn.api.records.ContainerState; |
| import org.apache.hadoop.yarn.api.records.YarnApplicationState; |
| import org.apache.hadoop.yarn.client.api.YarnClient; |
| import org.apache.hadoop.yarn.conf.YarnConfiguration; |
| import org.apache.hadoop.yarn.exceptions.YarnException; |
| import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo; |
| import org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest; |
| import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers; |
| import org.apache.hadoop.yarn.logaggregation.LogToolUtils; |
| import org.apache.hadoop.yarn.webapp.util.WebAppUtils; |
| import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils; |
| import org.codehaus.jettison.json.JSONArray; |
| import org.codehaus.jettison.json.JSONException; |
| import org.codehaus.jettison.json.JSONObject; |
| |
| import com.google.common.annotations.VisibleForTesting; |
| import com.sun.jersey.api.client.Client; |
| import com.sun.jersey.api.client.ClientHandlerException; |
| import com.sun.jersey.api.client.ClientRequest; |
| import com.sun.jersey.api.client.ClientResponse; |
| import com.sun.jersey.api.client.UniformInterfaceException; |
| import com.sun.jersey.api.client.WebResource; |
| import com.sun.jersey.api.client.filter.ClientFilter; |
| |
| @Public |
| @Evolving |
| public class LogsCLI extends Configured implements Tool { |
| |
| private static final String CONTAINER_ID_OPTION = "containerId"; |
| private static final String APPLICATION_ID_OPTION = "applicationId"; |
| private static final String NODE_ADDRESS_OPTION = "nodeAddress"; |
| private static final String APP_OWNER_OPTION = "appOwner"; |
| private static final String AM_CONTAINER_OPTION = "am"; |
| private static final String PER_CONTAINER_LOG_FILES_OPTION = "log_files"; |
| private static final String PER_CONTAINER_LOG_FILES_REGEX_OPTION |
| = "log_files_pattern"; |
| private static final String LIST_NODES_OPTION = "list_nodes"; |
| private static final String SHOW_APPLICATION_LOG_INFO |
| = "show_application_log_info"; |
| private static final String SHOW_CONTAINER_LOG_INFO |
| = "show_container_log_info"; |
| private static final String OUT_OPTION = "out"; |
| private static final String SIZE_OPTION = "size"; |
| private static final String CLIENT_MAX_RETRY_OPTION = "client_max_retries"; |
| private static final String CLIENT_RETRY_INTERVAL_OPTION |
| = "client_retry_interval_ms"; |
| public static final String HELP_CMD = "help"; |
| private static final String SIZE_LIMIT_OPTION = "size_limit_mb"; |
| |
| private PrintStream outStream = System.out; |
| private YarnClient yarnClient = null; |
| private Client webServiceClient = null; |
| |
| private static final int DEFAULT_MAX_RETRIES = 30; |
| private static final long DEFAULT_RETRY_INTERVAL = 1000; |
| |
| private static final long LOG_SIZE_LIMIT_DEFAULT = 10240L; |
| |
| private long logSizeLeft = LOG_SIZE_LIMIT_DEFAULT * 1024 * 1024; |
| private long specifedLogLimits = LOG_SIZE_LIMIT_DEFAULT; |
| |
| @Private |
| @VisibleForTesting |
| ClientConnectionRetry connectionRetry; |
| |
| @Override |
| public int run(String[] args) throws Exception { |
| try { |
| yarnClient = createYarnClient(); |
| webServiceClient = Client.create(); |
| return runCommand(args); |
| } finally { |
| if (yarnClient != null) { |
| yarnClient.close(); |
| } |
| } |
| } |
| |
| private int runCommand(String[] args) throws Exception { |
| Options opts = createCommandOpts(); |
| Options printOpts = createPrintOpts(opts); |
| if (args.length < 1) { |
| printHelpMessage(printOpts); |
| return -1; |
| } |
| if (args[0].equals("-help")) { |
| printHelpMessage(printOpts); |
| return 0; |
| } |
| CommandLineParser parser = new GnuParser(); |
| String appIdStr = null; |
| String containerIdStr = null; |
| String nodeAddress = null; |
| String appOwner = null; |
| boolean getAMContainerLogs = false; |
| boolean nodesList = false; |
| boolean showApplicationLogInfo = false; |
| boolean showContainerLogInfo = false; |
| boolean useRegex = false; |
| String[] logFiles = null; |
| String[] logFilesRegex = null; |
| List<String> amContainersList = new ArrayList<String>(); |
| String localDir = null; |
| long bytes = Long.MAX_VALUE; |
| boolean ignoreSizeLimit = false; |
| int maxRetries = DEFAULT_MAX_RETRIES; |
| long retryInterval = DEFAULT_RETRY_INTERVAL; |
| try { |
| CommandLine commandLine = parser.parse(opts, args, false); |
| appIdStr = commandLine.getOptionValue(APPLICATION_ID_OPTION); |
| containerIdStr = commandLine.getOptionValue(CONTAINER_ID_OPTION); |
| nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION); |
| appOwner = commandLine.getOptionValue(APP_OWNER_OPTION); |
| getAMContainerLogs = commandLine.hasOption(AM_CONTAINER_OPTION); |
| nodesList = commandLine.hasOption(LIST_NODES_OPTION); |
| localDir = commandLine.getOptionValue(OUT_OPTION); |
| showApplicationLogInfo = commandLine.hasOption( |
| SHOW_APPLICATION_LOG_INFO); |
| showContainerLogInfo = commandLine.hasOption(SHOW_CONTAINER_LOG_INFO); |
| if (getAMContainerLogs) { |
| try { |
| amContainersList = parseAMContainer(commandLine, printOpts); |
| } catch (NumberFormatException ex) { |
| System.err.println(ex.getMessage()); |
| return -1; |
| } |
| } |
| if (commandLine.hasOption(PER_CONTAINER_LOG_FILES_OPTION)) { |
| logFiles = commandLine.getOptionValues(PER_CONTAINER_LOG_FILES_OPTION); |
| } |
| if (commandLine.hasOption(PER_CONTAINER_LOG_FILES_REGEX_OPTION)) { |
| logFilesRegex = commandLine.getOptionValues( |
| PER_CONTAINER_LOG_FILES_REGEX_OPTION); |
| useRegex = true; |
| } |
| if (commandLine.hasOption(SIZE_OPTION)) { |
| bytes = Long.parseLong(commandLine.getOptionValue(SIZE_OPTION)); |
| } |
| if (commandLine.hasOption(CLIENT_MAX_RETRY_OPTION)) { |
| maxRetries = Integer.parseInt(commandLine.getOptionValue( |
| CLIENT_MAX_RETRY_OPTION)); |
| } |
| if (commandLine.hasOption(CLIENT_RETRY_INTERVAL_OPTION)) { |
| retryInterval = Long.parseLong(commandLine.getOptionValue( |
| CLIENT_RETRY_INTERVAL_OPTION)); |
| } |
| if (commandLine.hasOption(SIZE_LIMIT_OPTION)) { |
| specifedLogLimits = Long.parseLong(commandLine.getOptionValue( |
| SIZE_LIMIT_OPTION)); |
| logSizeLeft = specifedLogLimits * 1024 * 1024; |
| } |
| if (logSizeLeft < 0L) { |
| ignoreSizeLimit = true; |
| } |
| } catch (ParseException e) { |
| System.err.println("options parsing failed: " + e.getMessage()); |
| printHelpMessage(printOpts); |
| return -1; |
| } |
| |
| if (appIdStr == null && containerIdStr == null) { |
| System.err.println("Both applicationId and containerId are missing, " |
| + " one of them must be specified."); |
| printHelpMessage(printOpts); |
| return -1; |
| } |
| |
| ApplicationId appId = null; |
| if (appIdStr != null) { |
| try { |
| appId = ApplicationId.fromString(appIdStr); |
| } catch (Exception e) { |
| System.err.println("Invalid ApplicationId specified"); |
| return -1; |
| } |
| } |
| |
| if (containerIdStr != null) { |
| try { |
| ContainerId containerId = ContainerId.fromString(containerIdStr); |
| if (appId == null) { |
| appId = containerId.getApplicationAttemptId().getApplicationId(); |
| } else if (!containerId.getApplicationAttemptId().getApplicationId() |
| .equals(appId)) { |
| System.err.println("The Application:" + appId |
| + " does not have the container:" + containerId); |
| return -1; |
| } |
| } catch (Exception e) { |
| System.err.println("Invalid ContainerId specified"); |
| return -1; |
| } |
| } |
| |
| if (showApplicationLogInfo && showContainerLogInfo) { |
| System.err.println("Invalid options. Can only accept one of " |
| + "show_application_log_info/show_container_log_info."); |
| return -1; |
| } |
| |
| if (logFiles != null && logFiles.length > 0 && logFilesRegex != null |
| && logFilesRegex.length > 0) { |
| System.err.println("Invalid options. Can only accept one of " |
| + "log_files/log_files_pattern."); |
| return -1; |
| } |
| if (localDir != null) { |
| File file = new File(localDir); |
| if (file.exists() && file.isFile()) { |
| System.err.println("Invalid value for -out option. " |
| + "Please provide a directory."); |
| return -1; |
| } |
| } |
| |
| // Set up Retry WebService Client |
| connectionRetry = new ClientConnectionRetry(maxRetries, retryInterval); |
| ClientJerseyRetryFilter retryFilter = new ClientJerseyRetryFilter(); |
| webServiceClient.addFilter(retryFilter); |
| |
| LogCLIHelpers logCliHelper = new LogCLIHelpers(); |
| logCliHelper.setConf(getConf()); |
| |
| YarnApplicationState appState = YarnApplicationState.NEW; |
| ApplicationReport appReport = null; |
| try { |
| appReport = getApplicationReport(appId); |
| appState = appReport.getYarnApplicationState(); |
| if (appState == YarnApplicationState.NEW |
| || appState == YarnApplicationState.NEW_SAVING |
| || appState == YarnApplicationState.SUBMITTED) { |
| System.err.println("Logs are not available right now."); |
| return -1; |
| } |
| } catch (IOException | YarnException e) { |
| // If we can not get appReport from either RM or ATS |
| // We will assume that this app has already finished. |
| appState = YarnApplicationState.FINISHED; |
| System.err.println("Unable to get ApplicationState." |
| + " Attempting to fetch logs directly from the filesystem."); |
| } |
| |
| if (appOwner == null || appOwner.isEmpty()) { |
| appOwner = guessAppOwner(appReport, appId); |
| if (appOwner == null) { |
| System.err.println("Can not find the appOwner. " |
| + "Please specify the correct appOwner"); |
| System.err.println("Could not locate application logs for " + appId); |
| return -1; |
| } |
| } |
| |
| Set<String> logs = new HashSet<String>(); |
| if (fetchAllLogFiles(logFiles, logFilesRegex)) { |
| logs.add("ALL"); |
| } else if (logFiles != null && logFiles.length > 0) { |
| logs.addAll(Arrays.asList(logFiles)); |
| } else if (logFilesRegex != null && logFilesRegex.length > 0) { |
| logs.addAll(Arrays.asList(logFilesRegex)); |
| } |
| |
| |
| ContainerLogsRequest request = new ContainerLogsRequest(appId, |
| isApplicationFinished(appState), appOwner, nodeAddress, null, |
| containerIdStr, localDir, logs, bytes, null); |
| |
| if (showContainerLogInfo) { |
| return showContainerLogInfo(request, logCliHelper); |
| } |
| |
| if (nodesList) { |
| return showNodeLists(request, logCliHelper); |
| } |
| |
| if (showApplicationLogInfo) { |
| return showApplicationLogInfo(request, logCliHelper); |
| } |
| // To get am logs |
| if (getAMContainerLogs) { |
| return fetchAMContainerLogs(request, amContainersList, |
| logCliHelper, useRegex, ignoreSizeLimit); |
| } |
| |
| int resultCode = 0; |
| if (containerIdStr != null) { |
| return fetchContainerLogs(request, logCliHelper, useRegex, |
| ignoreSizeLimit); |
| } else { |
| if (nodeAddress == null) { |
| resultCode = fetchApplicationLogs(request, logCliHelper, useRegex, |
| ignoreSizeLimit); |
| } else { |
| System.err.println("Should at least provide ContainerId!"); |
| printHelpMessage(printOpts); |
| resultCode = -1; |
| } |
| } |
| return resultCode; |
| } |
| |
| private ApplicationReport getApplicationReport(ApplicationId appId) |
| throws IOException, YarnException { |
| return yarnClient.getApplicationReport(appId); |
| } |
| |
| @VisibleForTesting |
| protected YarnClient createYarnClient() { |
| YarnClient client = YarnClient.createYarnClient(); |
| client.init(getConf()); |
| client.start(); |
| return client; |
| } |
| |
| public static void main(String[] args) throws Exception { |
| Configuration conf = new YarnConfiguration(); |
| LogsCLI logDumper = new LogsCLI(); |
| logDumper.setConf(conf); |
| int exitCode = logDumper.run(args); |
| System.exit(exitCode); |
| } |
| |
| private void printHelpMessage(Options options) { |
| outStream.println("Retrieve logs for YARN applications."); |
| HelpFormatter formatter = new HelpFormatter(); |
| formatter.printHelp("yarn logs -applicationId <application ID> [OPTIONS]", |
| new Options()); |
| formatter.setSyntaxPrefix(""); |
| formatter.printHelp("general options are:", options); |
| } |
| |
| protected List<JSONObject> getAMContainerInfoForRMWebService( |
| Configuration conf, String appId) throws ClientHandlerException, |
| UniformInterfaceException, JSONException { |
| String webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(conf); |
| |
| WebResource webResource = webServiceClient.resource(webAppAddress); |
| |
| ClientResponse response = |
| webResource.path("ws").path("v1").path("cluster").path("apps") |
| .path(appId).path("appattempts").accept(MediaType.APPLICATION_JSON) |
| .get(ClientResponse.class); |
| JSONObject json = |
| response.getEntity(JSONObject.class).getJSONObject("appAttempts"); |
| JSONArray requests = json.getJSONArray("appAttempt"); |
| List<JSONObject> amContainersList = new ArrayList<JSONObject>(); |
| for (int i = 0; i < requests.length(); i++) { |
| amContainersList.add(requests.getJSONObject(i)); |
| } |
| return amContainersList; |
| } |
| |
| private List<JSONObject> getAMContainerInfoForAHSWebService( |
| Configuration conf, String appId) throws ClientHandlerException, |
| UniformInterfaceException, JSONException { |
| String webAppAddress = |
| WebAppUtils.getHttpSchemePrefix(conf) |
| + WebAppUtils.getAHSWebAppURLWithoutScheme(conf); |
| WebResource webResource = webServiceClient.resource(webAppAddress); |
| |
| ClientResponse response = |
| webResource.path("ws").path("v1").path("applicationhistory") |
| .path("apps").path(appId).path("appattempts") |
| .accept(MediaType.APPLICATION_JSON) |
| .get(ClientResponse.class); |
| JSONObject json = response.getEntity(JSONObject.class); |
| JSONArray requests = json.getJSONArray("appAttempt"); |
| List<JSONObject> amContainersList = new ArrayList<JSONObject>(); |
| for (int i = 0; i < requests.length(); i++) { |
| amContainersList.add(requests.getJSONObject(i)); |
| } |
| Collections.reverse(amContainersList); |
| return amContainersList; |
| } |
| |
| private boolean fetchAllLogFiles(String[] logFiles, String[] logFilesRegex) { |
| |
| // If no value is specified for the PER_CONTAINER_LOG_FILES_OPTION option |
| // and PER_CONTAINER_LOG_FILES_REGEX_OPTION |
| // we will assume all logs. |
| if ((logFiles == null || logFiles.length == 0) && ( |
| logFilesRegex == null || logFilesRegex.length == 0)) { |
| return true; |
| } |
| |
| if (logFiles != null && logFiles.length > 0) { |
| List<String> logs = Arrays.asList(logFiles); |
| if (logs.contains("ALL") || logs.contains("*")) { |
| return true; |
| } |
| } |
| |
| if (logFilesRegex != null && logFilesRegex.length > 0) { |
| List<String> logsRegex = Arrays.asList(logFilesRegex); |
| if (logsRegex.contains(".*")) { |
| return true; |
| } |
| } |
| |
| return false; |
| } |
| |
| private List<Pair<ContainerLogFileInfo, String>> getContainerLogFiles( |
| Configuration conf, String containerIdStr, String nodeHttpAddress) |
| throws IOException { |
| List<Pair<ContainerLogFileInfo, String>> logFileInfos |
| = new ArrayList<>(); |
| try { |
| WebResource webResource = webServiceClient |
| .resource(WebAppUtils.getHttpSchemePrefix(conf) + nodeHttpAddress); |
| ClientResponse response = |
| webResource.path("ws").path("v1").path("node").path("containers") |
| .path(containerIdStr).path("logs") |
| .accept(MediaType.APPLICATION_JSON) |
| .get(ClientResponse.class); |
| if (response.getStatusInfo().getStatusCode() == |
| ClientResponse.Status.OK.getStatusCode()) { |
| try { |
| JSONArray array = new JSONArray(); |
| JSONObject json = response.getEntity(JSONObject.class); |
| if (!json.has("containerLogsInfo")) { |
| return logFileInfos; |
| } |
| Object logsInfoObj = json.get("containerLogsInfo"); |
| if (logsInfoObj instanceof JSONObject) { |
| array.put((JSONObject)logsInfoObj); |
| } else if (logsInfoObj instanceof JSONArray) { |
| JSONArray logsArray = (JSONArray)logsInfoObj; |
| for (int i=0; i < logsArray.length(); i++) { |
| array.put(logsArray.getJSONObject(i)); |
| } |
| } |
| for (int i = 0; i < array.length(); i++) { |
| JSONObject log = array.getJSONObject(i); |
| String aggregateType = log.has("logAggregationType") ? |
| log.getString("logAggregationType") : "N/A"; |
| if (!log.has("containerLogInfo")) { |
| continue; |
| } |
| Object ob = log.get("containerLogInfo"); |
| if (ob instanceof JSONArray) { |
| JSONArray obArray = (JSONArray)ob; |
| for (int j = 0; j < obArray.length(); j++) { |
| logFileInfos.add(new Pair<ContainerLogFileInfo, String>( |
| generatePerContainerLogFileInfoFromJSON( |
| obArray.getJSONObject(j)), aggregateType)); |
| } |
| } else if (ob instanceof JSONObject) { |
| logFileInfos.add(new Pair<ContainerLogFileInfo, String>( |
| generatePerContainerLogFileInfoFromJSON( |
| (JSONObject)ob), aggregateType)); |
| } |
| } |
| } catch (Exception e) { |
| System.err.println("Unable to parse json from webservice. Error:"); |
| System.err.println(e.getMessage()); |
| throw new IOException(e); |
| } |
| } |
| |
| } catch (ClientHandlerException | UniformInterfaceException ex) { |
| System.err.println("Unable to fetch log files list"); |
| throw new IOException(ex); |
| } |
| return logFileInfos; |
| } |
| |
| private ContainerLogFileInfo generatePerContainerLogFileInfoFromJSON( |
| JSONObject meta) throws JSONException { |
| String fileName = meta.has("fileName") ? |
| meta.getString("fileName") : "N/A"; |
| String fileSize = meta.has("fileSize") ? |
| meta.getString("fileSize") : "N/A"; |
| String lastModificationTime = meta.has("lastModifiedTime") ? |
| meta.getString("lastModifiedTime") : "N/A"; |
| return new ContainerLogFileInfo(fileName, fileSize, |
| lastModificationTime); |
| } |
| |
| @Private |
| @VisibleForTesting |
| public int printContainerLogsFromRunningApplication(Configuration conf, |
| ContainerLogsRequest request, LogCLIHelpers logCliHelper, |
| boolean useRegex, boolean ignoreSizeLimit) throws IOException { |
| String containerIdStr = request.getContainerId().toString(); |
| String localDir = request.getOutputLocalDir(); |
| String nodeId = request.getNodeId(); |
| PrintStream out = LogToolUtils.createPrintStream(localDir, nodeId, |
| containerIdStr); |
| try { |
| boolean foundAnyLogs = false; |
| byte[] buffer = new byte[65536]; |
| for (String logFile : request.getLogTypes()) { |
| InputStream is = null; |
| try { |
| ClientResponse response = getResponeFromNMWebService(conf, |
| webServiceClient, request, logFile); |
| if (response != null && response.getStatusInfo().getStatusCode() == |
| ClientResponse.Status.OK.getStatusCode()) { |
| is = response.getEntityInputStream(); |
| int len = 0; |
| while((len = is.read(buffer)) != -1) { |
| out.write(buffer, 0, len); |
| } |
| out.println(); |
| } else { |
| out.println("Can not get any logs for the log file: " + logFile); |
| String msg = "Response from the NodeManager:" + nodeId + |
| " WebService is " + ((response == null) ? "null": |
| "not successful," + " HTTP error code: " + |
| response.getStatus() + ", Server response:\n" + |
| response.getEntity(String.class)); |
| out.println(msg); |
| } |
| out.flush(); |
| foundAnyLogs = true; |
| } catch (ClientHandlerException | UniformInterfaceException ex) { |
| System.err.println("Can not find the log file:" + logFile |
| + " for the container:" + containerIdStr + " in NodeManager:" |
| + nodeId); |
| } finally { |
| IOUtils.closeQuietly(is); |
| } |
| } |
| |
| if (foundAnyLogs) { |
| return 0; |
| } else { |
| return -1; |
| } |
| } finally { |
| logCliHelper.closePrintStream(out); |
| } |
| } |
| |
| @Private |
| @VisibleForTesting |
| public ContainerReport getContainerReport(String containerIdStr) |
| throws YarnException, IOException { |
| return yarnClient.getContainerReport( |
| ContainerId.fromString(containerIdStr)); |
| } |
| |
| private boolean isApplicationFinished(YarnApplicationState appState) { |
| return appState == YarnApplicationState.FINISHED |
| || appState == YarnApplicationState.FAILED |
| || appState == YarnApplicationState.KILLED; |
| } |
| |
| private int printAMContainerLogs(Configuration conf, |
| ContainerLogsRequest request, List<String> amContainers, |
| LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit) |
| throws Exception { |
| List<JSONObject> amContainersList = null; |
| List<ContainerLogsRequest> requests = |
| new ArrayList<ContainerLogsRequest>(); |
| boolean getAMContainerLists = false; |
| String appId = request.getAppId().toString(); |
| StringBuilder errorMessage = new StringBuilder(); |
| // We will call RM webservice to get all AppAttempts information. |
| // If we get nothing, we will try to call AHS webservice to get AppAttempts |
| // which includes nodeAddress for the AM Containers. |
| try { |
| amContainersList = getAMContainerInfoForRMWebService(conf, appId); |
| if (amContainersList != null && !amContainersList.isEmpty()) { |
| getAMContainerLists = true; |
| for (JSONObject amContainer : amContainersList) { |
| ContainerLogsRequest amRequest = new ContainerLogsRequest(request); |
| amRequest.setContainerId(amContainer.getString("containerId")); |
| String httpAddress = amContainer.getString("nodeHttpAddress"); |
| if (httpAddress != null && !httpAddress.isEmpty()) { |
| amRequest.setNodeHttpAddress(httpAddress); |
| } |
| amRequest.setNodeId(amContainer.getString("nodeId")); |
| requests.add(amRequest); |
| } |
| } |
| } catch (Exception ex) { |
| errorMessage.append(ex.getMessage() + "\n"); |
| if (request.isAppFinished()) { |
| if (!conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, |
| YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { |
| errorMessage.append("Please enable the timeline service " |
| + "and make sure the timeline server is running."); |
| } else { |
| try { |
| amContainersList = getAMContainerInfoForAHSWebService(conf, appId); |
| if (amContainersList != null && !amContainersList.isEmpty()) { |
| getAMContainerLists = true; |
| for (JSONObject amContainer : amContainersList) { |
| ContainerLogsRequest amRequest = new ContainerLogsRequest( |
| request); |
| amRequest.setContainerId( |
| amContainer.getString("amContainerId")); |
| requests.add(amRequest); |
| } |
| } |
| } catch (Exception e) { |
| errorMessage.append(e.getMessage()); |
| } |
| } |
| } |
| } |
| |
| if (!getAMContainerLists) { |
| System.err.println("Unable to get AM container informations " |
| + "for the application:" + appId); |
| System.err.println(errorMessage); |
| System.err.println("Can not get AMContainers logs for " |
| + "the application:" + appId + " with the appOwner:" |
| + request.getAppOwner()); |
| return -1; |
| } |
| |
| List<ContainerLogsRequest> candidates = new ArrayList<>(); |
| if (amContainers.contains("ALL")) { |
| candidates.addAll(requests); |
| outStream.println(); |
| outStream.println("Specified ALL for -am option. " |
| + "Printed logs for all am containers."); |
| } else { |
| for (String amContainer : amContainers) { |
| int amContainerId = Integer.parseInt(amContainer.trim()); |
| if (amContainerId == -1) { |
| candidates.add(requests.get(requests.size() - 1)); |
| } else { |
| if (amContainerId <= requests.size()) { |
| candidates.add(requests.get(amContainerId - 1)); |
| } else { |
| System.err.println(String.format("ERROR: Specified AM containerId" |
| + " (%s) exceeds the number of AM containers (%s).", |
| amContainerId, requests.size())); |
| return -1; |
| } |
| } |
| } |
| } |
| Map<String, ContainerLogsRequest> newOptions = new HashMap<>(); |
| if (request.isAppFinished()) { |
| newOptions = getMatchedLogTypesForFinishedApp(candidates, |
| logCliHelper, useRegex, ignoreSizeLimit); |
| } else { |
| newOptions = getMatchedLogTypesForRunningApp(candidates, useRegex, |
| ignoreSizeLimit); |
| } |
| for (Entry<String, ContainerLogsRequest> amRequest |
| : newOptions.entrySet()) { |
| outputAMContainerLogs(amRequest.getValue(), conf, logCliHelper, |
| useRegex, ignoreSizeLimit); |
| } |
| return 0; |
| } |
| |
| private void outputAMContainerLogs(ContainerLogsRequest request, |
| Configuration conf, LogCLIHelpers logCliHelper, boolean useRegex, |
| boolean ignoreSizeLimit) throws Exception { |
| String nodeHttpAddress = request.getNodeHttpAddress(); |
| String containerId = request.getContainerId(); |
| String nodeId = request.getNodeId(); |
| |
| if (request.isAppFinished()) { |
| if (containerId != null && !containerId.isEmpty()) { |
| if (nodeId != null && !nodeId.isEmpty()) { |
| logCliHelper.dumpAContainerLogsForLogType(request); |
| } else { |
| logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId( |
| request); |
| } |
| } |
| } else { |
| if (nodeHttpAddress != null && containerId != null |
| && !nodeHttpAddress.isEmpty() && !containerId.isEmpty()) { |
| ContainerState containerState = getContainerReport(containerId) |
| .getContainerState(); |
| request.setContainerState(containerState); |
| printContainerLogsFromRunningApplication(conf, |
| request, logCliHelper, useRegex, ignoreSizeLimit); |
| } |
| } |
| } |
| |
| private int showContainerLogInfo(ContainerLogsRequest request, |
| LogCLIHelpers logCliHelper) throws IOException, YarnException, |
| ClientHandlerException, UniformInterfaceException, JSONException { |
| if (!request.isAppFinished()) { |
| return printContainerInfoFromRunningApplication(request, logCliHelper); |
| } else { |
| return logCliHelper.printAContainerLogMetadata( |
| request, System.out, System.err); |
| } |
| } |
| |
| private int showNodeLists(ContainerLogsRequest request, |
| LogCLIHelpers logCliHelper) throws IOException { |
| if (!request.isAppFinished()) { |
| System.err.println("The -list_nodes command can be only used with " |
| + "finished applications"); |
| return -1; |
| } else { |
| logCliHelper.printNodesList(request, System.out, System.err); |
| return 0; |
| } |
| } |
| |
| private int showApplicationLogInfo(ContainerLogsRequest request, |
| LogCLIHelpers logCliHelper) throws IOException, YarnException { |
| String appState = "Application State: " |
| + (request.isAppFinished() ? "Completed." : "Running."); |
| if (!request.isAppFinished()) { |
| List<ContainerReport> reports = |
| getContainerReportsFromRunningApplication(request); |
| List<ContainerReport> filterReports = filterContainersInfo( |
| request, reports); |
| if (filterReports.isEmpty()) { |
| System.err.println("Can not find any containers for the application:" |
| + request.getAppId() + "."); |
| return -1; |
| } |
| outStream.println(appState); |
| for (ContainerReport report : filterReports) { |
| outStream.println(String.format(LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, |
| report.getContainerId(), report.getAssignedNode())); |
| } |
| return 0; |
| } else { |
| outStream.println(appState); |
| logCliHelper.printContainersList(request, System.out, System.err); |
| return 0; |
| } |
| } |
| |
| /** |
| * Create Command Options. |
| * @return the command options |
| */ |
| private Options createCommandOpts() { |
| Options opts = new Options(); |
| opts.addOption(HELP_CMD, false, "Displays help for all commands."); |
| Option appIdOpt = |
| new Option(APPLICATION_ID_OPTION, true, "ApplicationId (required)"); |
| opts.addOption(appIdOpt); |
| opts.addOption(CONTAINER_ID_OPTION, true, "ContainerId. " |
| + "By default, it will print all available logs." |
| + " Work with -log_files to get only specific logs. If specified, the" |
| + " applicationId can be omitted"); |
| opts.addOption(NODE_ADDRESS_OPTION, true, "NodeAddress in the format " |
| + "nodename:port"); |
| opts.addOption(APP_OWNER_OPTION, true, |
| "AppOwner (assumed to be current user if not specified)"); |
| Option amOption = new Option(AM_CONTAINER_OPTION, true, |
| "Prints the AM Container logs for this application. " |
| + "Specify comma-separated value to get logs for related AM " |
| + "Container. For example, If we specify -am 1,2, we will get " |
| + "the logs for the first AM Container as well as the second " |
| + "AM Container. To get logs for all AM Containers, use -am ALL. " |
| + "To get logs for the latest AM Container, use -am -1. " |
| + "By default, it will print all available logs. Work with -log_files " |
| + "to get only specific logs."); |
| amOption.setValueSeparator(','); |
| amOption.setArgs(Option.UNLIMITED_VALUES); |
| amOption.setArgName("AM Containers"); |
| opts.addOption(amOption); |
| Option logFileOpt = new Option(PER_CONTAINER_LOG_FILES_OPTION, true, |
| "Specify comma-separated value " |
| + "to get exact matched log files. Use \"ALL\" or \"*\" to " |
| + "fetch all the log files for the container."); |
| logFileOpt.setValueSeparator(','); |
| logFileOpt.setArgs(Option.UNLIMITED_VALUES); |
| logFileOpt.setArgName("Log File Name"); |
| opts.addOption(logFileOpt); |
| Option logFileRegexOpt = new Option(PER_CONTAINER_LOG_FILES_REGEX_OPTION, |
| true, "Specify comma-separated value " |
| + "to get matched log files by using java regex. Use \".*\" to " |
| + "fetch all the log files for the container."); |
| logFileRegexOpt.setValueSeparator(','); |
| logFileRegexOpt.setArgs(Option.UNLIMITED_VALUES); |
| logFileRegexOpt.setArgName("Log File Pattern"); |
| opts.addOption(logFileRegexOpt); |
| opts.addOption(SHOW_CONTAINER_LOG_INFO, false, |
| "Show the container log metadata, " |
| + "including log-file names, the size of the log files. " |
| + "You can combine this with --containerId to get log metadata for " |
| + "the specific container, or with --nodeAddress to get log metadata " |
| + "for all the containers on the specific NodeManager."); |
| opts.addOption(SHOW_APPLICATION_LOG_INFO, false, "Show the " |
| + "containerIds which belong to the specific Application. " |
| + "You can combine this with --nodeAddress to get containerIds " |
| + "for all the containers on the specific NodeManager."); |
| opts.addOption(LIST_NODES_OPTION, false, |
| "Show the list of nodes that successfully aggregated logs. " |
| + "This option can only be used with finished applications."); |
| opts.addOption(OUT_OPTION, true, "Local directory for storing individual " |
| + "container logs. The container logs will be stored based on the " |
| + "node the container ran on."); |
| opts.addOption(SIZE_OPTION, true, "Prints the log file's first 'n' bytes " |
| + "or the last 'n' bytes. Use negative values as bytes to read from " |
| + "the end and positive values as bytes to read from the beginning."); |
| opts.addOption(CLIENT_MAX_RETRY_OPTION, true, "Set max retry number for a" |
| + " retry client to get the container logs for the running " |
| + "applications. Use a negative value to make retry forever. " |
| + "The default value is 30."); |
| opts.addOption(CLIENT_RETRY_INTERVAL_OPTION, true, |
| "Work with --client_max_retries to create a retry client. " |
| + "The default value is 1000."); |
| opts.addOption(SIZE_LIMIT_OPTION, true, "Use this option to limit " |
| + "the size of the total logs which could be fetched. " |
| + "By default, we only allow to fetch at most " |
| + LOG_SIZE_LIMIT_DEFAULT + " MB logs. If the total log size is " |
| + "larger than the specified number, the CLI would fail. " |
| + "The user could specify -1 to ignore the size limit " |
| + "and fetch all logs."); |
| opts.getOption(APPLICATION_ID_OPTION).setArgName("Application ID"); |
| opts.getOption(CONTAINER_ID_OPTION).setArgName("Container ID"); |
| opts.getOption(NODE_ADDRESS_OPTION).setArgName("Node Address"); |
| opts.getOption(APP_OWNER_OPTION).setArgName("Application Owner"); |
| opts.getOption(AM_CONTAINER_OPTION).setArgName("AM Containers"); |
| opts.getOption(OUT_OPTION).setArgName("Local Directory"); |
| opts.getOption(SIZE_OPTION).setArgName("size"); |
| opts.getOption(CLIENT_MAX_RETRY_OPTION).setArgName("Max Retries"); |
| opts.getOption(CLIENT_RETRY_INTERVAL_OPTION) |
| .setArgName("Retry Interval"); |
| opts.getOption(SIZE_LIMIT_OPTION).setArgName("Size Limit"); |
| return opts; |
| } |
| |
| /** |
| * Create Print options for helper message. |
| * @param commandOpts the options |
| * @return the print options |
| */ |
| private Options createPrintOpts(Options commandOpts) { |
| Options printOpts = new Options(); |
| printOpts.addOption(commandOpts.getOption(HELP_CMD)); |
| printOpts.addOption(commandOpts.getOption(CONTAINER_ID_OPTION)); |
| printOpts.addOption(commandOpts.getOption(NODE_ADDRESS_OPTION)); |
| printOpts.addOption(commandOpts.getOption(APP_OWNER_OPTION)); |
| printOpts.addOption(commandOpts.getOption(AM_CONTAINER_OPTION)); |
| printOpts.addOption(commandOpts.getOption(PER_CONTAINER_LOG_FILES_OPTION)); |
| printOpts.addOption(commandOpts.getOption(LIST_NODES_OPTION)); |
| printOpts.addOption(commandOpts.getOption(SHOW_APPLICATION_LOG_INFO)); |
| printOpts.addOption(commandOpts.getOption(SHOW_CONTAINER_LOG_INFO)); |
| printOpts.addOption(commandOpts.getOption(OUT_OPTION)); |
| printOpts.addOption(commandOpts.getOption(SIZE_OPTION)); |
| printOpts.addOption(commandOpts.getOption( |
| PER_CONTAINER_LOG_FILES_REGEX_OPTION)); |
| printOpts.addOption(commandOpts.getOption(CLIENT_MAX_RETRY_OPTION)); |
| printOpts.addOption(commandOpts.getOption(CLIENT_RETRY_INTERVAL_OPTION)); |
| printOpts.addOption(commandOpts.getOption(SIZE_LIMIT_OPTION)); |
| return printOpts; |
| } |
| |
| private List<String> parseAMContainer(CommandLine commandLine, |
| Options printOpts) throws NumberFormatException { |
| List<String> amContainersList = new ArrayList<String>(); |
| String[] amContainers = commandLine.getOptionValues(AM_CONTAINER_OPTION); |
| for (String am : amContainers) { |
| boolean errorInput = false; |
| if (!am.trim().equalsIgnoreCase("ALL")) { |
| try { |
| int id = Integer.parseInt(am.trim()); |
| if (id != -1 && id <= 0) { |
| errorInput = true; |
| } |
| } catch (NumberFormatException ex) { |
| errorInput = true; |
| } |
| if (errorInput) { |
| String errMessage = |
| "Invalid input for option -am. Valid inputs are 'ALL', -1 " |
| + "and any other integer which is larger than 0."; |
| printHelpMessage(printOpts); |
| throw new NumberFormatException(errMessage); |
| } |
| amContainersList.add(am.trim()); |
| } else { |
| amContainersList.add("ALL"); |
| break; |
| } |
| } |
| return amContainersList; |
| } |
| |
| private int fetchAMContainerLogs(ContainerLogsRequest request, |
| List<String> amContainersList, LogCLIHelpers logCliHelper, |
| boolean useRegex, boolean ignoreSizeLimit) throws Exception { |
| return printAMContainerLogs(getConf(), request, amContainersList, |
| logCliHelper, useRegex, ignoreSizeLimit); |
| } |
| |
| private int fetchContainerLogs(ContainerLogsRequest request, |
| LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit) |
| throws IOException, ClientHandlerException, UniformInterfaceException, |
| JSONException { |
| String appIdStr = request.getAppId().toString(); |
| String containerIdStr = request.getContainerId(); |
| String nodeAddress = request.getNodeId(); |
| String appOwner = request.getAppOwner(); |
| boolean isAppFinished = request.isAppFinished(); |
| // if the application is in the final state, |
| // we could directly get logs from HDFS. |
| if (isAppFinished) { |
| // if user specified "ALL" as the logFiles param, pass empty list |
| // to logCliHelper so that it fetches all the logs |
| ContainerLogsRequest newOptions = getMatchedLogOptions( |
| request, logCliHelper, useRegex, ignoreSizeLimit); |
| if (newOptions == null) { |
| System.err.println("Can not find any log file matching the pattern: " |
| + request.getLogTypes() + " for the container: " |
| + request.getContainerId() + " within the application: " |
| + request.getAppId()); |
| return -1; |
| } |
| if (nodeAddress != null && !nodeAddress.isEmpty()) { |
| return logCliHelper.dumpAContainerLogsForLogType(newOptions); |
| } else { |
| return logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId( |
| newOptions); |
| } |
| } |
| String nodeHttpAddress = null; |
| String nodeId = null; |
| try { |
| // If the nodeAddress is not provided, we will try to get |
| // the ContainerReport. In the containerReport, we could get |
| // nodeAddress and nodeHttpAddress |
| ContainerReport report = getContainerReport(containerIdStr); |
| nodeHttpAddress = report.getNodeHttpAddress(); |
| if (nodeHttpAddress != null && !nodeHttpAddress.isEmpty()) { |
| nodeHttpAddress = nodeHttpAddress.replaceFirst( |
| WebAppUtils.getHttpSchemePrefix(getConf()), ""); |
| request.setNodeHttpAddress(nodeHttpAddress); |
| } |
| nodeId = report.getAssignedNode().toString(); |
| request.setNodeId(nodeId); |
| request.setContainerState(report.getContainerState()); |
| } catch (IOException | YarnException ex) { |
| nodeHttpAddress = getNodeHttpAddressFromRMWebString(request); |
| if (nodeHttpAddress != null && !nodeHttpAddress.isEmpty()) { |
| request.setNodeHttpAddress(nodeHttpAddress); |
| } else { |
| // for the case, we have already uploaded partial logs in HDFS |
| int result = -1; |
| ContainerLogsRequest newOptions = getMatchedLogOptions( |
| request, logCliHelper, useRegex, ignoreSizeLimit); |
| if (newOptions == null) { |
| System.err.println("Can not find any log file matching the pattern: " |
| + request.getLogTypes() + " for the container: " |
| + request.getContainerId() + " within the application: " |
| + request.getAppId()); |
| } else { |
| if (nodeAddress != null && !nodeAddress.isEmpty()) { |
| result = logCliHelper.dumpAContainerLogsForLogType(newOptions); |
| } else { |
| result = logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId( |
| newOptions); |
| } |
| } |
| if (result == -1) { |
| System.err.println( |
| "Unable to get logs for this container:" |
| + containerIdStr + " for the application:" |
| + appIdStr + " with the appOwner: " + appOwner); |
| System.err.println("The application: " + appIdStr |
| + " is still running, and we can not get Container report " |
| + "for the container: " + containerIdStr + ". Please try later " |
| + "or after the application finishes."); |
| } |
| return result; |
| } |
| } |
| // If the application is not in the final state, |
| // we will provide the NodeHttpAddress and get the container logs |
| // by calling NodeManager webservice. |
| ContainerLogsRequest newRequest = getMatchedOptionForRunningApp( |
| request, useRegex, ignoreSizeLimit); |
| if (newRequest == null) { |
| return -1; |
| } |
| return printContainerLogsFromRunningApplication(getConf(), request, |
| logCliHelper, useRegex, ignoreSizeLimit); |
| } |
| |
| private int fetchApplicationLogs(ContainerLogsRequest options, |
| LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit) |
| throws IOException, YarnException { |
| // If the application has finished, we would fetch the logs |
| // from HDFS. |
| // If the application is still running, we would get the full |
| // list of the containers first, then fetch the logs for each |
| // container from NM. |
| int resultCode = -1; |
| if (options.isAppFinished()) { |
| ContainerLogsRequest newOptions = getMatchedLogOptions( |
| options, logCliHelper, useRegex, ignoreSizeLimit); |
| if (newOptions == null) { |
| System.err.println("Can not find any log file matching the pattern: " |
| + options.getLogTypes() + " for the application: " |
| + options.getAppId()); |
| } else { |
| resultCode = |
| logCliHelper.dumpAllContainersLogs(newOptions); |
| } |
| } else { |
| List<ContainerLogsRequest> containerLogRequests = |
| getContainersLogRequestForRunningApplication(options); |
| |
| // get all matched container log types and check the total log size. |
| Map<String, ContainerLogsRequest> matchedLogTypes = |
| getMatchedLogTypesForRunningApp(containerLogRequests, |
| useRegex, ignoreSizeLimit); |
| |
| for (Entry<String, ContainerLogsRequest> container |
| : matchedLogTypes.entrySet()) { |
| int result = printContainerLogsFromRunningApplication(getConf(), |
| container.getValue(), logCliHelper, |
| useRegex, ignoreSizeLimit); |
| if (result == 0) { |
| resultCode = 0; |
| } |
| } |
| } |
| if (resultCode == -1) { |
| System.err.println("Can not find the logs for the application: " |
| + options.getAppId() + " with the appOwner: " |
| + options.getAppOwner()); |
| } |
| return resultCode; |
| } |
| |
| private String guessAppOwner(ApplicationReport appReport, |
| ApplicationId appId) throws IOException { |
| String appOwner = null; |
| if (appReport != null) { |
| //always use the app owner from the app report if possible |
| appOwner = appReport.getUser(); |
| } else { |
| appOwner = UserGroupInformation.getCurrentUser().getShortUserName(); |
| appOwner = LogCLIHelpers.getOwnerForAppIdOrNull( |
| appId, appOwner, getConf()); |
| } |
| return appOwner; |
| } |
| |
| private ContainerLogsRequest getMatchedLogOptions( |
| ContainerLogsRequest request, LogCLIHelpers logCliHelper, |
| boolean useRegex, boolean ignoreSizeLimit) throws IOException { |
| ContainerLogsRequest newOptions = new ContainerLogsRequest(request); |
| Set<ContainerLogFileInfo> files = logCliHelper.listContainerLogs( |
| request); |
| Set<String> matchedFiles = getMatchedLogFiles(request, files, |
| useRegex, ignoreSizeLimit); |
| if (matchedFiles.isEmpty()) { |
| return null; |
| } else { |
| newOptions.setLogTypes(matchedFiles); |
| return newOptions; |
| } |
| } |
| |
| private Set<String> getMatchedLogFiles(ContainerLogsRequest options, |
| Collection<ContainerLogFileInfo> candidate, boolean useRegex, |
| boolean ignoreSizeLimit) throws IOException { |
| Set<String> matchedFiles = new HashSet<String>(); |
| Set<String> filePattern = options.getLogTypes(); |
| long size = options.getBytes(); |
| boolean getAll = options.getLogTypes().contains("ALL"); |
| Iterator<ContainerLogFileInfo> iterator = candidate.iterator(); |
| while(iterator.hasNext()) { |
| boolean matchedFile = false; |
| ContainerLogFileInfo logInfo = iterator.next(); |
| if (getAll) { |
| matchedFile = true; |
| } else if (useRegex) { |
| if (isFileMatching(logInfo.getFileName(), filePattern)) { |
| matchedFile = true; |
| } |
| } else { |
| if (filePattern.contains(logInfo.getFileName())) { |
| matchedFile = true; |
| } |
| } |
| if (matchedFile) { |
| matchedFiles.add(logInfo.getFileName()); |
| if (!ignoreSizeLimit) { |
| decrLogSizeLimit(Math.min( |
| Long.parseLong(logInfo.getFileSize()), size)); |
| if (getLogSizeLimitLeft() < 0) { |
| throw new RuntimeException("The total log size is too large." |
| + "The log size limit is " + specifedLogLimits + "MB. " |
| + "Please specify a proper value --size option or if you " |
| + "really want to fetch all, please " |
| + "specify -1 for --size_limit_mb option."); |
| } |
| } |
| } |
| } |
| return matchedFiles; |
| } |
| |
| private boolean isFileMatching(String fileType, |
| Set<String> logTypes) { |
| for (String logType : logTypes) { |
| Pattern filterPattern = Pattern.compile(logType); |
| boolean match = filterPattern.matcher(fileType).find(); |
| if (match) { |
| return true; |
| } |
| } |
| return false; |
| } |
| |
| private List<ContainerLogsRequest> |
| getContainersLogRequestForRunningApplication( |
| ContainerLogsRequest options) throws YarnException, IOException { |
| List<ContainerLogsRequest> newOptionsList = |
| new ArrayList<ContainerLogsRequest>(); |
| List<ContainerReport> reports = |
| getContainerReportsFromRunningApplication(options); |
| for (ContainerReport container : reports) { |
| ContainerLogsRequest newOptions = new ContainerLogsRequest(options); |
| newOptions.setContainerId(container.getContainerId().toString()); |
| newOptions.setNodeId(container.getAssignedNode().toString()); |
| String httpAddress = container.getNodeHttpAddress(); |
| if (httpAddress != null && !httpAddress.isEmpty()) { |
| newOptions.setNodeHttpAddress(httpAddress |
| .replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), "")); |
| } |
| newOptions.setContainerState(container.getContainerState()); |
| newOptionsList.add(newOptions); |
| } |
| return newOptionsList; |
| } |
| |
| private List<ContainerReport> getContainerReportsFromRunningApplication( |
| ContainerLogsRequest options) throws YarnException, IOException { |
| List<ContainerReport> reports = new ArrayList<ContainerReport>(); |
| List<ApplicationAttemptReport> attempts = |
| yarnClient.getApplicationAttempts(options.getAppId()); |
| Map<ContainerId, ContainerReport> containerMap = new TreeMap< |
| ContainerId, ContainerReport>(); |
| for (ApplicationAttemptReport attempt : attempts) { |
| List<ContainerReport> containers = yarnClient.getContainers( |
| attempt.getApplicationAttemptId()); |
| for (ContainerReport container : containers) { |
| if (!containerMap.containsKey(container.getContainerId())) { |
| containerMap.put(container.getContainerId(), container); |
| } |
| } |
| } |
| reports.addAll(containerMap.values()); |
| return reports; |
| } |
| |
| // filter the containerReports based on the nodeId and ContainerId |
| private List<ContainerReport> filterContainersInfo( |
| ContainerLogsRequest options, List<ContainerReport> containers) { |
| List<ContainerReport> filterReports = new ArrayList<ContainerReport>( |
| containers); |
| String nodeId = options.getNodeId(); |
| boolean filterBasedOnNodeId = (nodeId != null && !nodeId.isEmpty()); |
| String containerId = options.getContainerId(); |
| boolean filterBasedOnContainerId = (containerId != null |
| && !containerId.isEmpty()); |
| |
| if (filterBasedOnNodeId || filterBasedOnContainerId) { |
| // filter the reports based on the containerId and.or nodeId |
| for(ContainerReport report : containers) { |
| if (filterBasedOnContainerId) { |
| if (!report.getContainerId().toString() |
| .equalsIgnoreCase(containerId)) { |
| filterReports.remove(report); |
| } |
| } |
| |
| if (filterBasedOnNodeId) { |
| if (!report.getAssignedNode().toString().equalsIgnoreCase(nodeId)) { |
| filterReports.remove(report); |
| } |
| } |
| } |
| } |
| return filterReports; |
| } |
| |
| private int printContainerInfoFromRunningApplication( |
| ContainerLogsRequest options, LogCLIHelpers logCliHelper) |
| throws YarnException, IOException, ClientHandlerException, |
| UniformInterfaceException, JSONException { |
| String containerIdStr = options.getContainerId(); |
| String nodeIdStr = options.getNodeId(); |
| List<ContainerReport> reports = |
| getContainerReportsFromRunningApplication(options); |
| List<ContainerReport> filteredReports = filterContainersInfo( |
| options, reports); |
| if (filteredReports.isEmpty()) { |
| // if we specify the containerId as well as NodeAddress |
| String nodeHttpAddress = null; |
| if (options.getContainerId() != null |
| && !options.getContainerId().isEmpty()) { |
| nodeHttpAddress = getNodeHttpAddressFromRMWebString(options); |
| } |
| if (nodeHttpAddress != null) { |
| outputContainerLogMeta(options.getContainerId(), options.getNodeId(), |
| nodeHttpAddress); |
| return 0; |
| } else { |
| int result = logCliHelper.printAContainerLogMetadata( |
| options, System.out, System.err); |
| if (result == -1) { |
| StringBuilder sb = new StringBuilder(); |
| if (containerIdStr != null && !containerIdStr.isEmpty()) { |
| sb.append("Trying to get container with ContainerId: " |
| + containerIdStr + "\n"); |
| } |
| if (nodeIdStr != null && !nodeIdStr.isEmpty()) { |
| sb.append("Trying to get container from NodeManager: " |
| + nodeIdStr + "\n"); |
| } |
| sb.append("Can not find any matched containers for the application: " |
| + options.getAppId()); |
| System.err.println(sb.toString()); |
| } |
| return result; |
| } |
| } |
| for (ContainerReport report : filteredReports) { |
| String nodeId = report.getAssignedNode().toString(); |
| String nodeHttpAddress = report.getNodeHttpAddress().replaceFirst( |
| WebAppUtils.getHttpSchemePrefix(getConf()), ""); |
| String containerId = report.getContainerId().toString(); |
| outputContainerLogMeta(containerId, nodeId, nodeHttpAddress); |
| } |
| return 0; |
| } |
| |
| private void outputContainerLogMeta(String containerId, String nodeId, |
| String nodeHttpAddress) throws IOException { |
| String containerString = String.format( |
| LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, containerId, nodeId); |
| outStream.println(containerString); |
| outStream.println(StringUtils.repeat("=", containerString.length())); |
| outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN, |
| "LogFile", "LogLength", "LastModificationTime", "LogAggregationType"); |
| outStream.println(StringUtils.repeat("=", containerString.length() * 2)); |
| List<Pair<ContainerLogFileInfo, String>> infos = getContainerLogFiles( |
| getConf(), containerId, nodeHttpAddress); |
| for (Pair<ContainerLogFileInfo, String> info : infos) { |
| outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN, |
| info.getKey().getFileName(), info.getKey().getFileSize(), |
| info.getKey().getLastModifiedTime(), info.getValue()); |
| } |
| } |
| |
| @VisibleForTesting |
| public Set<String> getMatchedContainerLogFiles(ContainerLogsRequest request, |
| boolean useRegex, boolean ignoreSizeLimit) throws IOException { |
| // fetch all the log files for the container |
| // filter the log files based on the given -log_files pattern |
| List<Pair<ContainerLogFileInfo, String>> allLogFileInfos= |
| getContainerLogFiles(getConf(), request.getContainerId(), |
| request.getNodeHttpAddress()); |
| List<ContainerLogFileInfo> fileNames = new ArrayList< |
| ContainerLogFileInfo>(); |
| for (Pair<ContainerLogFileInfo, String> fileInfo : allLogFileInfos) { |
| fileNames.add(fileInfo.getKey()); |
| } |
| return getMatchedLogFiles(request, fileNames, |
| useRegex, ignoreSizeLimit); |
| } |
| |
| @VisibleForTesting |
| public ClientResponse getResponeFromNMWebService(Configuration conf, |
| Client webServiceClient, ContainerLogsRequest request, String logFile) { |
| WebResource webResource = |
| webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf) |
| + request.getNodeHttpAddress()); |
| return webResource.path("ws").path("v1").path("node") |
| .path("containers").path(request.getContainerId()).path("logs") |
| .path(logFile) |
| .queryParam("size", Long.toString(request.getBytes())) |
| .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); |
| } |
| |
| @VisibleForTesting |
| public String getNodeHttpAddressFromRMWebString(ContainerLogsRequest request) |
| throws ClientHandlerException, UniformInterfaceException, JSONException { |
| if (request.getNodeId() == null || request.getNodeId().isEmpty()) { |
| return null; |
| } |
| JSONObject nodeInfo = YarnWebServiceUtils |
| .getNodeInfoFromRMWebService(getConf(), request.getNodeId()) |
| .getJSONObject("node"); |
| return nodeInfo.has("nodeHTTPAddress") ? |
| nodeInfo.getString("nodeHTTPAddress") : null; |
| } |
| |
| // Class to handle retry |
| static class ClientConnectionRetry { |
| |
| // maxRetries < 0 means keep trying |
| @Private |
| @VisibleForTesting |
| public int maxRetries; |
| |
| @Private |
| @VisibleForTesting |
| public long retryInterval; |
| |
| // Indicates if retries happened last time. Only tests should read it. |
| // In unit tests, retryOn() calls should _not_ be concurrent. |
| private boolean retried = false; |
| |
| @Private |
| @VisibleForTesting |
| boolean getRetired() { |
| return retried; |
| } |
| |
| // Constructor with default retry settings |
| public ClientConnectionRetry(int inputMaxRetries, |
| long inputRetryInterval) { |
| this.maxRetries = inputMaxRetries; |
| this.retryInterval = inputRetryInterval; |
| } |
| |
| public Object retryOn(ClientRetryOp op) |
| throws RuntimeException, IOException { |
| int leftRetries = maxRetries; |
| retried = false; |
| |
| // keep trying |
| while (true) { |
| try { |
| // try perform the op, if fail, keep retrying |
| return op.run(); |
| } catch (IOException | RuntimeException e) { |
| // break if there's no retries left |
| if (leftRetries == 0) { |
| break; |
| } |
| if (op.shouldRetryOn(e)) { |
| logException(e, leftRetries); |
| } else { |
| throw e; |
| } |
| } |
| if (leftRetries > 0) { |
| leftRetries--; |
| } |
| retried = true; |
| try { |
| // sleep for the given time interval |
| Thread.sleep(retryInterval); |
| } catch (InterruptedException ie) { |
| System.out.println("Client retry sleep interrupted! "); |
| } |
| } |
| throw new RuntimeException("Connection retries limit exceeded."); |
| }; |
| |
| private void logException(Exception e, int leftRetries) { |
| if (leftRetries > 0) { |
| System.out.println("Exception caught by ClientConnectionRetry," |
| + " will try " + leftRetries + " more time(s).\nMessage: " |
| + e.getMessage()); |
| } else { |
| // note that maxRetries may be -1 at the very beginning |
| System.out.println("ConnectionException caught by ClientConnectionRetry," |
| + " will keep retrying.\nMessage: " |
| + e.getMessage()); |
| } |
| } |
| } |
| |
| private class ClientJerseyRetryFilter extends ClientFilter { |
| @Override |
| public ClientResponse handle(final ClientRequest cr) |
| throws ClientHandlerException { |
| // Set up the retry operation |
| ClientRetryOp jerseyRetryOp = new ClientRetryOp() { |
| @Override |
| public Object run() { |
| // Try pass the request, if fail, keep retrying |
| return getNext().handle(cr); |
| } |
| |
| @Override |
| public boolean shouldRetryOn(Exception e) { |
| // Only retry on connection exceptions |
| return (e instanceof ClientHandlerException) |
| && (e.getCause() instanceof ConnectException || |
| e.getCause() instanceof SocketTimeoutException || |
| e.getCause() instanceof SocketException); |
| } |
| }; |
| try { |
| return (ClientResponse) connectionRetry.retryOn(jerseyRetryOp); |
| } catch (IOException e) { |
| throw new ClientHandlerException("Jersey retry failed!\nMessage: " |
| + e.getMessage()); |
| } |
| } |
| } |
| |
| // Abstract class for an operation that should be retried by client |
| private static abstract class ClientRetryOp { |
| // The operation that should be retried |
| public abstract Object run() throws IOException; |
| // The method to indicate if we should retry given the incoming exception |
| public abstract boolean shouldRetryOn(Exception e); |
| } |
| |
| private long getLogSizeLimitLeft() { |
| return this.logSizeLeft; |
| } |
| |
| private void decrLogSizeLimit(long used) { |
| this.logSizeLeft -= used; |
| } |
| |
| @Private |
| @VisibleForTesting |
| public ContainerLogsRequest getMatchedOptionForRunningApp( |
| ContainerLogsRequest container, boolean useRegex, |
| boolean ignoreSizeLimit) throws IOException { |
| String containerIdStr = container.getContainerId().toString(); |
| String nodeHttpAddress = container.getNodeHttpAddress(); |
| if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { |
| System.err.println("Can not get the logs for the container: " |
| + containerIdStr); |
| System.err.println("The node http address is required to get container " |
| + "logs for the Running application."); |
| return null; |
| } |
| |
| Set<String> matchedFiles = getMatchedContainerLogFiles(container, |
| useRegex, ignoreSizeLimit); |
| if (matchedFiles.isEmpty()) { |
| System.err.println("Can not find any log file matching the pattern: " |
| + container.getLogTypes() + " for the container: " + containerIdStr |
| + " within the application: " + container.getAppId()); |
| return null; |
| } |
| container.setLogTypes(matchedFiles); |
| return container; |
| } |
| |
| @Private |
| @VisibleForTesting |
| public Map<String, ContainerLogsRequest> getMatchedLogTypesForRunningApp( |
| List<ContainerLogsRequest> containerLogRequests, boolean useRegex, |
| boolean ignoreSizeLimit) { |
| Map<String, ContainerLogsRequest> containerMatchedLog = new HashMap<>(); |
| for (ContainerLogsRequest container : containerLogRequests) { |
| try { |
| ContainerLogsRequest request = getMatchedOptionForRunningApp( |
| container, useRegex, ignoreSizeLimit); |
| if (request == null) { |
| continue; |
| } |
| containerMatchedLog.put(container.getContainerId(), request); |
| } catch(IOException ex) { |
| System.err.println(ex); |
| continue; |
| } |
| } |
| return containerMatchedLog; |
| } |
| |
| private Map<String, ContainerLogsRequest> getMatchedLogTypesForFinishedApp( |
| List<ContainerLogsRequest> containerLogRequests, |
| LogCLIHelpers logCliHelper, boolean useRegex, |
| boolean ignoreSizeLimit) { |
| Map<String, ContainerLogsRequest> containerMatchedLog = new HashMap<>(); |
| for (ContainerLogsRequest container : containerLogRequests) { |
| try { |
| ContainerLogsRequest request = getMatchedLogOptions(container, |
| logCliHelper, useRegex, ignoreSizeLimit); |
| if (request == null) { |
| System.err.println("Can not find any log file matching the pattern: " |
| + container.getLogTypes() + " for the container: " |
| + container.getContainerId() + " within the application: " |
| + container.getAppId()); |
| continue; |
| } |
| containerMatchedLog.put(container.getContainerId(), request); |
| } catch (IOException ex) { |
| System.err.println(ex); |
| continue; |
| } |
| } |
| return containerMatchedLog; |
| } |
| } |
