hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemInstrumentation.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.fs.azure.metrics;

import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;

import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsInfo;
import org.apache.hadoop.metrics2.MetricsSource;
import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.MetricsRegistry;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;

/**
 * A metrics source for the WASB file system to track all the metrics we care
 * about for getting a clear picture of the performance/reliability/interaction
 * of the Hadoop cluster with Azure Storage.
 */
@Metrics(about="Metrics for WASB", context="azureFileSystem")
@InterfaceAudience.Public
@InterfaceStability.Evolving
public final class AzureFileSystemInstrumentation implements MetricsSource {

  public static final String METRIC_TAG_FILESYSTEM_ID = "wasbFileSystemId";
  public static final String METRIC_TAG_ACCOUNT_NAME = "accountName";
  public static final String METRIC_TAG_CONTAINTER_NAME = "containerName";

  public static final String WASB_WEB_RESPONSES = "wasb_web_responses";
  public static final String WASB_BYTES_WRITTEN =
      "wasb_bytes_written_last_second";
  public static final String WASB_BYTES_READ =
      "wasb_bytes_read_last_second";
  public static final String WASB_RAW_BYTES_UPLOADED =
      "wasb_raw_bytes_uploaded";
  public static final String WASB_RAW_BYTES_DOWNLOADED =
      "wasb_raw_bytes_downloaded";
  public static final String WASB_FILES_CREATED = "wasb_files_created";
  public static final String WASB_FILES_DELETED = "wasb_files_deleted";
  public static final String WASB_DIRECTORIES_CREATED = "wasb_directories_created";
  public static final String WASB_DIRECTORIES_DELETED = "wasb_directories_deleted";
  public static final String WASB_UPLOAD_RATE =
      "wasb_maximum_upload_bytes_per_second";
  public static final String WASB_DOWNLOAD_RATE =
      "wasb_maximum_download_bytes_per_second";
  public static final String WASB_UPLOAD_LATENCY =
      "wasb_average_block_upload_latency_ms";
  public static final String WASB_DOWNLOAD_LATENCY =
      "wasb_average_block_download_latency_ms";
  public static final String WASB_CLIENT_ERRORS = "wasb_client_errors";
  public static final String WASB_SERVER_ERRORS = "wasb_server_errors";

  /**
   * Config key for how big the rolling window size for latency metrics should
   * be (in seconds).
   */
  private static final String KEY_ROLLING_WINDOW_SIZE = "fs.azure.metrics.rolling.window.size";

  private final MetricsRegistry registry =
      new MetricsRegistry("azureFileSystem")
      .setContext("azureFileSystem");
  private final MutableCounterLong numberOfWebResponses =
      registry.newCounter(
          WASB_WEB_RESPONSES,
          "Total number of web responses obtained from Azure Storage",
          0L);
  private AtomicLong inMemoryNumberOfWebResponses = new AtomicLong(0);
  private final MutableCounterLong numberOfFilesCreated =
      registry.newCounter(
          WASB_FILES_CREATED,
          "Total number of files created through the WASB file system.",
          0L);
  private final MutableCounterLong numberOfFilesDeleted =
      registry.newCounter(
          WASB_FILES_DELETED,
          "Total number of files deleted through the WASB file system.",
          0L);
  private final MutableCounterLong numberOfDirectoriesCreated =
      registry.newCounter(
          WASB_DIRECTORIES_CREATED,
          "Total number of directories created through the WASB file system.",
          0L);
  private final MutableCounterLong numberOfDirectoriesDeleted =
      registry.newCounter(
          WASB_DIRECTORIES_DELETED,
          "Total number of directories deleted through the WASB file system.",
          0L);
  private final MutableGaugeLong bytesWrittenInLastSecond =
      registry.newGauge(
          WASB_BYTES_WRITTEN,
          "Total number of bytes written to Azure Storage during the last second.",
          0L);
  private final MutableGaugeLong bytesReadInLastSecond =
      registry.newGauge(
          WASB_BYTES_READ,
          "Total number of bytes read from Azure Storage during the last second.",
          0L);
  private final MutableGaugeLong maximumUploadBytesPerSecond =
      registry.newGauge(
          WASB_UPLOAD_RATE,
          "The maximum upload rate encountered to Azure Storage in bytes/second.",
          0L);
  private final MutableGaugeLong maximumDownloadBytesPerSecond =
      registry.newGauge(
          WASB_DOWNLOAD_RATE,
          "The maximum download rate encountered to Azure Storage in bytes/second.",
          0L);
  private final MutableCounterLong rawBytesUploaded =
      registry.newCounter(
          WASB_RAW_BYTES_UPLOADED,
          "Total number of raw bytes (including overhead) uploaded to Azure" 
          + " Storage.",
          0L);
  private final MutableCounterLong rawBytesDownloaded =
      registry.newCounter(
          WASB_RAW_BYTES_DOWNLOADED,
          "Total number of raw bytes (including overhead) downloaded from Azure" 
          + " Storage.",
          0L);
  private final MutableCounterLong clientErrors =
      registry.newCounter(
          WASB_CLIENT_ERRORS,
          "Total number of client-side errors by WASB (excluding 404).",
          0L);
  private final MutableCounterLong serverErrors =
      registry.newCounter(
          WASB_SERVER_ERRORS,
          "Total number of server-caused errors by WASB.",
          0L);
  private final MutableGaugeLong averageBlockUploadLatencyMs;
  private final MutableGaugeLong averageBlockDownloadLatencyMs;
  private long currentMaximumUploadBytesPerSecond;
  private long currentMaximumDownloadBytesPerSecond;
  private static final int DEFAULT_LATENCY_ROLLING_AVERAGE_WINDOW =
      5; // seconds
  private final RollingWindowAverage currentBlockUploadLatency;
  private final RollingWindowAverage currentBlockDownloadLatency;
  private UUID fileSystemInstanceId;

  public AzureFileSystemInstrumentation(Configuration conf) {
    fileSystemInstanceId = UUID.randomUUID();
    registry.tag("wasbFileSystemId",
        "A unique identifier for the file ",
        fileSystemInstanceId.toString());
    final int rollingWindowSizeInSeconds =
        conf.getInt(KEY_ROLLING_WINDOW_SIZE,
            DEFAULT_LATENCY_ROLLING_AVERAGE_WINDOW);
    averageBlockUploadLatencyMs =
        registry.newGauge(
            WASB_UPLOAD_LATENCY,
            String.format("The average latency in milliseconds of uploading a single block" 
            + ". The average latency is calculated over a %d-second rolling" 
            + " window.", rollingWindowSizeInSeconds),
            0L);
    averageBlockDownloadLatencyMs =
        registry.newGauge(
            WASB_DOWNLOAD_LATENCY,
            String.format("The average latency in milliseconds of downloading a single block" 
            + ". The average latency is calculated over a %d-second rolling" 
            + " window.", rollingWindowSizeInSeconds),
            0L);
    currentBlockUploadLatency =
        new RollingWindowAverage(rollingWindowSizeInSeconds * 1000);
    currentBlockDownloadLatency =
        new RollingWindowAverage(rollingWindowSizeInSeconds * 1000);
  }

  /**
   * The unique identifier for this file system in the metrics.
   * @return The unique identifier.
   */
  public UUID getFileSystemInstanceId() {
    return fileSystemInstanceId;
  }
  
  /**
   * Get the metrics registry information.
   * @return The metrics registry information.
   */
  public MetricsInfo getMetricsRegistryInfo() {
    return registry.info();
  }

  /**
   * Sets the account name to tag all the metrics with.
   * @param accountName The account name.
   */
  public void setAccountName(String accountName) {
    registry.tag("accountName",
        "Name of the Azure Storage account that these metrics are going against",
        accountName);
  }

  /**
   * Sets the container name to tag all the metrics with.
   * @param containerName The container name.
   */
  public void setContainerName(String containerName) {
    registry.tag("containerName",
        "Name of the Azure Storage container that these metrics are going against",
        containerName);
  }

  /**
   * Indicate that we just got a web response from Azure Storage. This should
   * be called for every web request/response we do (to get accurate metrics
   * of how we're hitting the storage service).
   */
  public void webResponse() {
    numberOfWebResponses.incr();
    inMemoryNumberOfWebResponses.incrementAndGet();
  }

  /**
   * Gets the current number of web responses obtained from Azure Storage.
   * @return The number of web responses.
   */
  public long getCurrentWebResponses() {
    return inMemoryNumberOfWebResponses.get();
  }

  /**
   * Indicate that we just created a file through WASB.
   */
  public void fileCreated() {
    numberOfFilesCreated.incr();
  }

  /**
   * Indicate that we just deleted a file through WASB.
   */
  public void fileDeleted() {
    numberOfFilesDeleted.incr();
  }

  /**
   * Indicate that we just created a directory through WASB.
   */
  public void directoryCreated() {
    numberOfDirectoriesCreated.incr();
  }

  /**
   * Indicate that we just deleted a directory through WASB.
   */
  public void directoryDeleted() {
    numberOfDirectoriesDeleted.incr();
  }

  /**
   * Sets the current gauge value for how many bytes were written in the last
   *  second.
   * @param currentBytesWritten The number of bytes.
   */
  public void updateBytesWrittenInLastSecond(long currentBytesWritten) {
    bytesWrittenInLastSecond.set(currentBytesWritten);
  }

  /**
   * Sets the current gauge value for how many bytes were read in the last
   *  second.
   * @param currentBytesRead The number of bytes.
   */
  public void updateBytesReadInLastSecond(long currentBytesRead) {
    bytesReadInLastSecond.set(currentBytesRead);
  }

  /**
   * Record the current bytes-per-second upload rate seen.
   * @param bytesPerSecond The bytes per second.
   */
  public synchronized void currentUploadBytesPerSecond(long bytesPerSecond) {
    if (bytesPerSecond > currentMaximumUploadBytesPerSecond) {
      currentMaximumUploadBytesPerSecond = bytesPerSecond;
      maximumUploadBytesPerSecond.set(bytesPerSecond);
    }
  }

  /**
   * Record the current bytes-per-second download rate seen.
   * @param bytesPerSecond The bytes per second.
   */
  public synchronized void currentDownloadBytesPerSecond(long bytesPerSecond) {
    if (bytesPerSecond > currentMaximumDownloadBytesPerSecond) {
      currentMaximumDownloadBytesPerSecond = bytesPerSecond;
      maximumDownloadBytesPerSecond.set(bytesPerSecond);
    }
  }

  /**
   * Indicate that we just uploaded some data to Azure storage.
   * @param numberOfBytes The raw number of bytes uploaded (including overhead).
   */
  public void rawBytesUploaded(long numberOfBytes) {
    rawBytesUploaded.incr(numberOfBytes);
  }

  /**
   * Indicate that we just downloaded some data to Azure storage.
   * @param numberOfBytes The raw number of bytes downloaded (including overhead).
   */
  public void rawBytesDownloaded(long numberOfBytes) {
    rawBytesDownloaded.incr(numberOfBytes);
  }

  /**
   * Indicate that we just uploaded a block and record its latency.
   * @param latency The latency in milliseconds.
   */
  public void blockUploaded(long latency) {
    currentBlockUploadLatency.addPoint(latency);
  }

  /**
   * Indicate that we just downloaded a block and record its latency.
   * @param latency The latency in milliseconds.
   */
  public void blockDownloaded(long latency) {
    currentBlockDownloadLatency.addPoint(latency);
  }

  /**
   * Indicate that we just encountered a client-side error.
   */
  public void clientErrorEncountered() {
    clientErrors.incr();
  }

  /**
   * Indicate that we just encountered a server-caused error.
   */
  public void serverErrorEncountered() {
    serverErrors.incr();
  }

  /**
   * Get the current rolling average of the upload latency.
   * @return rolling average of upload latency in milliseconds.
   */
  public long getBlockUploadLatency() {
    return currentBlockUploadLatency.getCurrentAverage();
  }

  /**
   * Get the current rolling average of the download latency.
   * @return rolling average of download latency in milliseconds.
   */
  public long getBlockDownloadLatency() {
    return currentBlockDownloadLatency.getCurrentAverage();
  }

  /**
   * Get the current maximum upload bandwidth.
   * @return maximum upload bandwidth in bytes per second.
   */
  public long getCurrentMaximumUploadBandwidth() {
    return currentMaximumUploadBytesPerSecond;
  }

  /**
   * Get the current maximum download bandwidth.
   * @return maximum download bandwidth in bytes per second.
   */
  public long getCurrentMaximumDownloadBandwidth() {
    return currentMaximumDownloadBytesPerSecond;
  }

  @Override
  public void getMetrics(MetricsCollector builder, boolean all) {
    averageBlockDownloadLatencyMs.set(
        currentBlockDownloadLatency.getCurrentAverage());
    averageBlockUploadLatencyMs.set(
        currentBlockUploadLatency.getCurrentAverage());
    registry.snapshot(builder.addRecord(registry.info().name()), true);
  }
}