java/kudu-csd/generate_mdl.py

#!/usr/bin/env python
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Script to generate a CM-compatible MDL file from the metrics
# metadata dumped by our daemon processes.
#
# Requires that the daemon processes have already been built and available
# in the build/latest/bin directory.
#
# Outputs the MDL file on stdout by default or to a file specified in the first
# argument.

import collections
try:
  import simplejson as json
except:
  import json
import os
import subprocess
import sys

BINARIES=["kudu-master", "kudu-tserver"]

RELATIVE_BUILD_DIR="../../build/latest/bin"

def find_binary(bin_name):
  dirname, _ = os.path.split(os.path.abspath(__file__))
  build_dir = os.path.join(dirname, RELATIVE_BUILD_DIR)
  path = os.path.join(build_dir, bin_name)
  if os.path.exists(path):
    return path
  raise Exception("Cannot find %s in build dir %s" % (bin_name, build_dir))

def load_all_metrics():
  """
  For each binary, dump and parse its metrics schema by running it with
  the --dump_metrics_json flag.
  """
  all_metrics = []
  for binary in BINARIES:
    binary = find_binary(binary)
    p = subprocess.Popen([binary, "--dump_metrics_json"],
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
    stdout, stderr = p.communicate()
    rc = p.returncode

    if rc != 0:
      print >>sys.stderr, "error: %s exited with return code %d:\n" % (binary, rc)
      print >>sys.stderr, stderr
      sys.exit(1)

    metrics_dump = json.loads(stdout)
    all_metrics.extend(m for m in metrics_dump['metrics'])
  return all_metrics

def append_sentence(a, b):
  if not a.endswith("."):
    a += "."
  return a + " " + b

def metric_to_mdl_entries(m):
  if m['type'] == 'histogram':
    return [
      dict(
        context=(m['name'] + "::total_count"),
        name=('kudu_' + m['name'].lower()) + '_count',
        counter=True,
        numeratorUnit='samples',
        description=append_sentence(m['description'],
                                    "This is the total number of recorded samples."),
        label=m['label'] + ": Samples"),
      dict(
        context=(m['name'] + "::total_sum"),
        name=('kudu_' + m['name'].lower()) + '_sum',
        counter=True,
        numeratorUnit=m['unit'],
        description=append_sentence(m['description'],
                                    "This is the total sum of recorded samples."),
        label=m['label'] + ": Total")
      ]

  return [dict(
    context=(m['name'] + "::value"),
    name=('kudu_' + m['name'].lower()),
    counter=(m['type'] == 'counter'),
    numeratorUnit=m['unit'],
    description=m['description'],
    label=m['label'])]

def metrics_to_mdl(metrics):
  """
  For each metric returned by the daemon, convert it to the MDL-compatible dictionary.
  Returns a map of entity_type_name -> [metric dicts].
  """
  seen = set()

  by_entity = collections.defaultdict(lambda: [])
  for m in metrics:
    # Don't process any metric more than once. Some metrics show up
    # in both daemons.
    key = (m['entity_type'], m['name'])
    if key in seen:
      continue
    seen.add(key)

    # Convert to the format that CM expects.
    by_entity[m['entity_type']].extend(metric_to_mdl_entries(m))
  return by_entity

def main():
  all_metrics = load_all_metrics()
  metrics_by_entity = metrics_to_mdl(all_metrics)
  server_metrics = metrics_by_entity['server']
  tablet_metrics = metrics_by_entity['tablet']

  output = dict(
    name="KUDU",
    version="0.6.0",
    metricDefinitions=[],
    nameForCrossEntityAggregateMetrics="kudus",
    roles=[
      dict(name="KUDU_TSERVER",
           nameForCrossEntityAggregateMetrics="kudu_tservers",
           metricDefinitions=server_metrics),
      dict(name="KUDU_MASTER",
           nameForCrossEntityAggregateMetrics="kudu_masters",
           metricDefinitions=server_metrics)
      ],
    metricEntityAttributeDefinitions=[
      dict(name="kuduTableId",
           label="Table ID",
           description="UUID for Kudu Table.",
           valueCaseSensitive=False),
      dict(name="kuduTableName",
           label="Table Name",
           description="Name for Kudu Table.",
           valueCaseSensitive=True),
      dict(name="kuduTableState",
           label="Table State",
           description="State for Kudu Table.",
           valueCaseSensitive=False),
      dict(name="kuduTabletId",
           label="Tablet ID",
           description="UUID for Kudu Tablet.",
           valueCaseSensitive=False),
      dict(name="kuduTabletState",
           label="Tablet State",
           description="State for Kudu Tablet.",
           valueCaseSensitive=False)
      # TODO: add the role's persistent UUID after discussing with
      # Chris on how to inject it into their CM entity.
      ],
    metricEntityTypeDefinitions=[
      dict(name="KUDU_TABLE",
           nameForCrossEntityAggregateMetrics="kudu_tables",
           immutableAttributeNames=["serviceName", "kuduTableId"],
           mutableAttributeNames=["kuduTableName", "kuduTableState"],
           entityNameFormat=["serviceName", "kuduTableId"],
           description="A Kudu table.",
           label="Kudu Table",
           labelPlural="Kudu Tables",
           entityLabelFormat="$kuduTableName ($serviceDisplayName)",
           parentMetricEntityTypeNames=["KUDU"],
           metricDefinitions=[]),
      dict(name="KUDU_TABLET",
           nameForCrossEntityAggregateMetrics="kudu_tablets",
           immutableAttributeNames=["serviceName", "kuduTableId", "kuduTabletId"],
           mutableAttributeNames=["kuduTabletState"],
           entityNameFormat=["serviceName", "kuduTabletId"],
           description="A Kudu tablet.",
           label="Kudu Tablet",
           labelPlural="Kudu Tablets",
           entityLabelFormat="$kuduTabletId ($kuduTableName) ($serviceDisplayName)",
           parentMetricEntityTypeNames=["KUDU_TABLE"],
           metricDefinitions=[]),
      dict(name="KUDU_REPLICA",
           nameForCrossEntityAggregateMetrics="kudu_replicas",
           immutableAttributeNames=["kuduTabletId", "serviceName", "roleName"],
           entityNameFormat=["roleName","kuduTabletId"],
           description="A Kudu replica.",
           label="Kudu Replica",
           labelPlural="Kudu Replicas",
           entityLabelFormat="$kuduTabletId ($kuduTableName) ($hostname)",
           parentMetricEntityTypeNames=["KUDU_TABLET","KUDU-KUDU_TSERVER"],
           metricDefinitions=tablet_metrics),
      ])

  
  f = sys.stdout
  if len(sys.argv) > 1:
    f = open(sys.argv[1], 'w')
  f.write(json.dumps(output, indent=4))

if __name__ == "__main__":
  main()