blob: 9ba2bce2289b18a3a797f010bce64ba5d65a57af [file] [log] [blame]
#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.
Autogenerate documentation for all process endpoints spawned by a
Mesos master and agent.
import argparse
import atexit
import json
import os
import posixpath
import re
import shutil
import subprocess
import sys
import time
import urllib.request
import urllib.error
import urllib.parse
# The host ip and master and agent ports.
HOST_IP = ""
# The master and agent programs to launch.
# We considered making the parameters to these commands something that
# a user could specify on the command line, but ultimately chose to
# hard code them. Different parameters may cause different endpoints
# to become available, and we should modify this script to ensure that
# we cover all of them instead of leaving that up to the user.
'--ip=%s' % (HOST_IP),
# NOTE: The agent flags here ensure that this script can run inside docker.
'--master=%s:%s' % (HOST_IP, MASTER_PORT),
# A header to add onto all generated markdown files.
title: %s
layout: documentation
<!--- This is an automatically generated file. DO NOT EDIT! --->
# A template of the title to add onto all generated markdown files.
MARKDOWN_TITLE = "Apache Mesos - HTTP Endpoints%s"
# A global timeout as well as a retry interval when hitting any http
# endpoints on the master or agent (in seconds).
class Subprocess():
"""The process running using this script."""
def __init__(self):
self.current = None
def cleanup(self):
"""Kill the process running once the script is done."""
if self.current:
# A pointer to the top level directory of the mesos project.
GIT_TOP_DIR = subprocess.check_output(
with open(os.path.join(GIT_TOP_DIR, 'CHANGELOG'), 'r') as f:
if 'mesos' not in f.readline().lower():
print(('You must run this command from within'
' the Mesos source repository!'), file=sys.stderr)
def parse_options():
"""Parses command line options and populates the dictionary."""
options = {}
parser = argparse.ArgumentParser(
description='Generate markdown files from all installed HTTP '
'endpoints on a Mesos master and agent process.')
'-c', '--command-path',
default=os.path.join(GIT_TOP_DIR, "build/bin"),
help='Path to the Mesos master and agent commands.\n'
'(default: %(default)s)')
'-o', '--output-path',
default=os.path.join(GIT_TOP_DIR, "docs/endpoints"),
help='Path to the top level directory where all\n'
'generated markdown files will be placed.\n'
'(default: %(default)s)')
args = parser.parse_args()
options['command_path'] = args.command_path
options['output_path'] = args.output_path
return options
def get_url_until_success(url):
"""Continuously tries to open a url until it succeeds or times out."""
time_spent = 0
while time_spent < RECEIVE_TIMEOUT:
helps = urllib.request.urlopen(url)
except Exception:
time_spent += RETRY_INTERVAL
if time_spent >= RECEIVE_TIMEOUT:
print('Timeout attempting to hit url: %s' % (url), file=sys.stderr)
def get_help(ip, port):
Grabs the help strings for all endpoints at http://ip:port as a JSON object.
url = 'http://%s:%d/help?format=json' % (ip, port)
return json.loads(get_url_until_success(url))
def generalize_endpoint_id(p_id):
"""Generalizes the id of the form e.g. process(1) to process(id)."""
return re.sub(r'\([0-9]+\)', '(id)', p_id)
def normalize_endpoint_id(p_id):
"""Normalizes the id of the form e.g. process(id) to process."""
return re.sub(r'\([0-9]+\)', '', p_id)
def get_endpoint_path(p_id, name):
Generates the canonical endpoint path, given id and name.
Examples: ('process', '/') -> '/process'
('process(id)', '/') -> '/process(id)'
('process', '/endpoint') -> '/process/endpoint'
# Tokenize the endpoint by '/' (filtering
# out any empty strings between '/'s)
path_parts = [_f for _f in name.split('/') if _f]
# Conditionally prepend the 'id' to the list of path parts.
# Following the notion of a 'delegate' in Mesos, we want our
# preferred endpoint paths for the delegate process to be
# '/endpoint' instead of '/process/endpoint'. Since this script only
# starts 1 master and 1 agent, our only delegate processes are
# "master" and "slave(id)". If the id matches one of these, we don't
# prepend it, otherwise we do.
p_id = generalize_endpoint_id(p_id)
delegates = ["master", "slave(id)"]
if p_id not in delegates:
path_parts = [p_id] + path_parts
return posixpath.join('/', *path_parts)
def get_relative_md_path(p_id, name):
Generates the relative path of the generated .md file from id and name.
This path is relative to the options['output_path'] directory.
Examples: master/
new_id = normalize_endpoint_id(p_id)
# Strip the leading slash
new_name = name[1:]
if new_name:
return os.path.join(new_id, new_name + '.md')
return os.path.join(new_id + '.md')
def write_markdown(path, output, title):
"""Writes 'output' to the file at 'path'."""
print('generating: %s' % (path))
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
outfile = open(path, 'w+')
# Add our header and remove all '\n's at the end of the output if
# there are any.
output = (MARKDOWN_HEADER % title) + '\n' + output.rstrip()
def dump_index_markdown(master_help, agent_help, options):
Dumps an index for linking to the master and agent help files.
This file is dumped into a directory rooted at
# The output template for the HTTP endpoints index.
# We use a helper function below to insert text into the '%s' format
# strings contained in the "Master Endpoints" and "Agent Endpoints"
# sections of this template.
output = """# HTTP Endpoints #
Below is a list of HTTP endpoints available for a given Mesos process.
Depending on your configuration, some subset of these endpoints will be
available on your Mesos master or agent. Additionally, a `/help`
endpoint will be available that displays help similar to what you see
** NOTE: ** If you are using Mesos 1.1 or later, we recommend using the
new [v1 Operator HTTP API](../ instead of the
unversioned REST endpoints listed below. These endpoints will be
deprecated in the future.
** NOTE: ** The documentation for these endpoints is auto-generated from
the Mesos source code. See `support/`.
## Master Endpoints ##
Below are the endpoints that are available on a Mesos master. These
endpoints are reachable at the address `http://ip:port/endpoint`.
For example, ``.
## Agent Endpoints ##
Below are the endpoints that are available on a Mesos agent. These
endpoints are reachable at the address `http://ip:port/endpoint`.
For example, ``.
def generate_links(master_or_agent_help):
Iterates over the input JSON and creates a list of links to
to the markdown files generated by this script. These links
are grouped per process, with the process's name serving as a
header for each group. All links are relative to
For example:
### profiler ###
* [/profiler/start] (profiler/
* [/profiler/stop] (profiler/
### version ###
* [/version] (
output = ""
for process in master_or_agent_help['processes']:
p_id = process['id']
output += '### %s ###\n' % (generalize_endpoint_id(p_id))
for endpoint in process['endpoints']:
name = endpoint['name']
output += '* [%s](%s)\n' % (get_endpoint_path(p_id, name),
get_relative_md_path(p_id, name))
output += '\n'
# Remove any trailing newlines
return output.rstrip()
output = output % (generate_links(master_help),
path = os.path.join(options['output_path'], '')
write_markdown(path, output, MARKDOWN_TITLE % "")
def dump_markdown(master_or_agent_help, options):
Dumps JSON encoded help strings into markdown files.
These files are dumped into a directory rooted at
for process in master_or_agent_help['processes']:
p_id = process['id']
for endpoint in process['endpoints']:
name = endpoint['name']
text = endpoint['text']
title = get_endpoint_path(p_id, name)
relative_path = get_relative_md_path(p_id, name)
path = os.path.join(options['output_path'], relative_path)
write_markdown(path, text, MARKDOWN_TITLE % (" - " + title))
def start_master(options):
Starts the Mesos master using the specified command.
This method returns the Popen object used to start it so it can
be killed later on.
cmd = os.path.join('.', options['command_path'], MASTER_COMMAND[0])
master = subprocess.Popen([cmd] + MASTER_COMMAND[1:])
# Wait for the master to become responsive
get_url_until_success("http://%s:%d/health" % (HOST_IP, MASTER_PORT))
return master
def start_agent(options):
Starts the Mesos agent using the specified command.
This method returns the Popen object used to start it so it can
be killed later on.
cmd = os.path.join('.', options['command_path'], AGENT_COMMAND[0])
agent = subprocess.Popen([cmd] + AGENT_COMMAND[1:])
# Wait for the agent to become responsive.
get_url_until_success('http://%s:%d/health' % (HOST_IP, AGENT_PORT))
return agent
def main():
Called when the Python script is used, we do not directly write code
after 'if __name__ == "__main__"' as we cannot set variables in that case.
# A dictionary of the command line options passed in.
options = parse_options()
# A pointer to the current subprocess for the master or agent.
# This is useful for tracking the master or agent subprocesses so
# that we can kill them if the script exits prematurely.
subproc = Subprocess()
subproc.current = start_master(options)
master_help = get_help(HOST_IP, MASTER_PORT)
subproc.current = start_agent(options)
agent_help = get_help(HOST_IP, AGENT_PORT)
shutil.rmtree(options['output_path'], ignore_errors=True)
dump_index_markdown(master_help, agent_help, options)
dump_markdown(master_help, options)
dump_markdown(agent_help, options)
if __name__ == '__main__':