| #!/usr/bin/env python3 |
| # |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| """ |
| Autogenerate documentation for all process endpoints spawned by a |
| Mesos master and agent. |
| """ |
| |
| import argparse |
| import atexit |
| import json |
| import os |
| import posixpath |
| import re |
| import shutil |
| import subprocess |
| import sys |
| import time |
| import urllib.request |
| import urllib.error |
| import urllib.parse |
| |
| |
| # The host ip and master and agent ports. |
| HOST_IP = "127.0.0.1" |
| MASTER_PORT = 5050 |
| AGENT_PORT = 5051 |
| |
| # The master and agent programs to launch. |
| # We considered making the parameters to these commands something that |
| # a user could specify on the command line, but ultimately chose to |
| # hard code them. Different parameters may cause different endpoints |
| # to become available, and we should modify this script to ensure that |
| # we cover all of them instead of leaving that up to the user. |
| MASTER_COMMAND = [ |
| 'mesos-master.sh', |
| '--ip=%s' % (HOST_IP), |
| '--registry=in_memory', |
| '--work_dir=/tmp/mesos' |
| ] |
| |
| # NOTE: The agent flags here ensure that this script can run inside docker. |
| AGENT_COMMAND = [ |
| 'mesos-agent.sh', |
| '--master=%s:%s' % (HOST_IP, MASTER_PORT), |
| '--work_dir=/tmp/mesos', |
| '--systemd_enable_support=false', |
| '--launcher=posix' |
| ] |
| |
| |
| # A header to add onto all generated markdown files. |
| MARKDOWN_HEADER = """--- |
| title: %s |
| layout: documentation |
| --- |
| <!--- This is an automatically generated file. DO NOT EDIT! ---> |
| """ |
| |
| # A template of the title to add onto all generated markdown files. |
| MARKDOWN_TITLE = "Apache Mesos - HTTP Endpoints%s" |
| |
| # A global timeout as well as a retry interval when hitting any http |
| # endpoints on the master or agent (in seconds). |
| RECEIVE_TIMEOUT = 600 |
| RETRY_INTERVAL = 0.10 |
| |
| |
| class Subprocess(): |
| """The process running using this script.""" |
| def __init__(self): |
| self.current = None |
| |
| def cleanup(self): |
| """Kill the process running once the script is done.""" |
| if self.current: |
| self.current.kill() |
| |
| |
| # A pointer to the top level directory of the mesos project. |
| GIT_TOP_DIR = subprocess.check_output( |
| ['git', |
| 'rev-parse', |
| '--show-cdup']).decode(sys.stdout.encoding).strip() |
| |
| with open(os.path.join(GIT_TOP_DIR, 'CHANGELOG'), 'r') as f: |
| if 'mesos' not in f.readline().lower(): |
| print(('You must run this command from within' |
| ' the Mesos source repository!'), file=sys.stderr) |
| sys.exit(1) |
| |
| |
| def parse_options(): |
| """Parses command line options and populates the dictionary.""" |
| |
| options = {} |
| |
| parser = argparse.ArgumentParser( |
| formatter_class=argparse.RawTextHelpFormatter, |
| description='Generate markdown files from all installed HTTP ' |
| 'endpoints on a Mesos master and agent process.') |
| |
| parser.add_argument( |
| '-c', '--command-path', |
| metavar='COMMAND_PATH', |
| default=os.path.join(GIT_TOP_DIR, "build/bin"), |
| help='Path to the Mesos master and agent commands.\n' |
| '(default: %(default)s)') |
| |
| parser.add_argument( |
| '-o', '--output-path', |
| metavar='OUTPUT_PATH', |
| default=os.path.join(GIT_TOP_DIR, "docs/endpoints"), |
| help='Path to the top level directory where all\n' |
| 'generated markdown files will be placed.\n' |
| '(default: %(default)s)') |
| |
| args = parser.parse_args() |
| |
| options['command_path'] = args.command_path |
| options['output_path'] = args.output_path |
| |
| return options |
| |
| |
| def get_url_until_success(url): |
| """Continuously tries to open a url until it succeeds or times out.""" |
| time_spent = 0 |
| while time_spent < RECEIVE_TIMEOUT: |
| try: |
| helps = urllib.request.urlopen(url) |
| break |
| except Exception: |
| time.sleep(RETRY_INTERVAL) |
| time_spent += RETRY_INTERVAL |
| |
| if time_spent >= RECEIVE_TIMEOUT: |
| print('Timeout attempting to hit url: %s' % (url), file=sys.stderr) |
| sys.exit(1) |
| |
| return helps.read() |
| |
| |
| def get_help(ip, port): |
| """ |
| Grabs the help strings for all endpoints at http://ip:port as a JSON object. |
| """ |
| url = 'http://%s:%d/help?format=json' % (ip, port) |
| return json.loads(get_url_until_success(url)) |
| |
| |
| def generalize_endpoint_id(p_id): |
| """Generalizes the id of the form e.g. process(1) to process(id).""" |
| return re.sub(r'\([0-9]+\)', '(id)', p_id) |
| |
| |
| def normalize_endpoint_id(p_id): |
| """Normalizes the id of the form e.g. process(id) to process.""" |
| return re.sub(r'\([0-9]+\)', '', p_id) |
| |
| |
| def get_endpoint_path(p_id, name): |
| """ |
| Generates the canonical endpoint path, given id and name. |
| |
| Examples: ('process', '/') -> '/process' |
| ('process(id)', '/') -> '/process(id)' |
| ('process', '/endpoint') -> '/process/endpoint' |
| """ |
| # Tokenize the endpoint by '/' (filtering |
| # out any empty strings between '/'s) |
| path_parts = [_f for _f in name.split('/') if _f] |
| |
| # Conditionally prepend the 'id' to the list of path parts. |
| # Following the notion of a 'delegate' in Mesos, we want our |
| # preferred endpoint paths for the delegate process to be |
| # '/endpoint' instead of '/process/endpoint'. Since this script only |
| # starts 1 master and 1 agent, our only delegate processes are |
| # "master" and "slave(id)". If the id matches one of these, we don't |
| # prepend it, otherwise we do. |
| p_id = generalize_endpoint_id(p_id) |
| delegates = ["master", "slave(id)"] |
| if p_id not in delegates: |
| path_parts = [p_id] + path_parts |
| |
| return posixpath.join('/', *path_parts) |
| |
| |
| def get_relative_md_path(p_id, name): |
| """ |
| Generates the relative path of the generated .md file from id and name. |
| |
| This path is relative to the options['output_path'] directory. |
| |
| Examples: master/health.md |
| master/maintenance/schedule.md |
| registrar/registry.md |
| version.md |
| """ |
| new_id = normalize_endpoint_id(p_id) |
| # Strip the leading slash |
| new_name = name[1:] |
| |
| if new_name: |
| return os.path.join(new_id, new_name + '.md') |
| return os.path.join(new_id + '.md') |
| |
| |
| def write_markdown(path, output, title): |
| """Writes 'output' to the file at 'path'.""" |
| print('generating: %s' % (path)) |
| |
| dirname = os.path.dirname(path) |
| if not os.path.exists(dirname): |
| os.makedirs(dirname) |
| |
| outfile = open(path, 'w+') |
| |
| # Add our header and remove all '\n's at the end of the output if |
| # there are any. |
| output = (MARKDOWN_HEADER % title) + '\n' + output.rstrip() |
| |
| outfile.write(output) |
| outfile.close() |
| |
| |
| def dump_index_markdown(master_help, agent_help, options): |
| """ |
| Dumps an index for linking to the master and agent help files. |
| |
| This file is dumped into a directory rooted at |
| options['output_path']. |
| """ |
| |
| # The output template for the HTTP endpoints index. |
| # We use a helper function below to insert text into the '%s' format |
| # strings contained in the "Master Endpoints" and "Agent Endpoints" |
| # sections of this template. |
| output = """# HTTP Endpoints # |
| |
| Below is a list of HTTP endpoints available for a given Mesos process. |
| |
| Depending on your configuration, some subset of these endpoints will be |
| available on your Mesos master or agent. Additionally, a `/help` |
| endpoint will be available that displays help similar to what you see |
| below. |
| |
| ** NOTE: ** If you are using Mesos 1.1 or later, we recommend using the |
| new [v1 Operator HTTP API](../operator-http-api.md) instead of the |
| unversioned REST endpoints listed below. These endpoints will be |
| deprecated in the future. |
| |
| |
| ** NOTE: ** The documentation for these endpoints is auto-generated from |
| the Mesos source code. See `support/generate-endpoint-help.py`. |
| |
| ## Master Endpoints ## |
| |
| Below are the endpoints that are available on a Mesos master. These |
| endpoints are reachable at the address `http://ip:port/endpoint`. |
| |
| For example, `http://master.com:5050/files/browse`. |
| |
| %s |
| |
| ## Agent Endpoints ## |
| |
| Below are the endpoints that are available on a Mesos agent. These |
| endpoints are reachable at the address `http://ip:port/endpoint`. |
| |
| For example, `http://agent.com:5051/files/browse`. |
| |
| %s |
| """ |
| |
| def generate_links(master_or_agent_help): |
| """ |
| Iterates over the input JSON and creates a list of links to |
| to the markdown files generated by this script. These links |
| are grouped per process, with the process's name serving as a |
| header for each group. All links are relative to |
| options['output_path']. |
| |
| For example: |
| ### profiler ### |
| * [/profiler/start] (profiler/start.md) |
| * [/profiler/stop] (profiler/stop.md) |
| |
| ### version ### |
| * [/version] (version.md) |
| """ |
| output = "" |
| for process in master_or_agent_help['processes']: |
| p_id = process['id'] |
| output += '### %s ###\n' % (generalize_endpoint_id(p_id)) |
| for endpoint in process['endpoints']: |
| name = endpoint['name'] |
| output += '* [%s](%s)\n' % (get_endpoint_path(p_id, name), |
| get_relative_md_path(p_id, name)) |
| output += '\n' |
| |
| # Remove any trailing newlines |
| return output.rstrip() |
| |
| output = output % (generate_links(master_help), |
| generate_links(agent_help)) |
| |
| path = os.path.join(options['output_path'], 'index.md') |
| write_markdown(path, output, MARKDOWN_TITLE % "") |
| |
| |
| def dump_markdown(master_or_agent_help, options): |
| """ |
| Dumps JSON encoded help strings into markdown files. |
| |
| These files are dumped into a directory rooted at |
| options['output_path']. |
| """ |
| for process in master_or_agent_help['processes']: |
| p_id = process['id'] |
| for endpoint in process['endpoints']: |
| name = endpoint['name'] |
| text = endpoint['text'] |
| title = get_endpoint_path(p_id, name) |
| |
| relative_path = get_relative_md_path(p_id, name) |
| path = os.path.join(options['output_path'], relative_path) |
| write_markdown(path, text, MARKDOWN_TITLE % (" - " + title)) |
| |
| |
| def start_master(options): |
| """ |
| Starts the Mesos master using the specified command. |
| |
| This method returns the Popen object used to start it so it can |
| be killed later on. |
| """ |
| cmd = os.path.join('.', options['command_path'], MASTER_COMMAND[0]) |
| master = subprocess.Popen([cmd] + MASTER_COMMAND[1:]) |
| |
| # Wait for the master to become responsive |
| get_url_until_success("http://%s:%d/health" % (HOST_IP, MASTER_PORT)) |
| return master |
| |
| |
| def start_agent(options): |
| """ |
| Starts the Mesos agent using the specified command. |
| |
| This method returns the Popen object used to start it so it can |
| be killed later on. |
| """ |
| cmd = os.path.join('.', options['command_path'], AGENT_COMMAND[0]) |
| agent = subprocess.Popen([cmd] + AGENT_COMMAND[1:]) |
| |
| # Wait for the agent to become responsive. |
| get_url_until_success('http://%s:%d/health' % (HOST_IP, AGENT_PORT)) |
| return agent |
| |
| |
| def main(): |
| """ |
| Called when the Python script is used, we do not directly write code |
| after 'if __name__ == "__main__"' as we cannot set variables in that case. |
| """ |
| # A dictionary of the command line options passed in. |
| options = parse_options() |
| |
| # A pointer to the current subprocess for the master or agent. |
| # This is useful for tracking the master or agent subprocesses so |
| # that we can kill them if the script exits prematurely. |
| subproc = Subprocess() |
| atexit.register(subproc.cleanup) |
| subproc.current = start_master(options) |
| master_help = get_help(HOST_IP, MASTER_PORT) |
| subproc.current.kill() |
| |
| subproc.current = start_agent(options) |
| agent_help = get_help(HOST_IP, AGENT_PORT) |
| subproc.current.kill() |
| |
| shutil.rmtree(options['output_path'], ignore_errors=True) |
| os.makedirs(options['output_path']) |
| |
| dump_index_markdown(master_help, agent_help, options) |
| dump_markdown(master_help, options) |
| dump_markdown(agent_help, options) |
| |
| if __name__ == '__main__': |
| main() |