tools/gen-config - couchdb-infra-cm - Git at Google

 #!/usr/bin/env python3

 import argparse as ap
 import configparser as cp
 import json
 import os
 import re
 import textwrap

 import requests
 import yaml


 # Environments read from config file
 # {"cloudant": {"api_key": "abc123", ...}
 #
 ENV = {}

 # Hard-coded VMs managed by someone else without access to cloud.ibm.com
 EXTRA = {}

 IBM_CLOUD_URL = "https://us-south.iaas.cloud.ibm.com/v1"
 IAM_URL = "https://iam.cloud.ibm.com/identity/token"

 IBM_CLOUD_GENERATION = "2"
 IBM_CLOUD_VERSION = "2019-08-09"

 # Authed API request sessions, one per environment
 # {"cloudant": <Session>, ...}
 #
 SESS = {}


 def load_environment():
     path = os.path.expanduser("~/.couchdb-infra-cm.cfg")
     if not os.path.exists(path):
         print(f"Missing config file: {path}")
         exit(1)
     parser = cp.ConfigParser()
     parser.read([path])

     for section in parser.sections():
         if section.startswith("extra"):
             split = section.split(".")
             if len(split) != 2:
                 print(f"Invalid 'extra' section {section}")
                 exit(1)

             (_, name) = split

             EXTRA[name] = {
                 "name" : name,
                 "instance_id" : parser.get(section, "id", fallback=name),
                 "ip_addr": parser.get(section, "ip_addr"),
                 "user": parser.get(section, "user"),
                 "system": {
                     "arch": parser.get(section, "arch"),
                     "num_cpus": int(parser.get(section, "num_cpus")),
                     "ram": int(parser.get(section, "ram"))
                 }
             }

         if not section.startswith("ibmcloud"):
             continue

         split = section.split(".")
         if len(split) == 2:
             (_, env) = split
         elif len(split) == 1:
             env = "<default>"
         else:
             print(f"Invalid 'ibmcloud' section {section}")
             exit(1)

         ENV[env] = {
             "api_key" : parser.get(section, "api_key"),
             "iam_url" : parser.get(section, "iam_url",
                                    fallback=IAM_URL),
             "api_url" : parser.get(section, "api_url",
                                    fallback=IBM_CLOUD_URL),
             "api_generation" : parser.get(section, "api_generation",
                                           fallback=IBM_CLOUD_GENERATION),
             "api_version" : parser.get(section, "api_version",
                                        fallback=IBM_CLOUD_VERSION),
             "crn" : parser.get(section, "crn", fallback=None),
             "instance_id" : parser.get(section, "instance_id",
                                        fallback=None)
         }


 def load_iam_tokens():
     for env in ENV:
         sess = requests.session()
         headers = {
             "Accept": "application/json"
         }
         data = {
             "grant_type": "urn:ibm:params:oauth:grant-type:apikey",
             "apikey": ENV[env]["api_key"]
         }
         resp = sess.post(ENV[env]["iam_url"], headers=headers, data=data)
         resp.raise_for_status()
         body = resp.json()
         token = body["token_type"] + " " + body["access_token"]
         sess.headers["Authorization"] = token
         for hk, hv in env_headers(env).items():
             sess.headers[hk] = hv
         SESS[env] = sess


 def init():
     load_environment()
     load_iam_tokens()


 def list_instances():
     for env in ENV:
         if env != "power":
             yield from list_x86_instances(env)
     for inst  in EXTRA.values():
         yield inst


 def list_x86_instances(env):
     url = ENV[env]["api_url"] + "/instances"
     sess = SESS[env]
     while url:
         resp = sess.get(url, params=params(env))
         body = resp.json()
         for instance in body["instances"]:
             interface_url = instance["primary_network_interface"]["href"]
             resp = sess.get(interface_url, params=params(env))
             instance["primary_network_interface"] = resp.json()
             yield instance
         url = body.get("next")

 def params(env):
     params = {"limit": 100}
     if ENV[env].get("api_version"):
         params["version"] = ENV[env]["api_version"]
     if ENV[env].get("api_generation"):
         params["generation"] = ENV[env]["api_generation"]
     return params


 def env_headers(env):
     headers = {}
     if ENV[env].get("crn"):
         headers["crn"] = ENV[env]["crn"]
     return headers


 def load_bastion(bastions, instance):
     if instance["status"] != "running":
         return

     name = instance["name"]
     if name in bastions:
         print(f"Duplicate bastion found {name}")
         exit(2)

     ip_addr = None
     net_iface = instance["primary_network_interface"]
     floating_ips = net_iface.get("floating_ips", [])
     if not floating_ips:
         print(f"Bastion is missing a public IP: {name}")
         exit(2)
     ip_addr = floating_ips[0]["address"]

     bastions[name] = {
         "instance": {
             "id": instance["id"],
             "name": instance["name"],
             "created_at": instance["created_at"],
             "profile": instance["profile"]["name"],
             "vpc": instance["vpc"]["name"],
             "zone": instance["zone"]["name"],
             "subnet": net_iface["subnet"]["name"]
         },
         "ip_addrs": {
             "public": ip_addr,
             "private": get_private_ip(instance)
         },
         "system": {
             "arch": instance["vcpu"]["architecture"],
             "num_cpus": instance["vcpu"]["count"],
             "ram": instance["memory"]
         },
         "user": "root"
     }


 def load_ci_agent(ci_agents, instance):
     if instance["status"] != "running":
         return

     name = instance["name"]
     if name in ci_agents:
         print(f"Duplicate ci_agent found {name}")
         exit(2)
     net_iface = instance["primary_network_interface"]

     ci_agents[name] = {
         "instance": {
             "id": instance["id"],
             "name": instance["name"],
             "created_at": instance["created_at"],
             "profile": instance["profile"]["name"],
             "vpc": instance["vpc"]["name"],
             "zone": instance["zone"]["name"],
             "subnet": net_iface["subnet"]["name"]
         },
         "ip_addrs": {
             "bastion_ip": None,
             "bastion_host": None,
             "public": None,
             "private": get_private_ip(instance)
         },
         "system": {
             "arch": instance["vcpu"]["architecture"],
             "num_cpus": instance["vcpu"]["count"],
             "ram": instance["memory"]
         },
         "user": "root"
     }

 def load_s390x_ci_agent(ci_agents, instance):
     name = instance["name"]
     if name in ci_agents:
         print(f"Duplicate ci_agent found {name}")
         exit(2)
     ci_agents[name] = {
         "instance": {
             "id": instance["instance_id"],
             "name" : instance["name"],
             "subnet": None
         },
         "ip_addrs": {
             "bastion_ip": None,
             "bastion_host": None,
             "public": instance["ip_addr"]
         },
         "system": instance["system"],
         "user": instance["user"]
     }

 def load_ppc64le_ci_agent(ci_agents, instance):
     name = instance["name"]
     if name in ci_agents:
         print(f"Duplicate ci_agent found {name}")
         exit(2)

     ci_agents[name] = {
         "instance": {
             "id": instance["instance_id"],
             "name" : instance["name"],
             "subnet": None
         },
         "ip_addrs": {
             "bastion_ip": None,
             "bastion_host": None,
             "public": instance["ip_addr"]
         },
         "system": instance["system"],
         "user": instance["user"]
     }

 def get_private_ip(instance):
     ip = instance["primary_network_interface"]["primary_ipv4_address"]
     if ip:
         return ip

     for iface in instance["network_interfaces"]:
         if iface.get("primary_ipv4_address"):
             return iface["primary_ipv4_address"]

     raise Exception("Unable to locate a private IP address")


 def assign_bastions(bastions, ci_agents):
     subnets = {}
     for (host, bastion) in bastions.items():
         subnet = bastion["instance"]["subnet"]
         ip_addr = bastion["ip_addrs"]["public"]
         assert subnet not in subnets
         subnets[subnet] = (ip_addr, host)
     for (host, ci_agent) in ci_agents.items():
         if ci_agent["system"]["arch"] in ["ppc64le", "s390x"]:
             # Power & s390x an external IP without bastions
             continue
         subnet = ci_agent["instance"]["subnet"]
         assert subnet in subnets
         (bastion_ip, bastion_host) = subnets[subnet]
         ci_agent["ip_addrs"]["bastion_ip"] = bastion_ip
         ci_agent["ip_addrs"]["bastion_host"] = bastion_host


 def write_inventory(fname, bastions, ci_agents):
     inventory = {"all": {
         "children": {
             "bastions": {
                 "hosts": bastions
             },
             "ci_agents": {
                 "hosts": ci_agents
             }
         }
     }}

     with open(fname, "w") as handle:
         yaml.dump(inventory, stream=handle, default_flow_style=False)


 def write_ssh_cfg(filename, bastions, ci_agents):
     bastion_tmpl = textwrap.dedent("""\
         Host {host}
           Hostname {ip_addr}
           User {user}
           ForwardAgent yes
           StrictHostKeyChecking no
           ControlMaster auto
           ControlPath /tmp/ansible-%r@%h:%p
           ControlPersist 30m

         """)
     ci_agent_tmpl = textwrap.dedent("""\
         Host {host}
           Hostname {ip_addr}
           User {user}
           StrictHostKeyChecking no
           ProxyCommand /usr/bin/ssh -W %h:%p -q root@{bastion_host}

         """)
     with open(filename, "w") as handle:
         for host, info in sorted(bastions.items()):
             args = {
                 "user": info["user"],
                 "host": host,
                 "ip_addr": info["ip_addrs"]["public"]
             }
             entry = bastion_tmpl.format(**args)
             handle.write(entry)
         for host, info in sorted(ci_agents.items()):
             user = info["user"]
             if info["system"]["arch"] in ["ppc64le", "s390x"]:
                 # Power or s390x use an external IP directly
                 args = {
                     "user": user,
                     "host": host,
                     "ip_addr": info["ip_addrs"]["public"]
                 }
                 entry = bastion_tmpl.format(**args)
             else:
                 bastion_ip =  info["ip_addrs"]["bastion_ip"]
                 bastion_host = info["ip_addrs"]["bastion_host"]
                 args = {
                     "user": user,
                     "host": host,
                     "ip_addr": info["ip_addrs"]["private"],
                     "bastion_ip": bastion_ip,
                     "bastion_host": bastion_host
                 }
                 entry = ci_agent_tmpl.format(**args)
             handle.write(entry)


 def parse_args():
     parser = ap.ArgumentParser(description="Inventory Generation")
     parser.add_argument(
             "--inventory",
             default="production",
             metavar="FILE",
             type=str,
             help="Inventory filename to write"
         )
     parser.add_argument(
             "--ssh-cfg",
             default="ssh.cfg",
             metavar="FILE",
             type=str,
             help="SSH config filename to write"
         )
     return parser.parse_args()


 def main():
     args = parse_args()

     init()

     bastions = {}
     ci_agents = {}

     for instance in list_instances():
         if instance["name"].startswith("couchdb-bastion"):
             load_bastion(bastions, instance)
         elif instance["name"].startswith("couchdb-worker"):
             load_ci_agent(ci_agents, instance)
         elif instance["system"]["arch"] == "ppc64le":
             load_ppc64le_ci_agent(ci_agents, instance)
         elif instance["system"]["arch"] == "s390x":
             load_s390x_ci_agent(ci_agents, instance)

     assign_bastions(bastions, ci_agents)

     write_inventory(args.inventory, bastions, ci_agents)
     sshf = args.ssh_cfg
     write_ssh_cfg(sshf, bastions, ci_agents)
     sshf_full = os.path.abspath(os.path.expanduser(os.path.expandvars(sshf)))
     print("Add 'Include %s' to your ~/.ssh/config file" % sshf_full)


 if __name__ == "__main__":
     main()
	#!/usr/bin/env python3

	import argparse as ap
	import configparser as cp
	import json
	import os
	import re
	import textwrap

	import requests
	import yaml


	# Environments read from config file
	# {"cloudant": {"api_key": "abc123", ...}
	#
	ENV = {}

	# Hard-coded VMs managed by someone else without access to cloud.ibm.com
	EXTRA = {}

	IBM_CLOUD_URL = "https://us-south.iaas.cloud.ibm.com/v1"
	IAM_URL = "https://iam.cloud.ibm.com/identity/token"

	IBM_CLOUD_GENERATION = "2"
	IBM_CLOUD_VERSION = "2019-08-09"

	# Authed API request sessions, one per environment
	# {"cloudant": <Session>, ...}
	#
	SESS = {}


	def load_environment():
	path = os.path.expanduser("~/.couchdb-infra-cm.cfg")
	if not os.path.exists(path):
	print(f"Missing config file: {path}")
	exit(1)
	parser = cp.ConfigParser()
	parser.read([path])

	for section in parser.sections():
	if section.startswith("extra"):
	split = section.split(".")
	if len(split) != 2:
	print(f"Invalid 'extra' section {section}")
	exit(1)

	(_, name) = split

	EXTRA[name] = {
	"name" : name,
	"instance_id" : parser.get(section, "id", fallback=name),
	"ip_addr": parser.get(section, "ip_addr"),
	"user": parser.get(section, "user"),
	"system": {
	"arch": parser.get(section, "arch"),
	"num_cpus": int(parser.get(section, "num_cpus")),
	"ram": int(parser.get(section, "ram"))
	}
	}

	if not section.startswith("ibmcloud"):
	continue

	split = section.split(".")
	if len(split) == 2:
	(_, env) = split
	elif len(split) == 1:
	env = "<default>"
	else:
	print(f"Invalid 'ibmcloud' section {section}")
	exit(1)

	ENV[env] = {
	"api_key" : parser.get(section, "api_key"),
	"iam_url" : parser.get(section, "iam_url",
	fallback=IAM_URL),
	"api_url" : parser.get(section, "api_url",
	fallback=IBM_CLOUD_URL),
	"api_generation" : parser.get(section, "api_generation",
	fallback=IBM_CLOUD_GENERATION),
	"api_version" : parser.get(section, "api_version",
	fallback=IBM_CLOUD_VERSION),
	"crn" : parser.get(section, "crn", fallback=None),
	"instance_id" : parser.get(section, "instance_id",
	fallback=None)
	}


	def load_iam_tokens():
	for env in ENV:
	sess = requests.session()
	headers = {
	"Accept": "application/json"
	}
	data = {
	"grant_type": "urn:ibm:params:oauth:grant-type:apikey",
	"apikey": ENV[env]["api_key"]
	}
	resp = sess.post(ENV[env]["iam_url"], headers=headers, data=data)
	resp.raise_for_status()
	body = resp.json()
	token = body["token_type"] + " " + body["access_token"]
	sess.headers["Authorization"] = token
	for hk, hv in env_headers(env).items():
	sess.headers[hk] = hv
	SESS[env] = sess


	def init():
	load_environment()
	load_iam_tokens()


	def list_instances():
	for env in ENV:
	if env != "power":
	yield from list_x86_instances(env)
	for inst in EXTRA.values():
	yield inst


	def list_x86_instances(env):
	url = ENV[env]["api_url"] + "/instances"
	sess = SESS[env]
	while url:
	resp = sess.get(url, params=params(env))
	body = resp.json()
	for instance in body["instances"]:
	interface_url = instance["primary_network_interface"]["href"]
	resp = sess.get(interface_url, params=params(env))
	instance["primary_network_interface"] = resp.json()
	yield instance
	url = body.get("next")

	def params(env):
	params = {"limit": 100}
	if ENV[env].get("api_version"):
	params["version"] = ENV[env]["api_version"]
	if ENV[env].get("api_generation"):
	params["generation"] = ENV[env]["api_generation"]
	return params


	def env_headers(env):
	headers = {}
	if ENV[env].get("crn"):
	headers["crn"] = ENV[env]["crn"]
	return headers


	def load_bastion(bastions, instance):
	if instance["status"] != "running":
	return

	name = instance["name"]
	if name in bastions:
	print(f"Duplicate bastion found {name}")
	exit(2)

	ip_addr = None
	net_iface = instance["primary_network_interface"]
	floating_ips = net_iface.get("floating_ips", [])
	if not floating_ips:
	print(f"Bastion is missing a public IP: {name}")
	exit(2)
	ip_addr = floating_ips[0]["address"]

	bastions[name] = {
	"instance": {
	"id": instance["id"],
	"name": instance["name"],
	"created_at": instance["created_at"],
	"profile": instance["profile"]["name"],
	"vpc": instance["vpc"]["name"],
	"zone": instance["zone"]["name"],
	"subnet": net_iface["subnet"]["name"]
	},
	"ip_addrs": {
	"public": ip_addr,
	"private": get_private_ip(instance)
	},
	"system": {
	"arch": instance["vcpu"]["architecture"],
	"num_cpus": instance["vcpu"]["count"],
	"ram": instance["memory"]
	},
	"user": "root"
	}


	def load_ci_agent(ci_agents, instance):
	if instance["status"] != "running":
	return

	name = instance["name"]
	if name in ci_agents:
	print(f"Duplicate ci_agent found {name}")
	exit(2)
	net_iface = instance["primary_network_interface"]

	ci_agents[name] = {
	"instance": {
	"id": instance["id"],
	"name": instance["name"],
	"created_at": instance["created_at"],
	"profile": instance["profile"]["name"],
	"vpc": instance["vpc"]["name"],
	"zone": instance["zone"]["name"],
	"subnet": net_iface["subnet"]["name"]
	},
	"ip_addrs": {
	"bastion_ip": None,
	"bastion_host": None,
	"public": None,
	"private": get_private_ip(instance)
	},
	"system": {
	"arch": instance["vcpu"]["architecture"],
	"num_cpus": instance["vcpu"]["count"],
	"ram": instance["memory"]
	},
	"user": "root"
	}

	def load_s390x_ci_agent(ci_agents, instance):
	name = instance["name"]
	if name in ci_agents:
	print(f"Duplicate ci_agent found {name}")
	exit(2)
	ci_agents[name] = {
	"instance": {
	"id": instance["instance_id"],
	"name" : instance["name"],
	"subnet": None
	},
	"ip_addrs": {
	"bastion_ip": None,
	"bastion_host": None,
	"public": instance["ip_addr"]
	},
	"system": instance["system"],
	"user": instance["user"]
	}

	def load_ppc64le_ci_agent(ci_agents, instance):
	name = instance["name"]
	if name in ci_agents:
	print(f"Duplicate ci_agent found {name}")
	exit(2)

	ci_agents[name] = {
	"instance": {
	"id": instance["instance_id"],
	"name" : instance["name"],
	"subnet": None
	},
	"ip_addrs": {
	"bastion_ip": None,
	"bastion_host": None,
	"public": instance["ip_addr"]
	},
	"system": instance["system"],
	"user": instance["user"]
	}

	def get_private_ip(instance):
	ip = instance["primary_network_interface"]["primary_ipv4_address"]
	if ip:
	return ip

	for iface in instance["network_interfaces"]:
	if iface.get("primary_ipv4_address"):
	return iface["primary_ipv4_address"]

	raise Exception("Unable to locate a private IP address")


	def assign_bastions(bastions, ci_agents):
	subnets = {}
	for (host, bastion) in bastions.items():
	subnet = bastion["instance"]["subnet"]
	ip_addr = bastion["ip_addrs"]["public"]
	assert subnet not in subnets
	subnets[subnet] = (ip_addr, host)
	for (host, ci_agent) in ci_agents.items():
	if ci_agent["system"]["arch"] in ["ppc64le", "s390x"]:
	# Power & s390x an external IP without bastions
	continue
	subnet = ci_agent["instance"]["subnet"]
	assert subnet in subnets
	(bastion_ip, bastion_host) = subnets[subnet]
	ci_agent["ip_addrs"]["bastion_ip"] = bastion_ip
	ci_agent["ip_addrs"]["bastion_host"] = bastion_host


	def write_inventory(fname, bastions, ci_agents):
	inventory = {"all": {
	"children": {
	"bastions": {
	"hosts": bastions
	},
	"ci_agents": {
	"hosts": ci_agents
	}
	}
	}}

	with open(fname, "w") as handle:
	yaml.dump(inventory, stream=handle, default_flow_style=False)


	def write_ssh_cfg(filename, bastions, ci_agents):
	bastion_tmpl = textwrap.dedent("""\
	Host {host}
	Hostname {ip_addr}
	User {user}
	ForwardAgent yes
	StrictHostKeyChecking no
	ControlMaster auto
	ControlPath /tmp/ansible-%r@%h:%p
	ControlPersist 30m

	""")
	ci_agent_tmpl = textwrap.dedent("""\
	Host {host}
	Hostname {ip_addr}
	User {user}
	StrictHostKeyChecking no
	ProxyCommand /usr/bin/ssh -W %h:%p -q root@{bastion_host}

	""")
	with open(filename, "w") as handle:
	for host, info in sorted(bastions.items()):
	args = {
	"user": info["user"],
	"host": host,
	"ip_addr": info["ip_addrs"]["public"]
	}
	entry = bastion_tmpl.format(**args)
	handle.write(entry)
	for host, info in sorted(ci_agents.items()):
	user = info["user"]
	if info["system"]["arch"] in ["ppc64le", "s390x"]:
	# Power or s390x use an external IP directly
	args = {
	"user": user,
	"host": host,
	"ip_addr": info["ip_addrs"]["public"]
	}
	entry = bastion_tmpl.format(**args)
	else:
	bastion_ip = info["ip_addrs"]["bastion_ip"]
	bastion_host = info["ip_addrs"]["bastion_host"]
	args = {
	"user": user,
	"host": host,
	"ip_addr": info["ip_addrs"]["private"],
	"bastion_ip": bastion_ip,
	"bastion_host": bastion_host
	}
	entry = ci_agent_tmpl.format(**args)
	handle.write(entry)


	def parse_args():
	parser = ap.ArgumentParser(description="Inventory Generation")
	parser.add_argument(
	"--inventory",
	default="production",
	metavar="FILE",
	type=str,
	help="Inventory filename to write"
	)
	parser.add_argument(
	"--ssh-cfg",
	default="ssh.cfg",
	metavar="FILE",
	type=str,
	help="SSH config filename to write"
	)
	return parser.parse_args()


	def main():
	args = parse_args()

	init()

	bastions = {}
	ci_agents = {}

	for instance in list_instances():
	if instance["name"].startswith("couchdb-bastion"):
	load_bastion(bastions, instance)
	elif instance["name"].startswith("couchdb-worker"):
	load_ci_agent(ci_agents, instance)
	elif instance["system"]["arch"] == "ppc64le":
	load_ppc64le_ci_agent(ci_agents, instance)
	elif instance["system"]["arch"] == "s390x":
	load_s390x_ci_agent(ci_agents, instance)

	assign_bastions(bastions, ci_agents)

	write_inventory(args.inventory, bastions, ci_agents)
	sshf = args.ssh_cfg
	write_ssh_cfg(sshf, bastions, ci_agents)
	sshf_full = os.path.abspath(os.path.expanduser(os.path.expandvars(sshf)))
	print("Add 'Include %s' to your ~/.ssh/config file" % sshf_full)


	if __name__ == "__main__":
	main()