Merge pull request #1 from apache/multiple-environments
Make it possible to discover instances from various IBM cloud accounts
diff --git a/README.md b/README.md
index df88f98..421bbc1 100644
--- a/README.md
+++ b/README.md
@@ -7,10 +7,14 @@
Setup
---
- $ virtualenv venv
+ $ python3 -m venv venv
$ source venv/bin/activate
$ pip install -r requirements.txt
+On BigSur Mac may have to do:
+
+ $ env LDFLAGS="-L$(brew --prefix openssl@1.1)/lib" CFLAGS="-I$(brew --prefix openssl@1.1)/include" pip install -r requirements.txt
+
Provisioning VMs
---
diff --git a/production b/production
index da6f32d..896ab1c 100644
--- a/production
+++ b/production
@@ -141,8 +141,8 @@
ram: 8
couchdb-worker-x86-64-debian-dal-1-08:
instance:
- created_at: '2020-12-04T17:10:11Z'
- id: 0717_07f645ee-bc5b-4dfc-a342-b59b9fe9c3cd
+ created_at: '2021-03-08T21:13:51Z'
+ id: 0717_d015b177-e247-4c10-9fdd-a06009127409
name: couchdb-worker-x86-64-debian-dal-1-08
profile: cx2-4x8
subnet: couchdb-ci-farm-dal-1
@@ -150,7 +150,7 @@
zone: us-south-1
ip_addrs:
bastion: 169.48.153.153
- private: 10.240.0.7
+ private: 10.240.0.4
public: null
system:
arch: amd64
diff --git a/ssh.cfg b/ssh.cfg
index 699d104..3a596c5 100644
--- a/ssh.cfg
+++ b/ssh.cfg
@@ -71,7 +71,7 @@
ControlPersist 30m
Host couchdb-worker-x86-64-debian-dal-1-08
- Hostname 10.240.0.7
+ Hostname 10.240.0.4
User root
StrictHostKeyChecking no
ProxyCommand /usr/bin/ssh -F ./ssh.cfg -W %h:%p -q root@169.48.153.153
diff --git a/tools/gen-config b/tools/gen-config
index 4df5e2f..e1e0932 100755
--- a/tools/gen-config
+++ b/tools/gen-config
@@ -1,7 +1,7 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
import argparse as ap
-import ConfigParser as cp
+import configparser as cp
import json
import os
import re
@@ -11,91 +11,137 @@
import yaml
+# Environments read from config file
+# {"cloudant": {"api_key": "abc123", ...}
+# {"power": {"api_key", "cde456", ...}
+#
+ENV = {}
+
IBM_CLOUD_URL = "https://us-south.iaas.cloud.ibm.com/v1/"
IAM_URL = "https://iam.cloud.ibm.com/identity/token"
IBM_CLOUD_GENERATION = "2"
IBM_CLOUD_VERSION = "2019-08-09"
-API_KEY = None
-IAM_TOKEN = None
-SESS = requests.session()
+# Authed API request sessions, one per environment
+# {"cloudant": <Session>, ...}
+#
+SESS = {}
-def tostr(obj):
- ret = {}
- for k, v in obj.items():
- if isinstance(k, unicode):
- k = k.encode("utf-8")
- if isinstance(v, dict):
- ret[k] = tostr(v)
- elif isinstance(v, unicode):
- ret[k] = v.encode("utf-8")
- else:
- ret[k] = v
- return ret
-
-
-def load_api_key():
- global API_KEY
+def load_environment():
path = os.path.expanduser("~/.couchdb-infra-cm.cfg")
if not os.path.exists(path):
- print "Missing config file: " + path
+ print(f"Missing config file: {path}")
exit(1)
- parser = cp.SafeConfigParser()
+ parser = cp.ConfigParser()
parser.read([path])
- API_KEY = parser.get("ibmcloud", "api_key")
+
+ for section in parser.sections():
+ if not section.startswith("ibmcloud"):
+ continue
+
+ split = section.split(".")
+ if len(split) == 2:
+ (_, env) = split
+ elif len(split) == 1:
+ env = "<default>"
+ else:
+ print(f"Invalid 'ibmcloud' section {section}")
+ exit(1)
+
+ ENV[env] = {
+ "api_key" : parser.get(section, "api_key"),
+ "iam_url" : parser.get(section, "iam_url",
+ fallback=IAM_URL),
+ "api_url" : parser.get(section, "api_url",
+ fallback=IBM_CLOUD_URL),
+ "api_generation" : parser.get(section, "api_generation",
+ fallback=IBM_CLOUD_GENERATION),
+ "api_version" : parser.get(section, "api_version",
+ fallback=IBM_CLOUD_VERSION)
+ }
-def load_iam_token():
- global IAM_TOKEN
- headers = {
- "Accept": "application/json"
- }
- data = {
- "grant_type": "urn:ibm:params:oauth:grant-type:apikey",
- "apikey": API_KEY
- }
- resp = SESS.post(IAM_URL, headers=headers, data=data)
- resp.raise_for_status()
- body = resp.json()
- IAM_TOKEN = body["token_type"] + " " + body["access_token"]
- SESS.headers["Authorization"] = IAM_TOKEN
+def load_iam_tokens():
+ for env in ENV:
+ sess = requests.session()
+ headers = {
+ "Accept": "application/json"
+ }
+ data = {
+ "grant_type": "urn:ibm:params:oauth:grant-type:apikey",
+ "apikey": ENV[env]["api_key"]
+ }
+ resp = sess.post(ENV[env]["iam_url"], headers=headers, data=data)
+ resp.raise_for_status()
+ body = resp.json()
+ token = body["token_type"] + " " + body["access_token"]
+ sess.headers["Authorization"] = token
+ SESS[env] = sess
def init():
- load_api_key()
- load_iam_token()
+ load_environment()
+ load_iam_tokens()
def list_instances():
- url = IBM_CLOUD_URL + "/instances"
- params = {
- "version": IBM_CLOUD_VERSION,
- "generation": IBM_CLOUD_GENERATION,
- "limit": 100
- }
+ for env in ENV:
+ yield from list_x86_instances(env)
+ #yield from list_power_instances(env)
+
+
+def list_x86_instances(env):
+ url = ENV[env]["api_url"] + "/instances"
+ sess = SESS[env]
while url:
- resp = SESS.get(url, params=params)
+ resp = sess.get(url, params=params(env))
body = resp.json()
for instance in body["instances"]:
interface_url = instance["primary_network_interface"]["href"]
- resp = SESS.get(interface_url, params=params)
+ resp = sess.get(interface_url, params=params(env))
instance["primary_network_interface"] = resp.json()
yield instance
url = body.get("next")
+def list_power_instances(env):
+ url = ENV[env]["api_url"] + "/pvmInstances"
+ sess = SESS[env]
+ while url:
+ resp = sess.get(url, params=params(env))
+ body = resp.json()
+ for instance in body["instances"]:
+ interface_url = instance["primary_network_interface"]["href"]
+ resp = sess.get(interface_url, params=params(env))
+ instance["primary_network_interface"] = resp.json()
+ yield instance
+ url = body.get("next")
+
+
+def params(env):
+ return {
+ "version": ENV[env]["api_version"],
+ "generation" : ENV[env]["api_generation"],
+ "limit": 100
+ }
+
+
def load_bastion(bastions, instance):
if instance["status"] != "running":
return
name = instance["name"]
+ if name in bastions:
+ print(f"Duplicate bastion found {name}")
+ exit(2)
+
ip_addr = None
net_iface = instance["primary_network_interface"]
floating_ips = net_iface.get("floating_ips", [])
if not floating_ips:
- print "Bastion is missing a public IP: %s" % name
+ print(f"Bastion is missing a public IP: {name}")
exit(2)
ip_addr = floating_ips[0]["address"]
@@ -126,6 +172,9 @@
return
name = instance["name"]
+ if name in ci_agents:
+ print(f"Duplicate ci_agent found {name}")
+ exit(2)
net_iface = instance["primary_network_interface"]
ci_agents[name] = {
@@ -189,7 +238,7 @@
}}
with open(fname, "w") as handle:
- yaml.dump(tostr(inventory), stream=handle, default_flow_style=False)
+ yaml.dump(inventory, stream=handle, default_flow_style=False)
def write_ssh_cfg(filename, bastions, ci_agents):
@@ -251,6 +300,7 @@
)
return parser.parse_args()
+
def main():
args = parse_args()
