Updated scripts and configuration
diff --git a/README.md b/README.md
index 3eac5aa..5094d0b 100644
--- a/README.md
+++ b/README.md
@@ -19,21 +19,28 @@
[ibmcloud]
api_key = <REDACTED>
-Then simply run the script which will dump the current inventory to stdout. Redirect the output to whatever filename you so desire.
+The `tools/gen-inventory` script can then be used to generate our `production`
+inventory file:
-Provisioning a Node
+ $ ./tools/gen-inventory > production
+
+Setting up CI workers for Jenkins
---
-First, create a new VM of the desired type using whatever means necessary to have root SSH access along with a public IP address (or at least, some method that can be configured into Ansible though you're on your own at this point).
+Once the a new VM has been added into the `production` inventory whoever
+provisioned the VM will need to execute the first Ansible run so that
+the CouchDB infra group has access (where infra group is defined as
+the list of GitHub users in `roles/common/tasks/main.yml`).
-Then run:
+ $ ansible-playbook -i production ci_agents.yml
- ansible-playbook -i W.X.Y.Z, provision.yml
+Once this playbook finishes the new VM should be configured to be usable
+as a Jenkins agent.
-*Note:* Make sure to include the trailing comma (,) in the -i argument or you'll get an error about not being able to parse the inventory.
-Once this has run and you have updated the `production` inventory file (See the section above on generating inventory files) in this directory you can then run:
+Configuring Jenkins
+---
- ansible-playbook -i production ci_agents.yml
-
-And the node will be configured as a new CI agent.
\ No newline at end of file
+Once Ansible has run against a new VM configuring it as an agent in
+Jenkins is fairly straightforward. You can just copy an existing node's
+configuration and update the SSH host IP address.
\ No newline at end of file
diff --git a/ansible.cfg b/ansible.cfg
new file mode 100644
index 0000000..472e641
--- /dev/null
+++ b/ansible.cfg
@@ -0,0 +1,3 @@
+[defaults]
+inventory = ./production
+vault_password_file = ~/.couchdb-ansible-vault
diff --git a/group_vars/all.yaml b/group_vars/all.yaml
deleted file mode 100644
index 3e9e888..0000000
--- a/group_vars/all.yaml
+++ /dev/null
@@ -1 +0,0 @@
-ansible_ssh_user: couchdb
diff --git a/group_vars/ci_agents.yaml b/group_vars/ci_agents.yaml
new file mode 100644
index 0000000..7067c33
--- /dev/null
+++ b/group_vars/ci_agents.yaml
@@ -0,0 +1,2 @@
+ansible_ssh_user: root
+ansible_ssh_common_args: -o StrictHostKeyChecking=no
\ No newline at end of file
diff --git a/production b/production
index 940d582..31efd3c 100644
--- a/production
+++ b/production
@@ -2,21 +2,97 @@
children:
ci_agents:
hosts:
- 169.61.160.207:
+ 169.48.153.210:
boot_volume:
- device: xvda
- name: ci-farm-server-1-boot
+ device: 0717_5afac964-7ec6-4dad-a84d-b09b4d992949-vgqqr
+ name: couchdb-ci-worker-dal-1-2-boot
instance:
- created_at: '2019-08-09T17:10:36.149Z'
- id: ff9bd14d-c0ea-4f99-9d62-0c9b2f566264
- name: ci-farm-server-1
- profile: cc1-4x8
- subnet: couchdb-ci-farm-subnet-2
- vpc: couchdb-ci-farm
- zone: us-south-2
- ips:
- private: 10.240.64.4
- public: 169.61.160.207
+ created_at: '2019-12-11T16:51:02Z'
+ id: 0717_d97c67df-1f04-41f8-9461-9b1d5721e408
+ name: couchdb-ci-worker-dal-1-2
+ profile: cx2-4x8
+ subnet: couchdb-ci-farm-dal-1
+ vpc: couchdb-ci-farm-vpc
+ zone: us-south-1
+ ip_addrs:
+ private: 10.240.0.5
+ public: 169.48.153.210
+ system:
+ arch: amd64
+ num_cpus: 4
+ ram: 8
+ 169.48.153.7:
+ boot_volume:
+ device: 0717_72564344-27ce-4e79-91d8-aacfaba35421-vv2gd
+ name: couchdb-ci-worker-dal-1-1-boot
+ instance:
+ created_at: '2019-12-11T16:50:33Z'
+ id: 0717_4d64226a-ffad-4523-b5b3-78769a1d0bbe
+ name: couchdb-ci-worker-dal-1-1
+ profile: cx2-4x8
+ subnet: couchdb-ci-farm-dal-1
+ vpc: couchdb-ci-farm-vpc
+ zone: us-south-1
+ ip_addrs:
+ private: 10.240.0.4
+ public: 169.48.153.7
+ system:
+ arch: amd64
+ num_cpus: 4
+ ram: 8
+ 169.48.154.118:
+ boot_volume:
+ device: 0717_4abf905c-b565-4537-a4f3-b9e365d945ed-tbfg5
+ name: couchdb-ci-worker-dal-1-4-boot
+ instance:
+ created_at: '2019-12-11T16:51:39Z'
+ id: 0717_c4b21ff3-96e9-45a5-a77c-a90d6ac723dc
+ name: couchdb-ci-worker-dal-1-4
+ profile: cx2-4x8
+ subnet: couchdb-ci-farm-dal-1
+ vpc: couchdb-ci-farm-vpc
+ zone: us-south-1
+ ip_addrs:
+ private: 10.240.0.7
+ public: 169.48.154.118
+ system:
+ arch: amd64
+ num_cpus: 4
+ ram: 8
+ 169.48.154.14:
+ boot_volume:
+ device: 0717_f51ebb9c-5081-47f0-bbf9-07a1b1ba5e73-nwzzg
+ name: couchdb-ci-worker-dal-1-3-boot
+ instance:
+ created_at: '2019-12-11T16:51:21Z'
+ id: 0717_04df61d7-fb30-4251-9f59-7566c93c8a92
+ name: couchdb-ci-worker-dal-1-3
+ profile: cx2-4x8
+ subnet: couchdb-ci-farm-dal-1
+ vpc: couchdb-ci-farm-vpc
+ zone: us-south-1
+ ip_addrs:
+ private: 10.240.0.6
+ public: 169.48.154.14
+ system:
+ arch: amd64
+ num_cpus: 4
+ ram: 8
+ 169.48.154.35:
+ boot_volume:
+ device: 0717_1a5c43f9-a22a-4258-9514-13703dfc5fb0-wkn8z
+ name: couchdb-ci-worker-dal-1-5-boot
+ instance:
+ created_at: '2019-12-11T16:51:55Z'
+ id: 0717_e4857481-a79e-4848-a1c5-38e2577f815c
+ name: couchdb-ci-worker-dal-1-5
+ profile: cx2-4x8
+ subnet: couchdb-ci-farm-dal-1
+ vpc: couchdb-ci-farm-vpc
+ zone: us-south-1
+ ip_addrs:
+ private: 10.240.0.8
+ public: 169.48.154.35
system:
arch: amd64
num_cpus: 4
diff --git a/provision.yml b/provision.yml
deleted file mode 100644
index 24e54af..0000000
--- a/provision.yml
+++ /dev/null
@@ -1,30 +0,0 @@
----
-- hosts: all
- vars:
- ansible_ssh_user: root
- tasks:
- - name: Add user couchdb
- user:
- name: couchdb
- state: present
- shell: /bin/bash
-
- - name: Make sure sudo is installed
- apt:
- name: sudo
- state: latest
-
- - name: Grant sudo access to couchdb user
- copy:
- content: 'couchdb ALL=(ALL) NOPASSWD:ALL'
- dest: /etc/sudoers.d/couchdb
- mode: 0440
-
- - name: Allow CouchDB Infra access to couchdb user
- authorized_key:
- user: couchdb
- state: present
- key: "{{ item }}"
- with_items:
- - https://github.com/davisp.keys
- - https://github.com/wohali.keys
\ No newline at end of file
diff --git a/roles/ci_agent/files/kill-old-docker.sh b/roles/ci_agent/files/kill-old-docker.sh
new file mode 100644
index 0000000..25d8eb0
--- /dev/null
+++ b/roles/ci_agent/files/kill-old-docker.sh
@@ -0,0 +1,10 @@
+THRESHOLD=$(date --date "1 day ago" +%s)
+
+docker ps --format "{{.ID}} {{.CreatedAt}}" | while read LINE; do
+ ID=$(echo $LINE | awk '{print $1}');
+ DATE=$(echo $LINE | awk '{print $2" "$3" "$4}');
+ AGE=$(date --date "$DATE" +%s);
+ if [ $AGE -lt $THRESHOLD ]; then
+ docker rm --force $ID
+ fi;
+done
\ No newline at end of file
diff --git a/roles/ci_agent/tasks/main.yml b/roles/ci_agent/tasks/main.yml
index edf6791..055160f 100644
--- a/roles/ci_agent/tasks/main.yml
+++ b/roles/ci_agent/tasks/main.yml
@@ -41,15 +41,46 @@
packages:
- openjdk-8-jre-headless
-# Copy infra script to delete old docker images
-# Add cron to run docker deleter script
+- name: Add group jenkins
+ become: yes
+ group:
+ name: jenkins
+ gid: 910
-# Add jenkins user with specific uid
-# Write script to download a jar and run it
-# Install script as runit service
+- name: Add user jenkins
+ become: yes
+ user:
+ name: jenkins
+ uid: 910
+ groups:
+ - jenkins
+ - docker
+ state: present
+ shell: /bin/bash
-# Figure out how to configure the password things required
-# for running the jar bits
+- name: Add Apache Infra ssh key
+ become: yes
+ authorized_key:
+ user: jenkins
+ key: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAtxkcKDiPh1OaVzaVdc80daKq2sRy8aAgt8u2uEcLClzMrnv/g19db7XVggfT4+HPCqcbFbO3mtVnUnWWtuSEpDjqriWnEcSj2G1P53zsdKEu9qCGLmEFMgwcq8b5plv78PRdAQn09WCBI1QrNMypjxgCKhNNn45WqV4AD8Jp7/8=
-# Figure out if our Java agent requires all the jars
-# and tooling I saw in the docs (ant, maven, multiple jvms, etc)
+- name: Install kill-old-docker.sh
+ become: yes
+ copy:
+ src: kill-old-docker.sh
+ dest: /usr/local/bin/kill-old-docker.sh
+ mode: 0755
+
+- name: Add kill-old-docker.sh cron entry
+ become: yes
+ cron:
+ name: Kill old docker containers
+ hour: '19'
+ job: /usr/local/bin/kill-old-docker.sh
+
+- name: Add docker prune cron entry
+ become: yes
+ cron:
+ name: Docker prune
+ hour: '19'
+ job: /usr/bin/docker system prune -a -f --filter "until=72h"
diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml
index edd83cc..35159e1 100644
--- a/roles/common/tasks/main.yml
+++ b/roles/common/tasks/main.yml
@@ -1,3 +1,12 @@
+- name: Allow CouchDB Infra access
+ authorized_key:
+ user: root
+ state: present
+ key: "{{ item }}"
+ with_items:
+ - https://github.com/davisp.keys
+ - https://github.com/wohali.keys
+
- name: Install basic ubiquitous packages
become: yes
apt:
@@ -17,12 +26,14 @@
- iperf3
- mtr-tiny
- nload
+ - ntp
- python3
- runit
- runit-systemd
- screen
- software-properties-common
- strace
+ - sudo
- tcpdump
- tmux
- vim
diff --git a/tools/gen-inventory b/tools/gen-inventory
index 3f85b2a..95e9abb 100755
--- a/tools/gen-inventory
+++ b/tools/gen-inventory
@@ -12,15 +12,13 @@
IBM_CLOUD_URL = "https://us-south.iaas.cloud.ibm.com/v1/"
IAM_URL = "https://iam.cloud.ibm.com/identity/token"
-IBM_CLOUD_GENERATION = "1"
+IBM_CLOUD_GENERATION = "2"
IBM_CLOUD_VERSION = "2019-08-09"
API_KEY = None
IAM_TOKEN = None
SESS = requests.session()
-CI_AGENT_RE = re.compile("ci-farm-server-\d+")
-
def tostr(obj):
ret = {}
@@ -91,14 +89,15 @@
return
name = instance["name"]
- floating_ips = instance["primary_network_interface"]["floating_ips"]
- if not len(floating_ips):
- print "Instance {} does not have any floating ips".format(name)
- exit(2)
+ net_iface = instance["primary_network_interface"]
+ floating_ips = net_iface.get("floating_ips", [])
- public_ip = floating_ips[0]["address"]
+ if not floating_ips:
+ return
- ci_agents[public_ip] = {
+ ip_addr = floating_ips[0]["address"]
+
+ ci_agents[ip_addr] = {
"instance": {
"id": instance["id"],
"name": instance["name"],
@@ -106,11 +105,11 @@
"profile": instance["profile"]["name"],
"vpc": instance["vpc"]["name"],
"zone": instance["zone"]["name"],
- "subnet": instance["primary_network_interface"]["subnet"]["name"]
+ "subnet": net_iface["subnet"]["name"]
},
- "ips": {
- "public": public_ip,
- "private": instance["primary_network_interface"]["primary_ipv4_address"]
+ "ip_addrs": {
+ "public": ip_addr,
+ "private": net_iface["primary_ipv4_address"]
},
"boot_volume": {
"device": instance["boot_volume_attachment"]["device"]["id"],
@@ -130,12 +129,8 @@
ci_agents = {}
for instance in list_instances():
- if CI_AGENT_RE.match(instance["name"]):
+ if instance["name"].startswith("couchdb-ci-worker"):
load_ci_agent(ci_agents, instance)
- continue
-
- print "Unknown instance: " + instance["name"]
- exit(2)
inventory = {"all": {
"children": {
