| #cloud-config |
| # 2021-02-19 21:48 |
| |
| mounts: |
| - [tmpfs, /tmp, tmpfs, "defaults,noatime,size=10%"] |
| - [tmpfs, /var/lib/docker, tmpfs, "defaults,noatime,size=66%"] |
| - [tmpfs, /home/runner/actions-runner/_work, tmpfs, "defaults,noatime"] |
| |
| users: |
| - default |
| - name: runner |
| |
| packages: |
| - awscli |
| - build-essential |
| - docker.io |
| - git |
| - iptables-persistent |
| - jq |
| - parallel |
| - python3-dev |
| - python3-venv |
| - python3-wheel |
| - yarn |
| - vector |
| |
| runcmd: |
| - |
| - bash |
| - -c |
| # https://github.com/actions/virtual-environments/blob/525f79f479cca77aef4e0a680548b65534c64a18/images/linux/scripts/installers/docker-compose.sh |
| - | |
| URL=$(curl -s https://api.github.com/repos/docker/compose/releases/latest | jq -r '.assets[].browser_download_url | select(endswith("docker-compose-Linux-x86_64"))') |
| curl -L $URL -o /usr/local/bin/docker-compose |
| chmod +x /usr/local/bin/docker-compose |
| - |
| - bash |
| - -c |
| - | |
| set -exu -o pipefail |
| echo "AWS_DEFAULT_REGION=$(cloud-init query region)" >> /etc/environment |
| # Set an env var (that is visible in runners) that will let us know we are on a self-hosted runner |
| echo 'AIRFLOW_SELF_HOSTED_RUNNER="[\"self-hosted\"]"' >> /etc/environment |
| set -a |
| . /etc/environment |
| set +a |
| echo "ASG_GROUP_NAME=$(aws ec2 describe-tags --filter Name=resource-id,Values=$(cloud-init query instance_id) Name=key,Values=aws:autoscaling:groupName \ |
| | jq -r '@sh "\(.Tags[0].Value)"')" >> /etc/environment |
| - [systemctl, daemon-reload] |
| - |
| - bash |
| - -c |
| - | |
| python3 -mvenv /opt/runner-supervisor |
| /opt/runner-supervisor/bin/pip install -U pip python-dynamodb-lock-whatnick==0.9.3 click==7.1.2 psutil 'tenacity~=6.0' |
| - |
| - bash |
| - -c |
| - | |
| set -exu -o pipefail |
| |
| usermod -G docker -a runner |
| |
| mkdir -p ~runner/actions-runner |
| find ~runner -exec chown runner: {} + |
| cd ~runner/actions-runner |
| |
| RUNNER_VERSION="$0" |
| |
| curl -L "https://github.com/ashb/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" | tar -zx |
| |
| set -a |
| . /etc/environment |
| set +a |
| |
| aws s3 cp s3://airflow-ci-assets/runner-supervisor.py /opt/runner-supervisor/bin/runner-supervisor |
| chmod 755 /opt/runner-supervisor/bin/runner-supervisor |
| |
| # Log in to a paid docker user to get unlimited docker pulls |
| aws ssm get-parameter --with-decryption --name /runners/apache/airflow/dockerPassword | \ |
| jq .Parameter.Value -r | \ |
| sudo -u runner docker login --username airflowcirunners --password-stdin |
| - 2.277.1-airflow1 |
| - [systemctl, enable, --now, iptables.service] |
| # Restart docker after applying the user firewall -- else some rules/chains might be list! |
| - [systemctl, restart, docker.service] |
| - [systemctl, enable, --now, vector.service] |
| - [systemctl, enable, --now, actions.runner.service] |
| - |
| - bash |
| - -c |
| - | |
| echo "Pre-loading commonly used docker images from S3" |
| set -eux -o pipefail |
| aws s3 cp s3://airflow-ci-assets/pre-baked-images.tar.gz - | docker load |
| |
| write_files: |
| - path: /etc/systemd/system/actions.runner.service |
| content: | |
| [Unit] |
| Description=GitHub Actions Runner |
| After=network.target actions.runner-supervisor.service |
| Requires=actions.runner-supervisor.service |
| BindsTo=actions.runner-supervisor.service |
| |
| [Service] |
| ExecStartPre=!/usr/local/sbin/runner-cleanup-workdir.sh |
| ExecStart=/home/runner/actions-runner/run.sh --once --startuptype service |
| ExecStop=/usr/local/bin/stop-runner-if-no-job.sh $MAINPID |
| EnvironmentFile=/etc/environment |
| Environment=GITHUB_ACTIONS_RUNNER_CHANNEL_TIMEOUT=300 |
| User=runner |
| WorkingDirectory=/home/runner/actions-runner |
| KillMode=mixed |
| KillSignal=SIGTERM |
| TimeoutStopSec=5min |
| Restart=on-success |
| |
| [Install] |
| WantedBy=multi-user.target |
| |
| # Don't put this in ~runner, as these get written before the user is added, and this messes up creating the home dir |
| - path: /usr/local/sbin/runner-cleanup-workdir.sh |
| content: | |
| #!/bin/bash |
| echo "Left-over containers:" |
| docker ps -a |
| docker ps -qa | xargs --verbose --no-run-if-empty docker rm -fv |
| |
| if [[ -d ~runner/actions-runner/_work/airflow/airflow ]]; then |
| cd ~runner/actions-runner/_work/airflow/airflow |
| |
| chown --changes -R runner: . |
| if [[ -e .git ]]; then |
| sudo -u runner bash -c " |
| git reset --hard && \ |
| git submodule deinit --all -f && \ |
| git submodule foreach git clean -fxd && \ |
| git clean -fxd \ |
| " |
| fi |
| fi |
| owner: root:root |
| permissions: '0775' |
| - path: /usr/local/bin/stop-runner-if-no-job.sh |
| content: | |
| #!/bin/bash |
| set -u |
| |
| MAINPID="${MAINPID:-${1:-}}" |
| |
| if [[ -z "$MAINPID" ]]; then |
| echo "No MAINPID, assuming it already crashed!" |
| exit 0 |
| fi |
| |
| if pgrep --ns $MAINPID -a Runner.Worker > /dev/null; then |
| echo "Waiting for current job to finish" |
| while pgrep --ns $MAINPID -a Runner.Worker; do |
| # Job running -- just wait for it to exit |
| sleep 10 |
| done |
| fi |
| |
| # Request shutdown if it's still alive -- because we are in "stop" state it should not restart |
| if pkill --ns $MAINPID Runner.Listener; then |
| # Wait for it to shut down |
| echo "Waiting for main Runner.Listener process to stop" |
| while pgrep --ns $MAINPID -a Runner.Listener; do |
| sleep 5 |
| done |
| fi |
| owner: root:root |
| permissions: '0775' |
| - path: /etc/sudoers.d/runner |
| owner: root:root |
| permissions: '0440' |
| content: | |
| runner ALL=(ALL) NOPASSWD:/usr/sbin/swapoff -a, /usr/bin/rm -f /swapfile, /usr/bin/apt clean |
| - path: /etc/iptables/rules.v4 |
| content: | |
| # Generated by iptables-save v1.8.4 on Thu Jan 14 13:59:27 2021 |
| *filter |
| :INPUT ACCEPT [833:75929] |
| :FORWARD DROP [0:0] |
| :OUTPUT ACCEPT [794:143141] |
| :DOCKER-USER - [0:0] |
| -A FORWARD -j DOCKER-USER |
| # Dis-allow any docker container to access the metadata service |
| -A DOCKER-USER -d 169.254.169.254/32 -j REJECT --reject-with icmp-port-unreachable |
| -A DOCKER-USER -j RETURN |
| COMMIT |
| |
| - path: /usr/local/sbin/actions-runner-ec2-reporting |
| permissions: '0775' |
| content: | |
| #!/bin/bash |
| |
| if pgrep -c Runner.Worker >/dev/null; then |
| # Only report metric when we're doing something -- no point paying to submit zeros |
| aws cloudwatch put-metric-data --metric-name jobs-running --value "$(pgrep -c Runner.Worker)" --namespace github.actions |
| fi |
| - path: /etc/cron.d/cloudwatch-metrics-github-runners |
| content: | |
| */1 * * * * nobody /usr/local/sbin/actions-runner-ec2-reporting |
| |
| - path: /etc/systemd/system/actions.runner-supervisor.service |
| content: | |
| [Unit] |
| Description=Fetch credentials and supervise GitHub Actions Runner |
| After=network.target |
| Before=actions.runner.service |
| |
| [Service] |
| Type=notify |
| ExecStart=/opt/runner-supervisor/bin/python /opt/runner-supervisor/bin/runner-supervisor |
| # We need to run as root to have the ability to open the netlink connector socket |
| User=root |
| WorkingDirectory=/home/runner/actions-runner |
| Restart=always |
| EnvironmentFile=/etc/environment |
| - path: /etc/vector/vector.toml |
| content: | |
| data_dir = "/var/lib/vector" |
| |
| [api] |
| enabled = true |
| |
| # Input data. Change me to a valid input source. |
| [sources.logs] |
| type = "journald" |
| include_units = ["actions.runner.service", "actions.runner-supervisor.service"] |
| |
| [sources.runner-logs] |
| type = "file" |
| include = ["/home/runner/actions-runner/_diag/*.log"] |
| |
| [sources.runner-logs.multiline] |
| start_pattern = '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}' |
| mode = "halt_before" |
| condition_pattern = '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}' |
| timeout_ms = 250 |
| |
| [transforms.grok-runner-logs] |
| type = "grok_parser" |
| inputs=["runner-logs"] |
| pattern = "(?m)\\[%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{NOTSPACE:logger}\\] %{GREEDYDATA:message}" |
| types.timestamp = "timestamp|%F %TZ" |
| |
| [transforms.without_systemd_fields] |
| type = "remove_fields" |
| inputs = ["logs"] |
| fields = ["_CAP_EFFECTIVE", "_SYSTEMD_SLICE", "_SYSTEMD_CGROUP", |
| "_SYSTEMD_INVOCATION_ID", "_SELINUX_CONTEXT", "_COMM", "_BOOT_ID", |
| "_MACHINE_ID", "_STREAM_ID", "_PID", "_GID", "_UID","_TRANSPORT", |
| "__MONOTONIC_TIMESTAMP", "SYSLOG_IDENTIFIER", "PRIORITY", |
| "source_type"] |
| |
| # Output data |
| [sinks.cloudwatch] |
| inputs = ["without_systemd_fields", "grok-runner-logs"] |
| type = "aws_cloudwatch_logs" |
| encoding = "json" |
| create_missing_group = false |
| create_missing_stream = true |
| group_name = "GitHubRunners" |
| stream_name = "{{ host }}" |
| region = "eu-central-1" |
| apt: |
| sources: |
| yarn: |
| source: "deb https://dl.yarnpkg.com/debian/ stable main" |
| keyid: "1646B01B86E50310" |
| timber: |
| source: "deb https://repositories.timber.io/public/vector/deb/ubuntu focal main" |
| key: | |
| -----BEGIN PGP PUBLIC KEY BLOCK----- |
| Version: GnuPG v2 |
| |
| mQENBF9gFZ0BCADETtIHM8y5ehMoyNiZcriK+tHXyKnbZCKtMCKcC4ll94/6pekQ |
| jKIPWg8OXojkCtwua/TsddtQmOhUxAUtv6K0jO8r6sJ8rezMhuNH8J8rMqWgzv9d |
| 2+U7Z7GFgcP0OeD+KigtnR8uyp50suBmEDC8YytmmbESmG261Y38vZME0VvQ+CMy |
| Yi/FvKXBXugaiCtaz0a5jVE86qSZbKbuaTHGiLn05xjTqc4FfyP4fi4oT2r6GGyL |
| Bn5ob84OjXLQwfbZIIrNFR10BvL2SRLL0kKKVlMBBADodtkdwaTt0pGuyEJ+gVBz |
| 629PZBtSrwVRU399jGSfsxoiLca9//c7OJzHABEBAAG0OkNsb3Vkc21pdGggUGFj |
| a2FnZSAodGltYmVyL3ZlY3RvcikgPHN1cHBvcnRAY2xvdWRzbWl0aC5pbz6JATcE |
| EwEIACEFAl9gFZ0CGy8FCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQNUPbLQor |
| xLhf6gf8DyfIpKjvEeW/O8lRUTpkiPKezJbb+udZboCXJKDD02Q9PE3hfEfQRr5X |
| muytL7YMPvzqBVuP3xV5CN3zvtiQQbZiDhstImVyd+t24pQTkjzkvy+A2yvUuIkE |
| RWxuey41f5FNj/7wdfJnHoU9uJ/lvsb7DLXw7FBMZFNBR6LED/d+b61zMzVvmFZA |
| gsrCGwr/jfySwnpShmKdJaMTHQx0qt2RfXwNm2V6i900tAuMUWnmUIz5/9vENPKm |
| 0+31I43a/QgmIrKEePhwn2jfA1oRlYzdv+PbblSTfjTStem+GqQkj9bZsAuqVH8g |
| 3vq0NvX0k2CLi/W9mTiSdHXFChI15A== |
| =k36w |
| -----END PGP PUBLIC KEY BLOCK----- |
