| # -- coding: utf-8 -- |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # -------------------------------------------------------------------- # |
| # Notes |
| # -------------------------------------------------------------------- # |
| # Vrouter |
| # |
| # eth0 router gateway IP for isolated network |
| # eth1 Control IP for hypervisor |
| # eth2 public ip(s) |
| # |
| # VPC Router |
| # |
| # eth0 control interface |
| # eth1 public ip |
| # eth2+ Guest networks |
| # -------------------------------------------------------------------- # |
| import os |
| import logging |
| from . import CsHelper |
| from .CsFile import CsFile |
| from .CsProcess import CsProcess |
| from .CsApp import CsPasswdSvc |
| from .CsAddress import CsDevice |
| from .CsRoute import CsRoute |
| from .CsStaticRoutes import CsStaticRoutes |
| import socket |
| from time import sleep |
| |
| |
| class CsRedundant(object): |
| |
| CS_RAMDISK_DIR = "/ramdisk" |
| CS_PRIO_UP = 1 |
| CS_PRIO_DOWN = -1 |
| CS_ROUTER_DIR = "%s/rrouter" % CS_RAMDISK_DIR |
| CS_TEMPLATES = [ |
| "heartbeat.sh.templ", "check_heartbeat.sh.templ", |
| "arping_gateways.sh.templ" |
| ] |
| CS_TEMPLATES_DIR = "/opt/cloud/templates" |
| CONNTRACKD_BIN = "/usr/sbin/conntrackd" |
| CONNTRACKD_KEEPALIVED_CONFLOCK = "/var/lock/conntrack.lock" |
| CONNTRACKD_CONF = "/etc/conntrackd/conntrackd.conf" |
| RROUTER_LOG = "/var/log/cloud.log" |
| KEEPALIVED_CONF = "/etc/keepalived/keepalived.conf" |
| |
| def __init__(self, config): |
| self.cl = config.cmdline() |
| self.address = config.address() |
| self.config = config |
| |
| def set(self): |
| logging.debug("Router redundancy status is %s", self.cl.is_redundant()) |
| if self.cl.is_redundant(): |
| self._redundant_on() |
| else: |
| self._redundant_off() |
| |
| def _redundant_off(self): |
| CsHelper.service("conntrackd", "stop") |
| CsHelper.service("keepalived", "stop") |
| CsHelper.umount_tmpfs(self.CS_RAMDISK_DIR) |
| CsHelper.rmdir(self.CS_RAMDISK_DIR) |
| CsHelper.rm(self.CONNTRACKD_CONF) |
| CsHelper.rm(self.KEEPALIVED_CONF) |
| |
| def _redundant_on(self): |
| guest = self.address.get_guest_if() |
| |
| # No redundancy if there is no guest network |
| if guest is None: |
| self.set_backup(restart_conntrackd=False) |
| self._redundant_off() |
| return |
| |
| interfaces = [interface for interface in self.address.get_interfaces() if interface.is_guest()] |
| isDeviceReady = False |
| dev = '' |
| for interface in interfaces: |
| if dev == interface.get_device(): |
| continue |
| dev = interface.get_device() |
| logging.info("Wait for devices to be configured so we can start keepalived") |
| devConfigured = CsDevice(dev, self.config).waitfordevice() |
| if devConfigured: |
| command = "ip link show %s | grep 'state UP'" % dev |
| devUp = CsHelper.execute(command) |
| if devUp: |
| logging.info("Device %s is present, let's start keepalived now." % dev) |
| isDeviceReady = True |
| |
| if not isDeviceReady: |
| logging.info("Guest network not configured yet, let's stop router redundancy for now.") |
| CsHelper.service("conntrackd", "stop") |
| CsHelper.service("keepalived", "stop") |
| return |
| |
| CsHelper.mkdir(self.CS_RAMDISK_DIR, 0o755, False) |
| CsHelper.mount_tmpfs(self.CS_RAMDISK_DIR) |
| CsHelper.mkdir(self.CS_ROUTER_DIR, 0o755, False) |
| for s in self.CS_TEMPLATES: |
| d = s |
| if s.endswith(".templ"): |
| d = s.replace(".templ", "") |
| CsHelper.copy_if_needed( |
| "%s/%s" % (self.CS_TEMPLATES_DIR, s), "%s/%s" % (self.CS_ROUTER_DIR, d)) |
| |
| CsHelper.copy_if_needed( |
| "%s/%s" % (self.CS_TEMPLATES_DIR, "keepalived.conf.templ"), self.KEEPALIVED_CONF) |
| CsHelper.copy_if_needed( |
| "%s/%s" % (self.CS_TEMPLATES_DIR, "checkrouter.sh.templ"), "/opt/cloud/bin/checkrouter.sh") |
| |
| CsHelper.execute( |
| 'sed -i "s/--exec $DAEMON;/--exec $DAEMON -- --vrrp;/g" /etc/init.d/keepalived') |
| # checkrouter.sh configuration |
| check_router = CsFile("/opt/cloud/bin/checkrouter.sh") |
| check_router.greplace("[RROUTER_LOG]", self.RROUTER_LOG) |
| check_router.commit() |
| |
| # keepalived configuration |
| keepalived_conf = CsFile(self.KEEPALIVED_CONF) |
| keepalived_conf.search( |
| " router_id ", " router_id %s" % self.cl.get_name()) |
| keepalived_conf.search( |
| " interface ", " interface %s" % guest.get_device()) |
| keepalived_conf.search( |
| " advert_int ", " advert_int %s" % self.cl.get_advert_int()) |
| |
| keepalived_conf.greplace("[RROUTER_BIN_PATH]", self.CS_ROUTER_DIR) |
| keepalived_conf.section("authentication {", "}", [ |
| " auth_type AH \n", " auth_pass %s\n" % self.cl.get_router_password()[:8]]) |
| keepalived_conf.section( |
| "virtual_ipaddress {", "}", self._collect_ips()) |
| |
| # conntrackd configuration |
| conntrackd_template_conf = "%s/%s" % (self.CS_TEMPLATES_DIR, "conntrackd.conf.templ") |
| conntrackd_temp_bkp = "%s/%s" % (self.CS_TEMPLATES_DIR, "conntrackd.conf.templ.bkp") |
| |
| CsHelper.copy(conntrackd_template_conf, conntrackd_temp_bkp) |
| |
| conntrackd_tmpl = CsFile(conntrackd_template_conf) |
| conntrackd_tmpl.section("Multicast {", "}", [ |
| "IPv4_address 225.0.0.50\n", |
| "Group 3780\n", |
| "IPv4_interface %s\n" % guest.get_ip(), |
| "Interface %s\n" % guest.get_device(), |
| "SndSocketBuffer 1249280\n", |
| "RcvSocketBuffer 1249280\n", |
| "Checksum on\n"]) |
| conntrackd_tmpl.section("Address Ignore {", "}", self._collect_ignore_ips()) |
| conntrackd_tmpl.commit() |
| |
| conntrackd_conf = CsFile(self.CONNTRACKD_CONF) |
| |
| is_equals = conntrackd_tmpl.compare(conntrackd_conf) |
| |
| force_keepalived_restart = False |
| proc = CsProcess(['/etc/conntrackd/conntrackd.conf']) |
| |
| if not proc.find() or not is_equals: |
| CsHelper.copy(conntrackd_template_conf, self.CONNTRACKD_CONF) |
| CsHelper.service("conntrackd", "restart") |
| force_keepalived_restart = True |
| |
| # Restore the template file and remove the backup. |
| CsHelper.copy(conntrackd_temp_bkp, conntrackd_template_conf) |
| CsHelper.execute("rm -rf %s" % conntrackd_temp_bkp) |
| |
| # Configure heartbeat cron job - runs every 30 seconds |
| heartbeat_cron = CsFile("/etc/cron.d/heartbeat") |
| heartbeat_cron.add("SHELL=/bin/bash", 0) |
| heartbeat_cron.add( |
| "PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin", 1) |
| heartbeat_cron.add( |
| "* * * * * root $SHELL %s/check_heartbeat.sh 2>&1 > /dev/null" % self.CS_ROUTER_DIR, -1) |
| heartbeat_cron.add( |
| "* * * * * root sleep 30; $SHELL %s/check_heartbeat.sh 2>&1 > /dev/null" % self.CS_ROUTER_DIR, -1) |
| heartbeat_cron.commit() |
| |
| proc = CsProcess(['/usr/sbin/keepalived']) |
| if not proc.find(): |
| force_keepalived_restart = True |
| if keepalived_conf.is_changed() or force_keepalived_restart: |
| keepalived_conf.commit() |
| os.chmod(self.KEEPALIVED_CONF, 0o644) |
| if force_keepalived_restart or not self.cl.is_primary(): |
| CsHelper.service("keepalived", "restart") |
| else: |
| CsHelper.service("keepalived", "reload") |
| |
| def release_lock(self): |
| try: |
| os.remove("/tmp/primary_lock") |
| except OSError: |
| pass |
| |
| def set_lock(self): |
| """ |
| Make sure that primary state changes happen sequentially |
| """ |
| iterations = 10 |
| time_between = 1 |
| |
| for iter in range(0, iterations): |
| try: |
| s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) |
| s.bind('/tmp/primary_lock') |
| return s |
| except socket.error as e: |
| error_code = e.args[0] |
| error_string = e.args[1] |
| logging.info("Primary is already running, waiting") |
| sleep(time_between) |
| |
| def set_fault(self): |
| """ Set fault mode on this router """ |
| if not self.cl.is_redundant(): |
| logging.error("Set fault called on non-redundant router") |
| return |
| |
| self.set_lock() |
| logging.info("Setting router to fault") |
| |
| interfaces = [interface for interface in self.address.get_interfaces() if interface.is_public()] |
| for interface in interfaces: |
| CsHelper.execute("ifconfig %s down" % interface.get_device()) |
| |
| cmd = "%s -C %s" % (self.CONNTRACKD_BIN, self.CONNTRACKD_CONF) |
| CsHelper.execute("%s -s" % cmd) |
| CsHelper.service("ipsec", "stop") |
| CsHelper.service("xl2tpd", "stop") |
| CsHelper.service("dnsmasq", "stop") |
| CsHelper.service("radvd", "stop") |
| |
| interfaces = [interface for interface in self.address.get_interfaces() if interface.needs_vrrp()] |
| for interface in interfaces: |
| CsPasswdSvc(interface.get_gateway() + "," + interface.get_ip()).stop() |
| |
| self.cl.set_fault_state() |
| self.cl.save() |
| self.release_lock() |
| logging.info("Router switched to fault mode") |
| |
| interfaces = [interface for interface in self.address.get_interfaces() if interface.is_public()] |
| CsHelper.reconfigure_interfaces(self.cl, interfaces) |
| |
| def set_backup(self, restart_conntrackd=True): |
| """ Set the current router to backup """ |
| if not self.cl.is_redundant(): |
| logging.error("Set backup called on non-redundant router") |
| return |
| |
| self.set_lock() |
| logging.debug("Setting router to backup") |
| |
| dev = '' |
| interfaces = [interface for interface in self.address.get_interfaces() if interface.is_public()] |
| for interface in interfaces: |
| if dev == interface.get_device(): |
| continue |
| logging.info("Bringing public interface %s down" % interface.get_device()) |
| cmd2 = "ip link set %s down" % interface.get_device() |
| CsHelper.execute(cmd2) |
| dev = interface.get_device() |
| |
| self._remove_ipv6_guest_gateway() |
| |
| if restart_conntrackd: |
| CsHelper.service("conntrackd", "restart") |
| else: |
| CsHelper.service("conntrackd", "stop") |
| CsHelper.service("ipsec", "stop") |
| CsHelper.service("xl2tpd", "stop") |
| |
| interfaces = [interface for interface in self.address.get_interfaces() if interface.needs_vrrp()] |
| for interface in interfaces: |
| CsPasswdSvc(interface.get_gateway() + "," + interface.get_ip()).stop() |
| |
| CsHelper.service("dnsmasq", "stop") |
| |
| self.cl.set_primary_state(False) |
| self.cl.save() |
| self.release_lock() |
| |
| interfaces = [interface for interface in self.address.get_interfaces() if interface.is_public()] |
| CsHelper.reconfigure_interfaces(self.cl, interfaces) |
| logging.info("Router switched to backup mode") |
| |
| def set_primary(self): |
| """ Set the current router to primary """ |
| if not self.cl.is_redundant(): |
| logging.error("Set primary called on non-redundant router") |
| return |
| |
| self.set_lock() |
| logging.debug("Setting router to primary") |
| |
| dev = '' |
| interfaces = [interface for interface in self.address.get_interfaces() if interface.is_public()] |
| route = CsRoute() |
| for interface in interfaces: |
| if dev == interface.get_device(): |
| continue |
| dev = interface.get_device() |
| logging.info("Will proceed configuring device ==> %s" % dev) |
| cmd = "ip link set %s up" % dev |
| if CsDevice(dev, self.config).waitfordevice(): |
| CsHelper.execute(cmd) |
| logging.info("Bringing public interface %s up" % dev) |
| |
| try: |
| gateway = interface.get_gateway() |
| logging.info("Adding gateway ==> %s to device ==> %s" % (gateway, dev)) |
| if dev == CsHelper.PUBLIC_INTERFACES[self.cl.get_type()]: |
| route.add_defaultroute(gateway) |
| except Exception: |
| logging.error("ERROR getting gateway from device %s" % dev) |
| if dev == CsHelper.PUBLIC_INTERFACES[self.cl.get_type()]: |
| try: |
| self._add_ipv6_to_interface(interface, interface.get_ip6()) |
| if interface.get_gateway6(): |
| route.add_defaultroute_v6(interface.get_gateway6()) |
| except Exception as e: |
| logging.error("ERROR adding IPv6, getting IPv6 gateway from device %s: %s" % (dev, e)) |
| else: |
| logging.error("Device %s was not ready could not bring it up" % dev) |
| |
| self._add_ipv6_guest_gateway() |
| |
| logging.debug("Configuring static routes") |
| static_routes = CsStaticRoutes("staticroutes", self.config) |
| static_routes.process() |
| |
| cmd = "%s -C %s" % (self.CONNTRACKD_BIN, self.CONNTRACKD_CONF) |
| CsHelper.execute("%s -c" % cmd) |
| CsHelper.execute("%s -f" % cmd) |
| CsHelper.execute("%s -R" % cmd) |
| CsHelper.execute("%s -B" % cmd) |
| CsHelper.service("ipsec", "restart") |
| CsHelper.service("xl2tpd", "restart") |
| |
| interfaces = [interface for interface in self.address.get_interfaces() if interface.needs_vrrp()] |
| for interface in interfaces: |
| if interface.is_added(): |
| CsPasswdSvc(interface.get_gateway() + "," + interface.get_ip()).restart() |
| |
| CsHelper.service("dnsmasq", "restart") |
| self.cl.set_primary_state(True) |
| self.cl.save() |
| self.release_lock() |
| |
| interfaces = [interface for interface in self.address.get_interfaces() if interface.is_public()] |
| CsHelper.reconfigure_interfaces(self.cl, interfaces) |
| |
| public_devices = list(set([interface.get_device() for interface in interfaces])) |
| if len(public_devices) > 1: |
| # Handle specific failures when multiple public interfaces |
| |
| public_devices.sort() |
| |
| # Ensure the default route is added, or outgoing traffic from VMs with static NAT on |
| # the subsequent interfaces will go from the wrong IP |
| route = CsRoute() |
| dev = '' |
| for interface in interfaces: |
| if dev == interface.get_device(): |
| continue |
| dev = interface.get_device() |
| gateway = interface.get_gateway() |
| if gateway: |
| route.add_route(dev, gateway) |
| |
| # The first public interface has a static MAC address between VRs. Subsequent ones don't, |
| # so an ARP announcement is needed on failover |
| for device in public_devices[1:]: |
| logging.info("Sending garp messages for IPs on %s" % device) |
| for interface in interfaces: |
| if interface.get_device() == device: |
| CsHelper.execute("arping -I %s -U %s -c 1" % (device, interface.get_ip())) |
| |
| logging.info("Router switched to primary mode") |
| |
| def _collect_ignore_ips(self): |
| """ |
| This returns a list of ip objects that should be ignored |
| by conntrackd |
| """ |
| lines = [] |
| lines.append("\t\t\tIPv4_address %s\n" % "127.0.0.1") |
| lines.append("\t\t\tIPv4_address %s\n" % |
| self.address.get_control_if().get_ip()) |
| # FIXME - Do we need to also add any internal network gateways? |
| return lines |
| |
| def _collect_ips(self): |
| """ |
| Construct a list containing all the ips that need to be looked afer by vrrp |
| This is based upon the address_needs_vrrp method in CsAddress which looks at |
| the network type and decides if it is an internal address or an external one |
| |
| In a DomR there will only ever be one address in a VPC there can be many |
| The new code also gives the possibility to cloudstack to have a hybrid device |
| that could function as a router and VPC router at the same time |
| """ |
| lines = [] |
| for interface in self.address.get_interfaces(): |
| if interface.needs_vrrp(): |
| cmdline = self.config.get_cmdline_instance() |
| if not interface.is_added(): |
| continue |
| str = " %s brd %s dev %s\n" % (interface.get_gateway_cidr(), interface.get_broadcast(), interface.get_device()) |
| lines.append(str) |
| return lines |
| |
| def _add_ipv6_to_interface(self, interface, ipv6): |
| """ |
| Add an IPv6 to an interface. This is useful for adding, |
| - guest IPv6 gateway for primary VR guest NIC |
| - public IPv6 for primary VR public NIC as its IPv6 gets lost on link down |
| """ |
| dev = '' |
| if dev == interface.get_device() or not ipv6: |
| return |
| dev = interface.get_device() |
| command = "ip -6 address show %s | grep 'inet6 %s'" % (dev, ipv6) |
| ipConfigured = CsHelper.execute(command) |
| if ipConfigured: |
| logging.info("IPv6 address %s already present for %s" % (ipv6, dev)) |
| return |
| command = "ip link show %s | grep 'state UP'" % dev |
| devUp = CsHelper.execute(command) |
| if not devUp: |
| logging.error("ERROR setting IPv6 address for device %s as it is not ready" % dev) |
| return |
| logging.info("Device %s is present, let's add IPv6 address %s" % (dev, ipv6)) |
| cmd = "ip -6 addr add %s dev %s" % (ipv6, dev) |
| CsHelper.execute(cmd) |
| |
| def _remove_ipv6_to_interface(self, interface, ipv6): |
| """ |
| Remove an IPv6 to an interface. This is useful for removing, |
| - guest IPv6 gateway for primary VR guest NIC |
| """ |
| dev = '' |
| if dev == interface.get_device() or not ipv6: |
| return |
| dev = interface.get_device() |
| command = "ip -6 address show %s | grep 'inet6 %s'" % (dev, ipv6) |
| ipConfigured = CsHelper.execute(command) |
| if ipConfigured: |
| command = "ip link show %s | grep 'state UP'" % dev |
| devUp = CsHelper.execute(command) |
| if not devUp: |
| logging.error("ERROR setting IPv6 address for device %s as it is not ready" % dev) |
| return |
| logging.info("Device %s is present, let's remove IPv6 address %s" % (dev, ipv6)) |
| cmd = "ip -6 addr delete %s dev %s" % (ipv6, dev) |
| CsHelper.execute(cmd) |
| else: |
| logging.info("IPv6 address %s not present for %s" % (ipv6, dev)) |
| return |
| |
| def _enable_radvd(self, dev): |
| """ |
| Setup radvd for primary VR |
| """ |
| if dev == '': |
| return |
| CsHelper.service("radvd", "enable") |
| CsHelper.start_if_stopped("radvd") |
| |
| def _disable_radvd(self, dev): |
| """ |
| Disable radvd for non-primary VR |
| """ |
| if dev == '': |
| return |
| CsHelper.service("radvd", "stop") |
| CsHelper.service("radvd", "disable") |
| logging.info(CsHelper.execute("systemctl status radvd")) |
| |
| def _add_ipv6_guest_gateway(self): |
| """ |
| Configure guest network gateway as IPv6 address for guest interface |
| for redundant primary VR |
| """ |
| for interface in self.address.get_interfaces(): |
| if not interface.is_guest() or not interface.get_gateway6_cidr(): |
| continue |
| self._add_ipv6_to_interface(interface, interface.get_gateway6_cidr()) |
| self._enable_radvd(interface.get_device()) |
| |
| def _remove_ipv6_guest_gateway(self): |
| """ |
| Remove guest network gateway as IPv6 address for guest interface |
| for redundant backup VR |
| """ |
| for interface in self.address.get_interfaces(): |
| if not interface.is_guest() or not interface.get_gateway6_cidr(): |
| continue |
| self._remove_ipv6_to_interface(interface, interface.get_gateway6_cidr()) |
| self._disable_radvd(interface.get_device()) |
