| // Copyright Istio Authors |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| package capture |
| |
| import ( |
| "bufio" |
| "fmt" |
| "net" |
| "os" |
| "strings" |
| "time" |
| ) |
| |
| import ( |
| "github.com/vishvananda/netlink" |
| "istio.io/pkg/log" |
| ) |
| |
| import ( |
| "github.com/apache/dubbo-go-pixiu/tools/istio-iptables/pkg/builder" |
| "github.com/apache/dubbo-go-pixiu/tools/istio-iptables/pkg/config" |
| "github.com/apache/dubbo-go-pixiu/tools/istio-iptables/pkg/constants" |
| dep "github.com/apache/dubbo-go-pixiu/tools/istio-iptables/pkg/dependencies" |
| iptableslog "github.com/apache/dubbo-go-pixiu/tools/istio-iptables/pkg/log" |
| ) |
| |
| type Ops int |
| |
| const ( |
| // AppendOps performs append operations of rules |
| AppendOps Ops = iota |
| // DeleteOps performs delete operations of rules |
| DeleteOps |
| |
| // In TPROXY mode, mark the packet from envoy outbound to app by podIP, |
| // this is to prevent it being intercepted to envoy inbound listener. |
| outboundMark = "1338" |
| ) |
| |
| var opsToString = map[Ops]string{ |
| AppendOps: "-A", |
| DeleteOps: "-D", |
| } |
| |
| type IptablesConfigurator struct { |
| iptables *builder.IptablesBuilder |
| // TODO(abhide): Fix dep.Dependencies with better interface |
| ext dep.Dependencies |
| cfg *config.Config |
| } |
| |
| func NewIptablesConfigurator(cfg *config.Config, ext dep.Dependencies) *IptablesConfigurator { |
| return &IptablesConfigurator{ |
| iptables: builder.NewIptablesBuilder(cfg), |
| ext: ext, |
| cfg: cfg, |
| } |
| } |
| |
| type NetworkRange struct { |
| IsWildcard bool |
| IPNets []*net.IPNet |
| HasLoopBackIP bool |
| } |
| |
| func split(s string) []string { |
| return config.Split(s) |
| } |
| |
| func (cfg *IptablesConfigurator) separateV4V6(cidrList string) (NetworkRange, NetworkRange, error) { |
| if cidrList == "*" { |
| return NetworkRange{IsWildcard: true}, NetworkRange{IsWildcard: true}, nil |
| } |
| ipv6Ranges := NetworkRange{} |
| ipv4Ranges := NetworkRange{} |
| for _, ipRange := range split(cidrList) { |
| ip, ipNet, err := net.ParseCIDR(ipRange) |
| if err != nil { |
| _, err = fmt.Fprintf(os.Stderr, "Ignoring error for bug compatibility with istio-iptables: %s\n", err.Error()) |
| if err != nil { |
| return ipv4Ranges, ipv6Ranges, err |
| } |
| continue |
| } |
| if ip.To4() != nil { |
| ipv4Ranges.IPNets = append(ipv4Ranges.IPNets, ipNet) |
| if ip.IsLoopback() { |
| ipv4Ranges.HasLoopBackIP = true |
| } |
| } else { |
| ipv6Ranges.IPNets = append(ipv6Ranges.IPNets, ipNet) |
| if ip.IsLoopback() { |
| ipv6Ranges.HasLoopBackIP = true |
| } |
| } |
| } |
| return ipv4Ranges, ipv6Ranges, nil |
| } |
| |
| func (cfg *IptablesConfigurator) logConfig() { |
| // Dump out our environment for debugging purposes. |
| var b strings.Builder |
| b.WriteString(fmt.Sprintf("ENVOY_PORT=%s\n", os.Getenv("ENVOY_PORT"))) |
| b.WriteString(fmt.Sprintf("INBOUND_CAPTURE_PORT=%s\n", os.Getenv("INBOUND_CAPTURE_PORT"))) |
| b.WriteString(fmt.Sprintf("ISTIO_INBOUND_INTERCEPTION_MODE=%s\n", os.Getenv("ISTIO_INBOUND_INTERCEPTION_MODE"))) |
| b.WriteString(fmt.Sprintf("ISTIO_INBOUND_TPROXY_ROUTE_TABLE=%s\n", os.Getenv("ISTIO_INBOUND_TPROXY_ROUTE_TABLE"))) |
| b.WriteString(fmt.Sprintf("ISTIO_INBOUND_PORTS=%s\n", os.Getenv("ISTIO_INBOUND_PORTS"))) |
| b.WriteString(fmt.Sprintf("ISTIO_OUTBOUND_PORTS=%s\n", os.Getenv("ISTIO_OUTBOUND_PORTS"))) |
| b.WriteString(fmt.Sprintf("ISTIO_LOCAL_EXCLUDE_PORTS=%s\n", os.Getenv("ISTIO_LOCAL_EXCLUDE_PORTS"))) |
| b.WriteString(fmt.Sprintf("ISTIO_EXCLUDE_INTERFACES=%s\n", os.Getenv("ISTIO_EXCLUDE_INTERFACES"))) |
| b.WriteString(fmt.Sprintf("ISTIO_SERVICE_CIDR=%s\n", os.Getenv("ISTIO_SERVICE_CIDR"))) |
| b.WriteString(fmt.Sprintf("ISTIO_SERVICE_EXCLUDE_CIDR=%s\n", os.Getenv("ISTIO_SERVICE_EXCLUDE_CIDR"))) |
| b.WriteString(fmt.Sprintf("ISTIO_META_DNS_CAPTURE=%s\n", os.Getenv("ISTIO_META_DNS_CAPTURE"))) |
| b.WriteString(fmt.Sprintf("INVALID_DROP=%s\n", os.Getenv("INVALID_DROP"))) |
| log.Infof("Istio iptables environment:\n%s", b.String()) |
| cfg.cfg.Print() |
| } |
| |
| func (cfg *IptablesConfigurator) handleInboundPortsInclude() { |
| // Handling of inbound ports. Traffic will be redirected to Envoy, which will process and forward |
| // to the local service. If not set, no inbound port will be intercepted by istio iptablesOrFail. |
| var table string |
| if cfg.cfg.InboundPortsInclude != "" { |
| if cfg.cfg.InboundInterceptionMode == constants.TPROXY { |
| // When using TPROXY, create a new chain for routing all inbound traffic to |
| // Envoy. Any packet entering this chain gets marked with the ${INBOUND_TPROXY_MARK} mark, |
| // so that they get routed to the loopback interface in order to get redirected to Envoy. |
| // In the ISTIOINBOUND chain, '-j ISTIODIVERT' reroutes to the loopback |
| // interface. |
| // Mark all inbound packets. |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.ISTIODIVERT, constants.MANGLE, "-j", constants.MARK, "--set-mark", |
| cfg.cfg.InboundTProxyMark) |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.ISTIODIVERT, constants.MANGLE, "-j", constants.ACCEPT) |
| |
| // Create a new chain for redirecting inbound traffic to the common Envoy |
| // port. |
| // In the ISTIOINBOUND chain, '-j RETURN' bypasses Envoy and |
| // '-j ISTIOTPROXY' redirects to Envoy. |
| cfg.iptables.AppendVersionedRule("127.0.0.1/32", "::1/128", iptableslog.UndefinedCommand, |
| constants.ISTIOTPROXY, constants.MANGLE, "!", "-d", constants.IPVersionSpecific, |
| "-p", constants.TCP, "-j", constants.TPROXY, |
| "--tproxy-mark", cfg.cfg.InboundTProxyMark+"/0xffffffff", "--on-port", cfg.cfg.InboundCapturePort) |
| table = constants.MANGLE |
| } else { |
| table = constants.NAT |
| } |
| cfg.iptables.AppendRule(iptableslog.JumpInbound, constants.PREROUTING, table, "-p", constants.TCP, |
| "-j", constants.ISTIOINBOUND) |
| |
| if cfg.cfg.InboundPortsInclude == "*" { |
| // Apply any user-specified port exclusions. |
| if cfg.cfg.InboundPortsExclude != "" { |
| for _, port := range split(cfg.cfg.InboundPortsExclude) { |
| cfg.iptables.AppendRule(iptableslog.ExcludeInboundPort, constants.ISTIOINBOUND, table, "-p", constants.TCP, |
| "--dport", port, "-j", constants.RETURN) |
| } |
| } |
| // Redirect remaining inbound traffic to Envoy. |
| if cfg.cfg.InboundInterceptionMode == constants.TPROXY { |
| // If an inbound packet belongs to an established socket, route it to the |
| // loopback interface. |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.ISTIOINBOUND, constants.MANGLE, "-p", constants.TCP, |
| "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", constants.ISTIODIVERT) |
| // Otherwise, it's a new connection. Redirect it using TPROXY. |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.ISTIOINBOUND, constants.MANGLE, "-p", constants.TCP, |
| "-j", constants.ISTIOTPROXY) |
| } else { |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.ISTIOINBOUND, constants.NAT, "-p", constants.TCP, |
| "-j", constants.ISTIOINREDIRECT) |
| } |
| } else { |
| // User has specified a non-empty list of ports to be redirected to Envoy. |
| for _, port := range split(cfg.cfg.InboundPortsInclude) { |
| if cfg.cfg.InboundInterceptionMode == constants.TPROXY { |
| cfg.iptables.AppendRule(iptableslog.IncludeInboundPort, constants.ISTIOINBOUND, constants.MANGLE, "-p", constants.TCP, |
| "--dport", port, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", constants.ISTIODIVERT) |
| cfg.iptables.AppendRule(iptableslog.IncludeInboundPort, |
| constants.ISTIOINBOUND, constants.MANGLE, "-p", constants.TCP, "--dport", port, "-j", constants.ISTIOTPROXY) |
| } else { |
| cfg.iptables.AppendRule(iptableslog.IncludeInboundPort, |
| constants.ISTIOINBOUND, constants.NAT, "-p", constants.TCP, "--dport", port, "-j", constants.ISTIOINREDIRECT) |
| } |
| } |
| } |
| } |
| } |
| |
| func (cfg *IptablesConfigurator) handleOutboundIncludeRules( |
| rangeInclude NetworkRange, |
| appendRule func(command iptableslog.Command, chain string, table string, params ...string) *builder.IptablesBuilder, |
| insert func(command iptableslog.Command, chain string, table string, position int, params ...string) *builder.IptablesBuilder) { |
| // Apply outbound IP inclusions. |
| if rangeInclude.IsWildcard { |
| // Wildcard specified. Redirect all remaining outbound traffic to Envoy. |
| appendRule(iptableslog.UndefinedCommand, constants.ISTIOOUTPUT, constants.NAT, "-j", constants.ISTIOREDIRECT) |
| for _, internalInterface := range split(cfg.cfg.KubeVirtInterfaces) { |
| insert(iptableslog.KubevirtCommand, |
| constants.PREROUTING, constants.NAT, 1, "-i", internalInterface, "-j", constants.ISTIOREDIRECT) |
| } |
| } else if len(rangeInclude.IPNets) > 0 { |
| // User has specified a non-empty list of cidrs to be redirected to Envoy. |
| for _, cidr := range rangeInclude.IPNets { |
| for _, internalInterface := range split(cfg.cfg.KubeVirtInterfaces) { |
| insert(iptableslog.KubevirtCommand, constants.PREROUTING, constants.NAT, 1, "-i", internalInterface, |
| "-d", cidr.String(), "-j", constants.ISTIOREDIRECT) |
| } |
| appendRule(iptableslog.UndefinedCommand, |
| constants.ISTIOOUTPUT, constants.NAT, "-d", cidr.String(), "-j", constants.ISTIOREDIRECT) |
| } |
| // All other traffic is not redirected. |
| appendRule(iptableslog.UndefinedCommand, constants.ISTIOOUTPUT, constants.NAT, "-j", constants.RETURN) |
| } |
| } |
| |
| func (cfg *IptablesConfigurator) shortCircuitKubeInternalInterface() { |
| for _, internalInterface := range split(cfg.cfg.KubeVirtInterfaces) { |
| cfg.iptables.InsertRule(iptableslog.KubevirtCommand, constants.PREROUTING, constants.NAT, 1, "-i", internalInterface, "-j", constants.RETURN) |
| } |
| } |
| |
| func (cfg *IptablesConfigurator) shortCircuitExcludeInterfaces() { |
| for _, excludeInterface := range split(cfg.cfg.ExcludeInterfaces) { |
| cfg.iptables.AppendRule( |
| iptableslog.ExcludeInterfaceCommand, constants.PREROUTING, constants.NAT, "-i", excludeInterface, "-j", constants.RETURN) |
| cfg.iptables.AppendRule(iptableslog.ExcludeInterfaceCommand, constants.OUTPUT, constants.NAT, "-o", excludeInterface, "-j", constants.RETURN) |
| } |
| if cfg.cfg.InboundInterceptionMode == constants.TPROXY { |
| for _, excludeInterface := range split(cfg.cfg.ExcludeInterfaces) { |
| |
| cfg.iptables.AppendRule( |
| iptableslog.ExcludeInterfaceCommand, constants.PREROUTING, constants.MANGLE, "-i", excludeInterface, "-j", constants.RETURN) |
| cfg.iptables.AppendRule(iptableslog.ExcludeInterfaceCommand, constants.OUTPUT, constants.MANGLE, "-o", excludeInterface, "-j", constants.RETURN) |
| } |
| } |
| } |
| |
| func ignoreExists(err error) error { |
| if err == nil { |
| return nil |
| } |
| if strings.Contains(strings.ToLower(err.Error()), "file exists") { |
| return nil |
| } |
| return err |
| } |
| |
| func SplitV4V6(ips []string) (ipv4 []string, ipv6 []string) { |
| for _, i := range ips { |
| parsed := net.ParseIP(i) |
| if parsed.To4() != nil { |
| ipv4 = append(ipv4, i) |
| } else { |
| ipv6 = append(ipv6, i) |
| } |
| } |
| return |
| } |
| |
| func ConfigureRoutes(cfg *config.Config, ext dep.Dependencies) error { |
| if cfg.DryRun { |
| log.Infof("skipping configuring routes due to dry run mode") |
| return nil |
| } |
| if ext != nil && cfg.CNIMode { |
| command := os.Args[0] |
| return ext.Run(command, constants.CommandConfigureRoutes) |
| } |
| if err := configureIPv6Addresses(cfg); err != nil { |
| return err |
| } |
| if err := configureTProxyRoutes(cfg); err != nil { |
| return err |
| } |
| return nil |
| } |
| |
| // configureIPv6Addresses sets up a new IP address on local interface. This is used as the source IP |
| // for inbound traffic to distinguish traffic we want to capture vs traffic we do not. This is needed |
| // for IPv6 but not IPv4, as IPv4 defaults to `netmask 255.0.0.0`, which allows binding to addresses |
| // in the 127.x.y.z range, while IPv6 defaults to `prefixlen 128` which allows binding only to ::1. |
| // Equivalent to `ip -6 addr add "::6/128" dev lo` |
| func configureIPv6Addresses(cfg *config.Config) error { |
| if !cfg.EnableInboundIPv6 { |
| return nil |
| } |
| link, err := netlink.LinkByName("lo") |
| if err != nil { |
| return fmt.Errorf("failed to find 'lo' link: %v", err) |
| } |
| // Setup a new IP address on local interface. This is used as the source IP for inbound traffic |
| // to distinguish traffic we want to capture vs traffic we do not. |
| // Equivalent to `ip -6 addr add "::6/128" dev lo` |
| address := &net.IPNet{IP: net.ParseIP("::6"), Mask: net.CIDRMask(128, 128)} |
| addr := &netlink.Addr{IPNet: address} |
| |
| err = netlink.AddrAdd(link, addr) |
| if ignoreExists(err) != nil { |
| return fmt.Errorf("failed to add IPv6 inbound address: %v", err) |
| } |
| log.Infof("Added ::6 address") |
| return nil |
| } |
| |
| func (cfg *IptablesConfigurator) Run() { |
| defer func() { |
| // Best effort since we don't know if the commands exist |
| _ = cfg.ext.Run(constants.IPTABLESSAVE) |
| if cfg.cfg.EnableInboundIPv6 { |
| _ = cfg.ext.Run(constants.IP6TABLESSAVE) |
| } |
| }() |
| |
| // Since OUTBOUND_IP_RANGES_EXCLUDE could carry ipv4 and ipv6 ranges |
| // need to split them in different arrays one for ipv4 and one for ipv6 |
| // in order to not to fail |
| ipv4RangesExclude, ipv6RangesExclude, err := cfg.separateV4V6(cfg.cfg.OutboundIPRangesExclude) |
| if err != nil { |
| panic(err) |
| } |
| if ipv4RangesExclude.IsWildcard { |
| panic("Invalid value for OUTBOUND_IP_RANGES_EXCLUDE") |
| } |
| // FixMe: Do we need similar check for ipv6RangesExclude as well ?? |
| |
| ipv4RangesInclude, ipv6RangesInclude, err := cfg.separateV4V6(cfg.cfg.OutboundIPRangesInclude) |
| if err != nil { |
| panic(err) |
| } |
| |
| redirectDNS := cfg.cfg.RedirectDNS |
| cfg.logConfig() |
| |
| cfg.shortCircuitExcludeInterfaces() |
| |
| // Do not capture internal interface. |
| cfg.shortCircuitKubeInternalInterface() |
| |
| // Create a rule for invalid drop in PREROUTING chain in mangle table, so the iptables will drop the out of window packets instead of reset connection . |
| dropInvalid := cfg.cfg.DropInvalid |
| if dropInvalid { |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.PREROUTING, constants.MANGLE, "-m", "conntrack", "--ctstate", |
| "INVALID", "-j", constants.DROP) |
| } |
| |
| // Create a new chain for to hit tunnel port directly. Envoy will be listening on port acting as VPN tunnel. |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.ISTIOINBOUND, constants.NAT, "-p", constants.TCP, "--dport", |
| cfg.cfg.InboundTunnelPort, "-j", constants.RETURN) |
| |
| // Create a new chain for redirecting outbound traffic to the common Envoy port. |
| // In both chains, '-j RETURN' bypasses Envoy and '-j ISTIOREDIRECT' |
| // redirects to Envoy. |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, |
| constants.ISTIOREDIRECT, constants.NAT, "-p", constants.TCP, "-j", constants.REDIRECT, "--to-ports", cfg.cfg.ProxyPort) |
| |
| // Use this chain also for redirecting inbound traffic to the common Envoy port |
| // when not using TPROXY. |
| |
| cfg.iptables.AppendRule(iptableslog.InboundCapture, constants.ISTIOINREDIRECT, constants.NAT, "-p", constants.TCP, "-j", constants.REDIRECT, |
| "--to-ports", cfg.cfg.InboundCapturePort) |
| |
| cfg.handleInboundPortsInclude() |
| |
| // TODO: change the default behavior to not intercept any output - user may use http_proxy or another |
| // iptablesOrFail wrapper (like ufw). Current default is similar with 0.1 |
| // Jump to the ISTIOOUTPUT chain from OUTPUT chain for all tcp traffic, and UDP dns (if enabled) |
| cfg.iptables.AppendRule(iptableslog.JumpOutbound, constants.OUTPUT, constants.NAT, "-p", constants.TCP, "-j", constants.ISTIOOUTPUT) |
| // Apply port based exclusions. Must be applied before connections back to self are redirected. |
| if cfg.cfg.OutboundPortsExclude != "" { |
| for _, port := range split(cfg.cfg.OutboundPortsExclude) { |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.ISTIOOUTPUT, constants.NAT, "-p", constants.TCP, |
| "--dport", port, "-j", constants.RETURN) |
| } |
| } |
| |
| // 127.0.0.6/::7 is bind connect from inbound passthrough cluster |
| cfg.iptables.AppendVersionedRule("127.0.0.6/32", "::6/128", iptableslog.UndefinedCommand, constants.ISTIOOUTPUT, constants.NAT, |
| "-o", "lo", "-s", constants.IPVersionSpecific, "-j", constants.RETURN) |
| |
| for _, uid := range split(cfg.cfg.ProxyUID) { |
| // Redirect app calls back to itself via Envoy when using the service VIP |
| // e.g. appN => Envoy (client) => Envoy (server) => appN. |
| // nolint: lll |
| if redirectDNS { |
| // When DNS is enabled, we skip this for port 53. This ensures we do not have: |
| // app => istio-agent => Envoy inbound => dns server |
| // Instead, we just have: |
| // app => istio-agent => dns server |
| cfg.iptables.AppendVersionedRule("127.0.0.1/32", "::1/128", iptableslog.UndefinedCommand, constants.ISTIOOUTPUT, constants.NAT, |
| "-o", "lo", "!", "-d", constants.IPVersionSpecific, |
| "-p", "tcp", "!", "--dport", "53", |
| "-m", "owner", "--uid-owner", uid, "-j", constants.ISTIOINREDIRECT) |
| } else { |
| cfg.iptables.AppendVersionedRule("127.0.0.1/32", "::1/128", iptableslog.UndefinedCommand, constants.ISTIOOUTPUT, constants.NAT, |
| "-o", "lo", "!", "-d", constants.IPVersionSpecific, |
| "-m", "owner", "--uid-owner", uid, "-j", constants.ISTIOINREDIRECT) |
| } |
| |
| // Do not redirect app calls to back itself via Envoy when using the endpoint address |
| // e.g. appN => appN by lo |
| // If loopback explicitly set via OutboundIPRangesInclude, then don't return. |
| if !ipv4RangesInclude.HasLoopBackIP && !ipv6RangesInclude.HasLoopBackIP { |
| if redirectDNS { |
| // Users may have a DNS server that is on localhost. In these cases, applications may |
| // send TCP traffic to the DNS server that we actually *do* want to intercept. To |
| // handle this case, we exclude port 53 from this rule. Note: We cannot just move the |
| // port 53 redirection rule further up the list, as we will want to avoid capturing |
| // DNS requests from the proxy UID/GID |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.ISTIOOUTPUT, constants.NAT, "-o", "lo", "-p", "tcp", |
| "!", "--dport", "53", |
| "-m", "owner", "!", "--uid-owner", uid, "-j", constants.RETURN) |
| } else { |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.ISTIOOUTPUT, constants.NAT, |
| "-o", "lo", "-m", "owner", "!", "--uid-owner", uid, "-j", constants.RETURN) |
| } |
| } |
| |
| // Avoid infinite loops. Don't redirect Envoy traffic directly back to |
| // Envoy for non-loopback traffic. |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.ISTIOOUTPUT, constants.NAT, |
| "-m", "owner", "--uid-owner", uid, "-j", constants.RETURN) |
| } |
| |
| for _, gid := range split(cfg.cfg.ProxyGID) { |
| // Redirect app calls back to itself via Envoy when using the service VIP |
| // e.g. appN => Envoy (client) => Envoy (server) => appN. |
| cfg.iptables.AppendVersionedRule("127.0.0.1/32", "::1/128", iptableslog.UndefinedCommand, constants.ISTIOOUTPUT, constants.NAT, |
| "-o", "lo", "!", "-d", constants.IPVersionSpecific, |
| "-m", "owner", "--gid-owner", gid, "-j", constants.ISTIOINREDIRECT) |
| |
| // Do not redirect app calls to back itself via Envoy when using the endpoint address |
| // e.g. appN => appN by lo |
| // If loopback explicitly set via OutboundIPRangesInclude, then don't return. |
| if !ipv4RangesInclude.HasLoopBackIP && !ipv6RangesInclude.HasLoopBackIP { |
| if redirectDNS { |
| // Users may have a DNS server that is on localhost. In these cases, applications may |
| // send TCP traffic to the DNS server that we actually *do* want to intercept. To |
| // handle this case, we exclude port 53 from this rule. Note: We cannot just move the |
| // port 53 redirection rule further up the list, as we will want to avoid capturing |
| // DNS requests from the proxy UID/GID |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.ISTIOOUTPUT, constants.NAT, |
| "-o", "lo", "-p", "tcp", |
| "!", "--dport", "53", |
| "-m", "owner", "!", "--gid-owner", gid, "-j", constants.RETURN) |
| } else { |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.ISTIOOUTPUT, constants.NAT, |
| "-o", "lo", "-m", "owner", "!", "--gid-owner", gid, "-j", constants.RETURN) |
| } |
| } |
| |
| // Avoid infinite loops. Don't redirect Envoy traffic directly back to |
| // Envoy for non-loopback traffic. |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.ISTIOOUTPUT, constants.NAT, "-m", "owner", "--gid-owner", gid, "-j", constants.RETURN) |
| } |
| |
| ownerGroupsFilter := config.ParseInterceptFilter(cfg.cfg.OwnerGroupsInclude, cfg.cfg.OwnerGroupsExclude) |
| |
| cfg.handleCaptureByOwnerGroup(ownerGroupsFilter) |
| |
| if redirectDNS { |
| if cfg.cfg.CaptureAllDNS { |
| // Redirect all TCP dns traffic on port 53 to the agent on port 15053 |
| // This will be useful for the CNI case where pod DNS server address cannot be decided. |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, |
| constants.ISTIOOUTPUT, constants.NAT, |
| "-p", constants.TCP, |
| "--dport", "53", |
| "-j", constants.REDIRECT, |
| "--to-ports", constants.IstioAgentDNSListenerPort) |
| } else { |
| for _, s := range cfg.cfg.DNSServersV4 { |
| // redirect all TCP dns traffic on port 53 to the agent on port 15053 for all servers |
| // in etc/resolv.conf |
| // We avoid redirecting all IP ranges to avoid infinite loops when there are local DNS proxies |
| // such as: app -> istio dns server -> dnsmasq -> upstream |
| // This ensures that we do not get requests from dnsmasq sent back to the agent dns server in a loop. |
| // Note: If a user somehow configured etc/resolv.conf to point to dnsmasq and server X, and dnsmasq also |
| // pointed to server X, this would not work. However, the assumption is that is not a common case. |
| cfg.iptables.AppendRuleV4(iptableslog.UndefinedCommand, |
| constants.ISTIOOUTPUT, constants.NAT, |
| "-p", constants.TCP, |
| "--dport", "53", |
| "-d", s+"/32", |
| "-j", constants.REDIRECT, |
| "--to-ports", constants.IstioAgentDNSListenerPort) |
| } |
| for _, s := range cfg.cfg.DNSServersV6 { |
| cfg.iptables.AppendRuleV6(iptableslog.UndefinedCommand, |
| constants.ISTIOOUTPUT, constants.NAT, |
| "-p", constants.TCP, |
| "--dport", "53", |
| "-d", s+"/128", |
| "-j", constants.REDIRECT, |
| "--to-ports", constants.IstioAgentDNSListenerPort) |
| } |
| } |
| } |
| |
| // Skip redirection for Envoy-aware applications and |
| // container-to-container traffic both of which explicitly use |
| // localhost. |
| cfg.iptables.AppendVersionedRule("127.0.0.1/32", "::1/128", iptableslog.UndefinedCommand, constants.ISTIOOUTPUT, constants.NAT, |
| "-d", constants.IPVersionSpecific, "-j", constants.RETURN) |
| // Apply outbound IPv4 exclusions. Must be applied before inclusions. |
| for _, cidr := range ipv4RangesExclude.IPNets { |
| cfg.iptables.AppendRuleV4(iptableslog.UndefinedCommand, constants.ISTIOOUTPUT, constants.NAT, "-d", cidr.String(), "-j", constants.RETURN) |
| } |
| for _, cidr := range ipv6RangesExclude.IPNets { |
| cfg.iptables.AppendRuleV6(iptableslog.UndefinedCommand, constants.ISTIOOUTPUT, constants.NAT, "-d", cidr.String(), "-j", constants.RETURN) |
| } |
| |
| cfg.handleOutboundPortsInclude() |
| |
| cfg.handleOutboundIncludeRules(ipv4RangesInclude, cfg.iptables.AppendRuleV4, cfg.iptables.InsertRuleV4) |
| cfg.handleOutboundIncludeRules(ipv6RangesInclude, cfg.iptables.AppendRuleV6, cfg.iptables.InsertRuleV6) |
| |
| if redirectDNS { |
| HandleDNSUDP( |
| AppendOps, cfg.iptables, cfg.ext, "", |
| cfg.cfg.ProxyUID, cfg.cfg.ProxyGID, |
| cfg.cfg.DNSServersV4, cfg.cfg.DNSServersV6, cfg.cfg.CaptureAllDNS, |
| ownerGroupsFilter) |
| } |
| |
| if cfg.cfg.InboundInterceptionMode == constants.TPROXY { |
| // save packet mark set by envoy.filters.listener.original_src as connection mark |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.PREROUTING, constants.MANGLE, |
| "-p", constants.TCP, "-m", "mark", "--mark", cfg.cfg.InboundTProxyMark, "-j", "CONNMARK", "--save-mark") |
| // If the packet is already marked with 1337, then return. This is to prevent mark envoy --> app traffic again. |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.OUTPUT, constants.MANGLE, |
| "-p", constants.TCP, "-o", "lo", "-m", "mark", "--mark", cfg.cfg.InboundTProxyMark, "-j", constants.RETURN) |
| for _, uid := range split(cfg.cfg.ProxyUID) { |
| // mark outgoing packets from envoy to workload by pod ip |
| // app call VIP --> envoy outbound -(mark 1338)-> envoy inbound --> app |
| cfg.iptables.AppendVersionedRule("127.0.0.1/32", "::1/128", iptableslog.UndefinedCommand, constants.OUTPUT, constants.MANGLE, |
| "!", "-d", constants.IPVersionSpecific, "-p", constants.TCP, "-o", "lo", |
| "-m", "owner", "--uid-owner", uid, "-j", constants.MARK, "--set-mark", outboundMark) |
| } |
| for _, gid := range split(cfg.cfg.ProxyGID) { |
| // mark outgoing packets from envoy to workload by pod ip |
| // app call VIP --> envoy outbound -(mark 1338)-> envoy inbound --> app |
| cfg.iptables.AppendVersionedRule("127.0.0.1/32", "::1/128", iptableslog.UndefinedCommand, constants.OUTPUT, constants.MANGLE, |
| "!", "-d", constants.IPVersionSpecific, "-p", constants.TCP, "-o", "lo", |
| "-m", "owner", "--gid-owner", gid, "-j", constants.MARK, "--set-mark", outboundMark) |
| } |
| // mark outgoing packets from workload, match it to policy routing entry setup for TPROXY mode |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.OUTPUT, constants.MANGLE, |
| "-p", constants.TCP, "-m", "connmark", "--mark", cfg.cfg.InboundTProxyMark, "-j", "CONNMARK", "--restore-mark") |
| // prevent infinite redirect |
| cfg.iptables.InsertRule(iptableslog.UndefinedCommand, constants.ISTIOINBOUND, constants.MANGLE, 1, |
| "-p", constants.TCP, "-m", "mark", "--mark", cfg.cfg.InboundTProxyMark, "-j", constants.RETURN) |
| // prevent intercept traffic from envoy/pilot-agent ==> app by 127.0.0.6 --> podip |
| cfg.iptables.InsertRuleV4(iptableslog.UndefinedCommand, constants.ISTIOINBOUND, constants.MANGLE, 2, |
| "-p", constants.TCP, "-s", "127.0.0.6/32", "-i", "lo", "-j", constants.RETURN) |
| cfg.iptables.InsertRuleV6(iptableslog.UndefinedCommand, constants.ISTIOINBOUND, constants.MANGLE, 2, |
| "-p", constants.TCP, "-s", "::6/128", "-i", "lo", "-j", constants.RETURN) |
| // prevent intercept traffic from app ==> app by pod ip |
| cfg.iptables.InsertRule(iptableslog.UndefinedCommand, constants.ISTIOINBOUND, constants.MANGLE, 3, |
| "-p", constants.TCP, "-i", "lo", "-m", "mark", "!", "--mark", outboundMark, "-j", constants.RETURN) |
| } |
| cfg.executeCommands() |
| } |
| |
| type UDPRuleApplier struct { |
| iptables *builder.IptablesBuilder |
| ext dep.Dependencies |
| ops Ops |
| table string |
| chain string |
| cmd string |
| } |
| |
| func (f UDPRuleApplier) RunV4(args ...string) { |
| switch f.ops { |
| case AppendOps: |
| f.iptables.AppendRuleV4(iptableslog.UndefinedCommand, f.chain, f.table, args...) |
| case DeleteOps: |
| deleteArgs := []string{"-t", f.table, opsToString[f.ops], f.chain} |
| deleteArgs = append(deleteArgs, args...) |
| f.ext.RunQuietlyAndIgnore(f.cmd, deleteArgs...) |
| } |
| } |
| |
| func (f UDPRuleApplier) RunV6(args ...string) { |
| switch f.ops { |
| case AppendOps: |
| f.iptables.AppendRuleV6(iptableslog.UndefinedCommand, f.chain, f.table, args...) |
| case DeleteOps: |
| deleteArgs := []string{"-t", f.table, opsToString[f.ops], f.chain} |
| deleteArgs = append(deleteArgs, args...) |
| f.ext.RunQuietlyAndIgnore(f.cmd, deleteArgs...) |
| } |
| } |
| |
| func (f UDPRuleApplier) Run(args ...string) { |
| f.RunV4(args...) |
| f.RunV6(args...) |
| } |
| |
| func (f UDPRuleApplier) WithChain(chain string) UDPRuleApplier { |
| f.chain = chain |
| return f |
| } |
| |
| func (f UDPRuleApplier) WithTable(table string) UDPRuleApplier { |
| f.table = table |
| return f |
| } |
| |
| // HandleDNSUDP is a helper function to tackle with DNS UDP specific operations. |
| // This helps the creation logic of DNS UDP rules in sync with the deletion. |
| func HandleDNSUDP( |
| ops Ops, iptables *builder.IptablesBuilder, ext dep.Dependencies, |
| cmd, proxyUID, proxyGID string, dnsServersV4 []string, dnsServersV6 []string, captureAllDNS bool, |
| ownerGroupsFilter config.InterceptFilter) { |
| f := UDPRuleApplier{ |
| iptables: iptables, |
| ext: ext, |
| ops: ops, |
| table: constants.NAT, |
| chain: constants.OUTPUT, |
| cmd: cmd, |
| } |
| // Make sure that upstream DNS requests from agent/envoy dont get captured. |
| // TODO: add ip6 as well |
| for _, uid := range split(proxyUID) { |
| f.Run("-p", "udp", "--dport", "53", "-m", "owner", "--uid-owner", uid, "-j", constants.RETURN) |
| } |
| for _, gid := range split(proxyGID) { |
| f.Run("-p", "udp", "--dport", "53", "-m", "owner", "--gid-owner", gid, "-j", constants.RETURN) |
| } |
| |
| if ownerGroupsFilter.Except { |
| for _, group := range ownerGroupsFilter.Values { |
| f.Run("-p", "udp", "--dport", "53", "-m", "owner", "--gid-owner", group, "-j", constants.RETURN) |
| } |
| } else { |
| groupIsNoneOf := CombineMatchers(ownerGroupsFilter.Values, func(group string) []string { |
| return []string{"-m", "owner", "!", "--gid-owner", group} |
| }) |
| f.Run(Flatten([]string{"-p", "udp", "--dport", "53"}, groupIsNoneOf, []string{"-j", constants.RETURN})...) |
| } |
| |
| if captureAllDNS { |
| // Redirect all TCP dns traffic on port 53 to the agent on port 15053 |
| // This will be useful for the CNI case where pod DNS server address cannot be decided. |
| f.Run("-p", "udp", "--dport", "53", "-j", constants.REDIRECT, "--to-port", constants.IstioAgentDNSListenerPort) |
| } else { |
| // redirect all TCP dns traffic on port 53 to the agent on port 15053 for all servers |
| // in etc/resolv.conf |
| // We avoid redirecting all IP ranges to avoid infinite loops when there are local DNS proxies |
| // such as: app -> istio dns server -> dnsmasq -> upstream |
| // This ensures that we do not get requests from dnsmasq sent back to the agent dns server in a loop. |
| // Note: If a user somehow configured etc/resolv.conf to point to dnsmasq and server X, and dnsmasq also |
| // pointed to server X, this would not work. However, the assumption is that is not a common case. |
| for _, s := range dnsServersV4 { |
| f.RunV4("-p", "udp", "--dport", "53", "-d", s+"/32", |
| "-j", constants.REDIRECT, "--to-port", constants.IstioAgentDNSListenerPort) |
| } |
| for _, s := range dnsServersV6 { |
| f.RunV6("-p", "udp", "--dport", "53", "-d", s+"/128", |
| "-j", constants.REDIRECT, "--to-port", constants.IstioAgentDNSListenerPort) |
| } |
| } |
| // Split UDP DNS traffic to separate conntrack zones |
| addConntrackZoneDNSUDP(f.WithTable(constants.RAW), proxyUID, proxyGID, dnsServersV4, dnsServersV6, captureAllDNS) |
| } |
| |
| // addConntrackZoneDNSUDP is a helper function to add iptables rules to split DNS traffic |
| // in two separate conntrack zones to avoid issues with UDP conntrack race conditions. |
| // Traffic that goes from istio to DNS servers and vice versa are zone 1 and traffic from |
| // DNS client to istio and vice versa goes to zone 2 |
| func addConntrackZoneDNSUDP( |
| f UDPRuleApplier, proxyUID, proxyGID string, dnsServersV4 []string, dnsServersV6 []string, captureAllDNS bool) { |
| // TODO: add ip6 as well |
| for _, uid := range split(proxyUID) { |
| // Packets with dst port 53 from istio to zone 1. These are Istio calls to upstream resolvers |
| f.Run("-p", "udp", "--dport", "53", "-m", "owner", "--uid-owner", uid, "-j", constants.CT, "--zone", "1") |
| // Packets with src port 15053 from istio to zone 2. These are Istio response packets to application clients |
| f.Run("-p", "udp", "--sport", "15053", "-m", "owner", "--uid-owner", uid, "-j", constants.CT, "--zone", "2") |
| } |
| for _, gid := range split(proxyGID) { |
| // Packets with dst port 53 from istio to zone 1. These are Istio calls to upstream resolvers |
| f.Run("-p", "udp", "--dport", "53", "-m", "owner", "--gid-owner", gid, "-j", constants.CT, "--zone", "1") |
| // Packets with src port 15053 from istio to zone 2. These are Istio response packets to application clients |
| f.Run("-p", "udp", "--sport", "15053", "-m", "owner", "--gid-owner", gid, "-j", constants.CT, "--zone", "2") |
| |
| } |
| |
| if captureAllDNS { |
| // Not specifying destination address is useful for the CNI case where pod DNS server address cannot be decided. |
| |
| // Mark all UDP dns traffic with dst port 53 as zone 2. These are application client packets towards DNS resolvers. |
| f.Run("-p", "udp", "--dport", "53", |
| "-j", constants.CT, "--zone", "2") |
| // Mark all UDP dns traffic with src port 53 as zone 1. These are response packets from the DNS resolvers. |
| f.WithChain(constants.PREROUTING).Run("-p", "udp", "--sport", "53", |
| "-j", constants.CT, "--zone", "1") |
| } else { |
| // Go through all DNS servers in etc/resolv.conf and mark the packets based on these destination addresses. |
| for _, s := range dnsServersV4 { |
| // Mark all UDP dns traffic with dst port 53 as zone 2. These are application client packets towards DNS resolvers. |
| f.RunV4("-p", "udp", "--dport", "53", "-d", s+"/32", |
| "-j", constants.CT, "--zone", "2") |
| // Mark all UDP dns traffic with src port 53 as zone 1. These are response packets from the DNS resolvers. |
| f.WithChain(constants.PREROUTING).RunV4("-p", "udp", "--sport", "53", "-d", s+"/32", |
| "-j", constants.CT, "--zone", "1") |
| } |
| for _, s := range dnsServersV6 { |
| // Mark all UDP dns traffic with dst port 53 as zone 2. These are application client packets towards DNS resolvers. |
| f.RunV6("-p", "udp", "--dport", "53", "-d", s+"/128", |
| "-j", constants.CT, "--zone", "2") |
| // Mark all UDP dns traffic with src port 53 as zone 1. These are response packets from the DNS resolvers. |
| f.WithChain(constants.PREROUTING).RunV6("-p", "udp", "--sport", "53", "-d", s+"/128", |
| "-j", constants.CT, "--zone", "1") |
| } |
| } |
| } |
| |
| func (cfg *IptablesConfigurator) handleOutboundPortsInclude() { |
| if cfg.cfg.OutboundPortsInclude != "" { |
| for _, port := range split(cfg.cfg.OutboundPortsInclude) { |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, |
| constants.ISTIOOUTPUT, constants.NAT, "-p", constants.TCP, "--dport", port, "-j", constants.ISTIOREDIRECT) |
| } |
| } |
| } |
| |
| func (cfg *IptablesConfigurator) handleCaptureByOwnerGroup(filter config.InterceptFilter) { |
| if filter.Except { |
| for _, group := range filter.Values { |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.ISTIOOUTPUT, constants.NAT, |
| "-m", "owner", "--gid-owner", group, "-j", constants.RETURN) |
| } |
| } else { |
| groupIsNoneOf := CombineMatchers(filter.Values, func(group string) []string { |
| return []string{"-m", "owner", "!", "--gid-owner", group} |
| }) |
| cfg.iptables.AppendRule(iptableslog.UndefinedCommand, constants.ISTIOOUTPUT, constants.NAT, |
| append(groupIsNoneOf, "-j", constants.RETURN)...) |
| } |
| } |
| |
| func (cfg *IptablesConfigurator) createRulesFile(f *os.File, contents string) error { |
| defer f.Close() |
| log.Infof("Writing following contents to rules file: %v\n%v", f.Name(), strings.TrimSpace(contents)) |
| writer := bufio.NewWriter(f) |
| _, err := writer.WriteString(contents) |
| if err != nil { |
| return fmt.Errorf("unable to write iptables-restore file: %v", err) |
| } |
| err = writer.Flush() |
| return err |
| } |
| |
| func (cfg *IptablesConfigurator) executeIptablesCommands(commands [][]string) { |
| for _, cmd := range commands { |
| if len(cmd) > 1 { |
| cfg.ext.RunOrFail(cmd[0], cmd[1:]...) |
| } else { |
| cfg.ext.RunOrFail(cmd[0]) |
| } |
| } |
| } |
| |
| func (cfg *IptablesConfigurator) executeIptablesRestoreCommand(isIpv4 bool) error { |
| var data, filename, cmd string |
| if isIpv4 { |
| data = cfg.iptables.BuildV4Restore() |
| filename = fmt.Sprintf("iptables-rules-%d.txt", time.Now().UnixNano()) |
| cmd = constants.IPTABLESRESTORE |
| } else { |
| data = cfg.iptables.BuildV6Restore() |
| filename = fmt.Sprintf("ip6tables-rules-%d.txt", time.Now().UnixNano()) |
| cmd = constants.IP6TABLESRESTORE |
| } |
| var rulesFile *os.File |
| var err error |
| if cfg.cfg.OutputPath != "" { |
| // Print the iptables rules into the given output file. |
| rulesFile, err = os.OpenFile(cfg.cfg.OutputPath, os.O_CREATE|os.O_WRONLY, 0o644) |
| if err != nil { |
| return fmt.Errorf("unable to open iptables rules output file %v: %v", cfg.cfg.OutputPath, err) |
| } |
| } else { |
| // Otherwise create a temporary file to write iptables rules to, which will be cleaned up at the end. |
| rulesFile, err = os.CreateTemp("", filename) |
| if err != nil { |
| return fmt.Errorf("unable to create iptables-restore file: %v", err) |
| } |
| defer os.Remove(rulesFile.Name()) |
| } |
| if err := cfg.createRulesFile(rulesFile, data); err != nil { |
| return err |
| } |
| // --noflush to prevent flushing/deleting previous contents from table |
| cfg.ext.RunOrFail(cmd, "--noflush", rulesFile.Name()) |
| return nil |
| } |
| |
| func (cfg *IptablesConfigurator) executeCommands() { |
| if cfg.cfg.RestoreFormat { |
| // Execute iptables-restore |
| err := cfg.executeIptablesRestoreCommand(true) |
| if err != nil { |
| log.Errorf("Failed to execute iptables-restore command: %v", err) |
| os.Exit(1) |
| } |
| // Execute ip6tables-restore |
| err = cfg.executeIptablesRestoreCommand(false) |
| if err != nil { |
| log.Errorf("Failed to execute iptables-restore command: %v", err) |
| os.Exit(1) |
| } |
| } else { |
| // Execute iptables commands |
| cfg.executeIptablesCommands(cfg.iptables.BuildV4()) |
| // Execute ip6tables commands |
| cfg.executeIptablesCommands(cfg.iptables.BuildV6()) |
| |
| } |
| } |