blob: 30cf353cff9e480166a351aa5a4be6e165f13b83 [file] [log] [blame]
// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package envoy
import (
"errors"
"fmt"
"math"
"math/rand"
"os"
"strconv"
"strings"
"time"
)
import (
envoyBootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
"github.com/hashicorp/go-multierror"
"istio.io/pkg/log"
"sigs.k8s.io/yaml"
)
const (
// InvalidBaseID used to indicate that the Envoy BaseID has not been set. Attempting
// to Close this BaseID will have no effect.
InvalidBaseID = BaseID(math.MaxUint32)
)
// FlagName is the raw flag name passed to envoy.
type FlagName string
func (n FlagName) String() string {
return string(n)
}
// Option for an Envoy Instance.
type Option interface {
// FlagName returns the flag name used on the command line.
FlagName() FlagName
// FlagValue returns the flag value used on the command line. Can be empty for boolean flags.
FlagValue() string
apply(ctx *configContext)
validate(ctx *configContext) error
}
type Options []Option
// ToArgs creates the command line arguments for the list of options.
func (options Options) ToArgs() []string {
// Get the arguments for the command.
args := make([]string, 0, len(options)*2)
for _, o := range options {
name := o.FlagName()
if name != "" {
args = append(args, name.String())
value := o.FlagValue()
if value != "" {
args = append(args, value)
}
}
}
return args
}
// Validate the Options.
func (options Options) Validate() error {
return options.validate(newConfigContext())
}
// validate is an internal method for validation.
func (options Options) validate(ctx *configContext) error {
// Check for any duplicate user-specified options
if err := options.checkDuplicates(); err != nil {
return err
}
// Add a placeholder ConfigPath option. Can and should be overridden by user-provided value.
// Used for force config validation to ensure that either configPath or configYaml has been set.
opts := append(Options{ConfigPath("")}, options...)
// Apply the options to the context.
for _, o := range opts {
o.apply(ctx)
}
// Validate all of the options.
for _, o := range opts {
if err := o.validate(ctx); err != nil {
return err
}
}
return nil
}
// checkDuplicates ensures to make sure that there are no duplicate options.
func (options Options) checkDuplicates() error {
optionSet := make(map[FlagName]struct{})
for _, o := range options {
if _, ok := optionSet[o.FlagName()]; ok {
return fmt.Errorf("multiple options specified for %s", o.FlagName())
}
optionSet[o.FlagName()] = struct{}{}
}
return nil
}
// NewOptions creates new Options from the given raw Envoy arguments. Returns an error if a problem
// was encountered while parsing the arguments.
func NewOptions(args ...string) (Options, error) {
out := make(Options, 0, len(args))
var next *genericOption
for _, arg := range args {
arg = strings.TrimSpace(arg)
if strings.HasPrefix(arg, "-") {
// The argument is a new flag name.
if next != nil {
out = append(out, next)
}
flagName := FlagName(arg)
if v, ok := flagValidators[flagName]; ok {
// Known flag - use an existing validator.
next = &genericOption{
v: v,
}
} else {
// Unknown flag - No validator.
next = &genericOption{
v: &flagValidator{
flagName: flagName,
apply: func(*configContext, string) {},
validate: func(*configContext, string) error {
return nil
},
},
}
}
} else {
// The argument is a flag value.
if next == nil {
return nil, fmt.Errorf("raw argument missing flag name: %s", arg)
}
// Completed the current flag.
next.value = arg
out = append(out, next)
next = nil
}
}
if next != nil {
out = append(out, next)
}
return out, nil
}
// LogLevel is an Option that sets the Envoy log level.
type LogLevel string
var _ Option = LogLevel("")
const (
LogLevelTrace LogLevel = "trace"
LogLevelDebug LogLevel = "debug"
LogLevelInfo LogLevel = "info"
LogLevelWarning LogLevel = "warning"
LogLevelCritical LogLevel = "critical"
LogLevelOff LogLevel = "off"
)
func (l LogLevel) FlagName() FlagName {
return logLevelValidator.flagName
}
func (l LogLevel) FlagValue() string {
return string(l)
}
func (l LogLevel) apply(ctx *configContext) {
logLevelValidator.apply(ctx, l.FlagValue())
}
func (l LogLevel) validate(ctx *configContext) error {
return logLevelValidator.validate(ctx, l.FlagValue())
}
var logLevelValidator = registerFlagValidator(&flagValidator{
flagName: "--log-level",
apply: func(ctx *configContext, flagValue string) {
// Do nothing.
},
validate: func(ctx *configContext, flagValue string) error {
logLevel := LogLevel(flagValue)
switch logLevel {
case LogLevelTrace, LogLevelDebug, LogLevelInfo, LogLevelWarning, LogLevelCritical, LogLevelOff:
return nil
default:
return fmt.Errorf("unsupported log level: %v", logLevel)
}
},
})
// ComponentLogLevel defines the log level for a single component.
type ComponentLogLevel struct {
Name string
Level LogLevel
}
func (l ComponentLogLevel) String() string {
return l.Name + ":" + string(l.Level)
}
// ParseComponentLogLevels parses the given envoy --component-log-level value string.
func ParseComponentLogLevels(value string) ComponentLogLevels {
parts := strings.Split(value, ",")
out := make(ComponentLogLevels, 0, len(parts))
for _, part := range parts {
keyAndValue := strings.Split(part, ":")
if len(keyAndValue) == 2 {
out = append(out, ComponentLogLevel{
Name: keyAndValue[0],
Level: LogLevel(keyAndValue[1]),
})
}
}
return out
}
// ComponentLogLevels is an Option for multiple component log levels.
type ComponentLogLevels []ComponentLogLevel
var _ Option = ComponentLogLevels{}
func (l ComponentLogLevels) apply(ctx *configContext) {
componentLogLevelValidator.apply(ctx, l.FlagValue())
}
func (l ComponentLogLevels) validate(ctx *configContext) error {
return componentLogLevelValidator.validate(ctx, l.FlagValue())
}
func (l ComponentLogLevels) FlagName() FlagName {
return componentLogLevelValidator.flagName
}
func (l ComponentLogLevels) FlagValue() string {
strLevels := make([]string, 0, len(l))
for _, cl := range l {
strLevels = append(strLevels, cl.String())
}
return strings.Join(strLevels, ",")
}
var componentLogLevelValidator = registerFlagValidator(&flagValidator{
flagName: "--component-log-level",
apply: func(ctx *configContext, flagValue string) {
// Do nothing.
},
validate: func(ctx *configContext, flagValue string) error {
l := ParseComponentLogLevels(flagValue)
for i, cl := range l {
if cl.Name == "" {
return fmt.Errorf("name is empty for component log level %d", i)
}
if err := cl.Level.validate(ctx); err != nil {
return fmt.Errorf("level invalid for component log level %d: %v", i, err)
}
}
return nil
},
})
// IPVersion is an enumeration for IP versions for the --local-address-ip-version flag.
type IPVersion string
const (
IPV4 IPVersion = "v4"
IPV6 IPVersion = "v6"
)
// LocalAddressIPVersion sets the --local-address-ip-version flag, which sets the IP address
// version used for the local IP address. The default is V4.
func LocalAddressIPVersion(v IPVersion) Option {
return &genericOption{
value: string(v),
v: localAddressIPVersionValidator,
}
}
var localAddressIPVersionValidator = registerFlagValidator(&flagValidator{
flagName: "--local-address-ip-version",
validate: func(ctx *configContext, flagValue string) error {
ipVersion := IPVersion(flagValue)
switch ipVersion {
case IPV4, IPV6:
return nil
default:
return fmt.Errorf("invalid LocalAddressIPVersion %v", ipVersion)
}
},
})
// ConfigPath sets the --config-path flag, which provides Envoy with the
// to the v2 bootstrap configuration file. If not set, ConfigYaml is required.
func ConfigPath(path string) Option {
return &genericOption{
value: path,
v: configPathValidator,
}
}
var configPathValidator = registerFlagValidator(&flagValidator{
flagName: "--config-path",
apply: func(ctx *configContext, flagValue string) {
ctx.configPath = flagValue
},
validate: func(ctx *configContext, flagValue string) error {
// Ensure that either config path or configYaml is specified.
if ctx.configPath == "" && ctx.configYaml == "" {
return errors.New("must specify ConfigPath and/or ConfigYaml ")
}
if ctx.configPath != "" {
// Check that the path set in the config exists.
if _, err := os.Stat(ctx.configPath); os.IsNotExist(err) {
return fmt.Errorf("configPath file does not exist: %s", ctx.configPath)
}
}
return nil
},
})
// ConfigYaml sets the --config-yaml flag, which provides Envoy with the
// a YAML string for a v2 bootstrap configuration. If ConfigPath is also set, the values in this
// YAML string will override and merge with the bootstrap loaded from ConfigPath.
func ConfigYaml(yaml string) Option {
return &genericOption{
value: yaml,
v: configYamlValidator,
}
}
var configYamlValidator = registerFlagValidator(&flagValidator{
flagName: "--config-yaml",
apply: func(ctx *configContext, flagValue string) {
ctx.configYaml = flagValue
},
})
// BaseID is an Option that sets the --base-id flag. This is typically only needed when running multiple
// Envoys on the same machine (common in testing environments).
//
// Envoy will allocate shared memory if provided with a BaseID. This shared memory is used during hot restarts.
// It is up to the caller to free this memory by calling Close() on the BaseID when appropriate.
type BaseID uint32
var _ Option = BaseID(0)
func (bid BaseID) FlagName() FlagName {
return baseIDValidator.flagName
}
func (bid BaseID) FlagValue() string {
return strconv.FormatUint(uint64(bid), 10)
}
func (bid BaseID) apply(ctx *configContext) {
baseIDValidator.apply(ctx, bid.FlagValue())
}
func (bid BaseID) validate(ctx *configContext) error {
return baseIDValidator.validate(ctx, bid.FlagValue())
}
// GetInternalEnvoyValue returns the value used internally by Envoy.
func (bid BaseID) GetInternalEnvoyValue() uint64 {
return uint64(bid)
}
// Close removes the shared memory allocated by Envoy for this BaseID.
func (bid BaseID) Close() error {
if bid != InvalidBaseID {
// Envoy internally multiplies the base ID from the command line by 10 so that they have spread
// for domain sockets.
path := fmt.Sprintf("/dev/shm/envoy_shared_memory_%d", bid.GetInternalEnvoyValue())
if err := os.Remove(path); err != nil {
return fmt.Errorf("error deleting Envoy base ID %d shared memory %s: %v", bid, path, err)
}
log.Debugf("successfully freed Envoy base ID %d shared memory %s", bid, path)
}
return nil
}
var baseIDValidator = registerFlagValidator(&flagValidator{
flagName: "--base-id",
apply: func(ctx *configContext, flagValue string) {
if bid, err := strconv.ParseUint(flagValue, 10, 64); err == nil {
ctx.baseID = BaseID(bid)
}
},
validate: func(ctx *configContext, flagValue string) error {
if _, err := strconv.ParseUint(flagValue, 10, 64); err != nil {
return err
}
return nil
},
})
// GenerateBaseID is a method copied from Envoy server tests.
//
// Computes a numeric ID to incorporate into the names of shared-memory segments and
// domain sockets, to help keep them distinct from other tests that might be running concurrently.
func GenerateBaseID() BaseID {
// The PID is needed to isolate namespaces between concurrent processes in CI.
pid := uint32(os.Getpid())
// A random number is needed to avoid BaseID collisions for multiple Envoys started from the same
// process.
randNum := rand.Uint32()
// Pick a prime number to give more of the 32-bits of entropy to the PID, and the
// remainder to the random number.
fourDigitPrime := uint32(7919)
value := pid*fourDigitPrime + randNum%fourDigitPrime
// TODO(nmittler): Limit to uint16 - Large values seem to cause unexpected shared memory paths in envoy.
out := BaseID(value % math.MaxUint16)
return out
}
// Concurrency sets the --concurrency flag, which sets the number of worker threads to run.
func Concurrency(concurrency uint16) Option {
return &genericOption{
value: strconv.FormatUint(uint64(concurrency), 10),
v: concurrencyValidator,
}
}
var concurrencyValidator = registerFlagValidator(&flagValidator{
flagName: "--concurrency",
validate: func(ctx *configContext, flagValue string) error {
if _, err := strconv.ParseUint(flagValue, 10, 64); err != nil {
return err
}
return nil
},
})
// DisableHotRestart sets the --disable-hot-restart flag.
func DisableHotRestart(disable bool) Option {
var v *flagValidator
if disable {
v = disableHotRestartValidator
}
return &genericOption{
v: v,
value: "",
}
}
var disableHotRestartValidator = registerBoolFlagValidator("--disable-hot-restart")
// LogPath sets the --log-path flag, which specifies the output file for logs. If not set
// logs will be written to stderr.
func LogPath(path string) Option {
return &genericOption{
value: path,
v: logPathValidator,
}
}
var logPathValidator = registerFlagValidator(&flagValidator{
flagName: "--log-path",
})
// LogFormat sets the --log-format flag, which specifies the format string to use for log
// messages.
func LogFormat(format string) Option {
return &genericOption{
value: format,
v: logFormatValidator,
}
}
var logFormatValidator = registerFlagValidator(&flagValidator{
flagName: "--log-format",
})
// Epoch sets the --restart-epoch flag, which specifies the epoch used for hot restart.
type Epoch uint32
func (e Epoch) FlagName() FlagName {
return epochValidator.flagName
}
func (e Epoch) FlagValue() string {
return strconv.FormatUint(uint64(e), 10)
}
func (e Epoch) apply(ctx *configContext) {
epochValidator.apply(ctx, e.FlagValue())
}
func (e Epoch) validate(ctx *configContext) error {
return epochValidator.validate(ctx, e.FlagValue())
}
var epochValidator = registerFlagValidator(&flagValidator{
flagName: "--restart-epoch",
apply: func(ctx *configContext, flagValue string) {
if e, err := strconv.ParseUint(flagValue, 10, 32); err == nil {
ctx.epoch = Epoch(e)
}
},
validate: func(ctx *configContext, flagValue string) error {
if _, err := strconv.ParseUint(flagValue, 10, 32); err != nil {
return err
}
return nil
},
})
// ServiceCluster sets the --service-cluster flag, which defines the local service cluster
// name where Envoy is running
func ServiceCluster(c string) Option {
return &genericOption{
value: c,
v: serviceClusterValidator,
}
}
var serviceClusterValidator = registerFlagValidator(&flagValidator{
flagName: "--service-cluster",
})
// ServiceNode sets the --service-node flag, which defines the local service node name
// where Envoy is running
func ServiceNode(n string) Option {
return &genericOption{
value: n,
v: serviceNodeValidator,
}
}
var serviceNodeValidator = registerFlagValidator(&flagValidator{
flagName: "--service-node",
})
// DrainDuration sets the --drain-time-s flag, which defines the amount of time that Envoy will
// drain connections during a hot restart.
func DrainDuration(duration time.Duration) Option {
return &genericOption{
value: strconv.Itoa(int(duration / time.Second)),
v: drainDurationValidator,
}
}
var drainDurationValidator = registerFlagValidator(&flagValidator{
flagName: "--drain-time-s",
validate: func(ctx *configContext, flagValue string) error {
if _, err := strconv.ParseUint(flagValue, 10, 32); err != nil {
return err
}
return nil
},
})
// ParentShutdownDuration sets the --parent-shutdown-time-s flag, which defines the amount of
// time that Envoy will wait before shutting down the parent process during a hot restart
func ParentShutdownDuration(duration time.Duration) Option {
return &genericOption{
value: strconv.Itoa(int(duration / time.Second)),
v: parentShutdownDurationValidator,
}
}
var parentShutdownDurationValidator = registerFlagValidator(&flagValidator{
flagName: "--parent-shutdown-time-s",
validate: func(ctx *configContext, flagValue string) error {
if _, err := strconv.ParseUint(flagValue, 10, 32); err != nil {
return err
}
return nil
},
})
func registerBoolFlagValidator(flagName string) *flagValidator {
return registerFlagValidator(&flagValidator{
flagName: FlagName(flagName),
validate: func(ctx *configContext, flagValue string) error {
switch flagValue {
case "", "true":
return nil
default:
return fmt.Errorf("unexpected boolean value for flag %s: %s", flagName, flagValue)
}
},
})
}
func getAdminPortFromYaml(yamlData string) (uint32, error) {
jsonData, err := yaml.YAMLToJSON([]byte(yamlData))
if err != nil {
return 0, fmt.Errorf("error converting envoy bootstrap YAML to JSON: %v", err)
}
bootstrap := &envoyBootstrap.Bootstrap{}
if err := unmarshal(string(jsonData), bootstrap); err != nil {
return 0, fmt.Errorf("error parsing Envoy bootstrap JSON: %v", err)
}
if bootstrap.GetAdmin() == nil {
return 0, errors.New("unable to locate admin in envoy bootstrap")
}
if bootstrap.GetAdmin().GetAddress() == nil {
return 0, errors.New("unable to locate admin/address in envoy bootstrap")
}
if bootstrap.GetAdmin().GetAddress().GetSocketAddress() == nil {
return 0, errors.New("unable to locate admin/address/socket_address in envoy bootstrap")
}
if bootstrap.GetAdmin().GetAddress().GetSocketAddress().GetPortValue() == 0 {
return 0, errors.New("unable to locate admin/address/socket_address/port_value in envoy bootstrap")
}
return bootstrap.GetAdmin().GetAddress().GetSocketAddress().GetPortValue(), nil
}
// configContext stores the output of applied Options.
type configContext struct {
configPath string
configYaml string
baseID BaseID
epoch Epoch
}
func newConfigContext() *configContext {
return &configContext{
baseID: InvalidBaseID,
}
}
func (c *configContext) getAdminPort() (uint32, error) {
var err error
// First, check the config yaml, which overrides config-path.
if c.configYaml != "" {
if port, e := getAdminPortFromYaml(c.configYaml); e != nil {
err = fmt.Errorf("failed to locate admin port in envoy config-yaml: %v", e)
} else {
// Found the port!
return port, nil
}
}
// Haven't found it yet - check configPath.
if c.configPath == "" {
return 0, multierror.Append(err, errors.New("unable to process envoy bootstrap"))
}
content, e := os.ReadFile(c.configPath)
if e != nil {
return 0, multierror.Append(err, fmt.Errorf("failed reading config-path file %s: %v", c.configPath, e))
}
port, e := getAdminPortFromYaml(string(content))
if e != nil {
return 0, multierror.Append(err, fmt.Errorf("failed to locate admin port in envoy config-yaml: %v", e))
}
// Found the port!
return port, nil
}
var flagValidators = make(map[FlagName]*flagValidator)
type flagValidator struct {
flagName FlagName
apply func(ctx *configContext, flagValue string)
validate func(ctx *configContext, flagValue string) error
}
func registerFlagValidator(v *flagValidator) *flagValidator {
flagValidators[v.flagName] = v
// Fill in missing methods with defaults.
if v.apply == nil {
v.apply = func(*configContext, string) {}
}
if v.validate == nil {
v.validate = func(*configContext, string) error {
return nil
}
}
return v
}
var _ Option = &genericOption{}
type genericOption struct {
v *flagValidator
value string
}
func (o *genericOption) FlagName() FlagName {
if o.v != nil {
return o.v.flagName
}
return ""
}
func (o *genericOption) FlagValue() string {
if o.v != nil {
return o.value
}
return ""
}
func (o *genericOption) apply(ctx *configContext) {
if o.v != nil && o.v.apply != nil {
o.v.apply(ctx, o.value)
}
}
func (o *genericOption) validate(ctx *configContext) error {
if o.v != nil && o.v.validate != nil {
return o.v.validate(ctx, o.value)
}
return nil
}