pkg/operator/injector/injector.go

// Licensed to Apache Software Foundation (ASF) under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Apache Software Foundation (ASF) licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package injector

import (
	"context"
	"fmt"
	"strings"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"sigs.k8s.io/controller-runtime/pkg/client"
	logf "sigs.k8s.io/controller-runtime/pkg/log"
)

const (
	// the label means whether to enbale injection , "true" of "false"
	labelKeyagentInjector = "swck-java-agent-injected"
	// the annotation means which container to inject
	sidecarInjectContainerAnno = "strategy.skywalking.apache.org/inject.Container"
	// the annotation means whether to enable overlay agent, "true" of "false"
	sidecarAgentOverlayAnno = "strategy.skywalking.apache.org/agent.Overlay"
	// the annotation that specify the reason for injection failure
	sidecarInjectErrorAnno = "sidecar.skywalking.apache.org/error"
	// those annotations with the following prefixes represent sidecar information
	sidecarAnnotationPrefix = "sidecar.skywalking.apache.org/"
	// those annotations with the following prefixes represent agent information
	agentAnnotationPrefix = "agent.skywalking.apache.org/"
	// If user want to use other Plugins' config ,the annotation must have the following form
	// plugins.skywalking.apache.org/${config.name} = ${config.value}
	// for example , if user want to enable plugin.mongodb.trace_param
	// the annotation is plugins.skywalking.apache.org/plugin.mongodb.trace_param: "true"
	pluginsAnnotationPrefix = "plugins.skywalking.apache.org/"
	// If user want to use optional-plugins , the annotation must match a optinal plugin
	// such as optional.skywalking.apache.org: "*ehcache*"
	// Notice , If the injected container's image don't has the optional plugin ,
	// the container will panic
	optionsAnnotation = "optional.skywalking.apache.org"
	// If user want to use optional-reporter-plugins , the annotation must match a optinal-reporter plugin
	// such as optional-exporter.skywalking.apache.org: "kafka*"
	optionsReporterAnnotation = "optional-reporter.skywalking.apache.org"
)

// log is for logging in this package.
var log = logf.Log.WithName("injector")

// SidecarInjectField contains all info that will be injected
type SidecarInjectField struct {
	// determine whether to inject , default is not to inject
	NeedInject bool
	// determine whether to use annotation to overlay agent config ,
	// default is not to overlay，which means only use configmap to set agent config
	// Otherwise, the way to overlay is set jvm agent ,just like following
	// -javaagent: /sky/agent/skywalking-agent,jar={config1}={value1},{config2}={value2}
	AgentOverlay bool
	// Initcontainer is a container that has the agent folder
	Initcontainer corev1.Container
	// sidecarVolume is a shared directory between App's container and initcontainer
	SidecarVolume corev1.Volume
	// sidecarVolumeMount is a path that specifies a shared directory
	SidecarVolumeMount corev1.VolumeMount
	// configmapVolume is volume that provide user with configmap
	ConfigmapVolume corev1.Volume
	// configmapVolumeMount is the configmap's mountpath for user
	// Notice : the mount path will overwrite the original agent/config/agent.config
	// So the mount path must match the path of agent.config in the image
	ConfigmapVolumeMount corev1.VolumeMount
	// env is used to set java agent’s parameters
	Env corev1.EnvVar
	// the string is used to set jvm agent ,just like following
	// -javaagent: /sky/agent/skywalking-agent,jar=jvmAgentConfigStr
	JvmAgentConfigStr string
	// determine which container to inject , default is to inject all containers
	InjectContainer string
}

// NewSidecarInjectField will create a new SidecarInjectField
func NewSidecarInjectField() *SidecarInjectField {
	return new(SidecarInjectField)
}

// Inject will do real injection
func (s *SidecarInjectField) Inject(pod *corev1.Pod) {
	// add initcontrainers to spec
	if pod.Spec.InitContainers != nil {
		pod.Spec.InitContainers = append(pod.Spec.InitContainers, s.Initcontainer)
	} else {
		pod.Spec.InitContainers = []corev1.Container{s.Initcontainer}
	}

	// add volume to spec
	if pod.Spec.Volumes != nil {
		pod.Spec.Volumes = append(pod.Spec.Volumes, s.SidecarVolume, s.ConfigmapVolume)
	} else {
		pod.Spec.Volumes = []corev1.Volume{s.SidecarVolume, s.ConfigmapVolume}
	}

	// choose a specific container to inject
	containers := []*corev1.Container{}

	c := s.findInjectContainer(pod.Spec.Containers)
	if c != nil {
		containers = append(containers, c)
	} else {
		for i := range pod.Spec.Containers {
			containers = append(containers, &pod.Spec.Containers[i])
		}
	}

	// add volumemount and env to container
	for i := range containers {
		if (*containers[i]).VolumeMounts != nil {
			(*containers[i]).VolumeMounts = append((*containers[i]).VolumeMounts,
				s.SidecarVolumeMount, s.ConfigmapVolumeMount)
		} else {
			(*containers[i]).VolumeMounts = []corev1.VolumeMount{s.SidecarVolumeMount,
				s.ConfigmapVolumeMount}
		}
		if (*containers[i]).Env != nil {
			(*containers[i]).Env = append((*containers[i]).Env, s.Env)
		} else {
			(*containers[i]).Env = []corev1.EnvVar{s.Env}
		}
	}
}

// GetInjectStrategy gets user's injection strategy
func (s *SidecarInjectField) GetInjectStrategy(a Annotations, labels,
	annotation *map[string]string) {
	// set default value
	s.NeedInject = false
	s.InjectContainer = ""
	s.AgentOverlay = false

	// set NeedInject's value , if the pod has the label "swck-java-agent-injected=true",means need inject
	if *labels == nil {
		return
	}

	if strings.ToLower((*labels)[labelKeyagentInjector]) == "true" {
		s.NeedInject = true
	}

	if *annotation == nil {
		return
	}

	// set injectContainer's value
	if v, ok := (*annotation)[sidecarInjectContainerAnno]; ok {
		s.InjectContainer = v
	}

	// set AgentOverlay's value
	if v, ok := (*annotation)[sidecarAgentOverlayAnno]; ok {
		if strings.ToLower(v) == "true" {
			s.AgentOverlay = true
		}
	}
}

func (s *SidecarInjectField) findInjectContainer(containers []corev1.Container) *corev1.Container {
	// validate the container is or not exist
	if s.InjectContainer == "" {
		return nil
	}

	for i := range containers {
		if containers[i].Name == s.InjectContainer {
			return &containers[i]
		}
	}
	return nil
}
func (s *SidecarInjectField) injectErrorAnnotation(annotation *map[string]string, errorInfo string) {
	(*annotation)[sidecarInjectErrorAnno] = errorInfo
}

// SidecarOverlayandGetValue get final value of sidecar
func (s *SidecarInjectField) SidecarOverlayandGetValue(ao *AnnotationOverlay, annotation *map[string]string,
	a Annotation) (string, bool) {
	if _, ok := (*annotation)[a.Name]; ok {
		err := ao.SetOverlay(annotation, a)
		if err != nil {
			s.injectErrorAnnotation(annotation, err.Error())
			return "", false
		}
	}
	return ao.GetFinalValue(a), true
}

func (s *SidecarInjectField) setValue(config *string, ao *AnnotationOverlay, annotation *map[string]string,
	a Annotation) bool {
	if v, ok := s.SidecarOverlayandGetValue(ao, annotation, a); ok {
		*config = v
		return true
	}
	return false
}

// OverlaySidecar overlays default config
func (s *SidecarInjectField) OverlaySidecar(a Annotations, ao *AnnotationOverlay, annotation *map[string]string) bool {
	s.ConfigmapVolume.ConfigMap = new(corev1.ConfigMapVolumeSource)
	s.Initcontainer.Command = make([]string, 1)
	s.Initcontainer.Args = make([]string, 2)
	// every annotation map a pointer to the field of SidecarInjectField
	annoField := map[string]*string{
		"initcontainer.Name":             &s.Initcontainer.Name,
		"initcontainer.Image":            &s.Initcontainer.Image,
		"initcontainer.Command":          &s.Initcontainer.Command[0],
		"initcontainer.args.Option":      &s.Initcontainer.Args[0],
		"initcontainer.args.Command":     &s.Initcontainer.Args[1],
		"sidecarVolume.Name":             &s.SidecarVolume.Name,
		"sidecarVolumeMount.MountPath":   &s.SidecarVolumeMount.MountPath,
		"configmapVolume.ConfigMap.Name": &s.ConfigmapVolume.ConfigMap.Name,
		"configmapVolume.Name":           &s.ConfigmapVolume.Name,
		"configmapVolumeMount.MountPath": &s.ConfigmapVolumeMount.MountPath,
		"env.Name":                       &s.Env.Name,
		"env.Value":                      &s.Env.Value,
	}
	anno := GetAnnotationsByPrefix(a, sidecarAnnotationPrefix)
	for _, v := range anno.Annotations {
		fieldName := strings.TrimPrefix(v.Name, sidecarAnnotationPrefix)
		if pointer, ok := annoField[fieldName]; ok {
			if !s.setValue(pointer, ao, annotation, v) {
				return false
			}
		}
	}
	s.SidecarVolumeMount.Name = s.SidecarVolume.Name
	s.ConfigmapVolumeMount.Name = s.ConfigmapVolume.Name
	s.Initcontainer.VolumeMounts = []corev1.VolumeMount{s.SidecarVolumeMount}

	// the sidecar volume's type is determined
	s.SidecarVolume.VolumeSource.EmptyDir = nil

	return true
}

// AgentOverlayandGetValue will do real annotation overlay
func (s *SidecarInjectField) AgentOverlayandGetValue(ao *AnnotationOverlay, annotation *map[string]string,
	a Annotation) (string, bool) {
	if _, ok := (*annotation)[a.Name]; ok {
		err := ao.SetOverlay(annotation, a)
		if err != nil {
			s.injectErrorAnnotation(annotation, err.Error())
			return "", false
		}
	}
	return ao.GetOverlayValue(a), true
}

func (s *SidecarInjectField) setJvmAgentStr(ao *AnnotationOverlay, annotation *map[string]string, a Annotation) bool {
	v, ok := s.AgentOverlayandGetValue(ao, annotation, a)
	if v != "" && ok {
		// get {config1}={value1}
		configName := strings.TrimPrefix(a.Name, agentAnnotationPrefix)
		switch a.UseQuotes {
		case "option":
			configName = strings.Join([]string{"'", "'"}, configName)
		case "value":
			v = strings.Join([]string{"'", "'"}, v)
		}
		config := strings.Join([]string{configName, v}, "=")

		// add to jvmAgentConfigStr
		if s.JvmAgentConfigStr != "" {
			s.JvmAgentConfigStr = strings.Join([]string{s.JvmAgentConfigStr, config}, ",")
		} else {
			s.JvmAgentConfigStr = config
		}
	}
	return ok
}

// OverlayAgent overlays agent
func (s *SidecarInjectField) OverlayAgent(a Annotations, ao *AnnotationOverlay, annotation *map[string]string) bool {
	// jvmAgentConfigStr init
	s.JvmAgentConfigStr = ""
	anno := GetAnnotationsByPrefix(a, agentAnnotationPrefix)
	for _, v := range anno.Annotations {
		if !s.setJvmAgentStr(ao, annotation, v) {
			return false
		}
	}
	return true
}

// OverlayOptional overlays optional plugins and move optional plugins to the directory(/plugins)
// user must ensure that the optional plugins are in the injected container's image
// Notice , user must specify the correctness of the regular value
// such as optional.skywalking.apache.org: "*ehcache*" or optional-reporter.skywalking.apache.org: "kafka*"
// the final command will be "cp /optional-plugins/*ehcache*  /plugins/" or
// "cp /optional-exporter-plugins/kafka*  /plugins/"
func (s *SidecarInjectField) OverlayOptional(annotation *map[string]string) {
	sourceOptionalPath := strings.Join([]string{s.SidecarVolumeMount.MountPath, "optional-plugins/"}, "/")
	sourceOptionalReporterPath := strings.Join([]string{s.SidecarVolumeMount.MountPath, "optional-reporter-plugins/"}, "/")
	targetPath := strings.Join([]string{s.SidecarVolumeMount.MountPath, "plugins/"}, "/")

	for k, v := range *annotation {
		command := ""
		if strings.EqualFold(k, optionsAnnotation) {
			command = strings.Join([]string{"cp", sourceOptionalPath}, " ")
		} else if strings.EqualFold(k, optionsReporterAnnotation) {
			command = strings.Join([]string{"cp", sourceOptionalReporterPath}, " ")
		}
		if command != "" {
			command = command + v + " " + targetPath
			s.Initcontainer.Args[1] = strings.Join([]string{s.Initcontainer.Args[1], command}, " && ")
		}
	}

}

// OverlayPlugins will add Plugins' config to JvmAgentStr without verification
// Notice, if a config is not in agent.config, it will be seen as a plugin config
// user must ensure the accuracy of configuration.
// Otherwides,If a separator(, or =) in the option or value, it should be wrapped in quotes.
func (s *SidecarInjectField) OverlayPlugins(annotation *map[string]string) {
	for k, v := range *annotation {
		if strings.HasPrefix(k, pluginsAnnotationPrefix) {
			configName := strings.TrimPrefix(k, pluginsAnnotationPrefix)
			config := strings.Join([]string{configName, v}, "=")
			// add to jvmAgentConfigStr
			if s.JvmAgentConfigStr != "" {
				s.JvmAgentConfigStr = strings.Join([]string{s.JvmAgentConfigStr, config}, ",")
			} else {
				s.JvmAgentConfigStr = config
			}
		}
	}
}

// CreateConfigmap will create a configmap to set java agent config.
func (s *SidecarInjectField) CreateConfigmap(ctx context.Context, kubeclient client.Client, namespace string,
	annotation *map[string]string) bool {
	configmap := &corev1.ConfigMap{}
	configmapName := s.ConfigmapVolume.VolumeSource.ConfigMap.LocalObjectReference.Name
	// check whether the configmap is existed
	err := kubeclient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: configmapName}, configmap)
	if err != nil && !errors.IsNotFound(err) {
		log.Error(err, "Get Configmap failed", "configmapName", configmapName, "namespace", namespace)
		return false
	}
	// if configmap exist , validate it
	if !errors.IsNotFound(err) {
		ok, errinfo := ValidateConfigmap(configmap)
		if ok {
			log.Info("the configmap validate true", "configmapName", configmapName)
			return true
		}
		log.Error(errinfo, "the configmap validate false", "configmapName", configmapName)
	}
	// if configmap not exist or validate false , get default configmap
	defaultConfigmap := &corev1.ConfigMap{}
	if err := kubeclient.Get(ctx, client.ObjectKey{Namespace: DefaultConfigmapNamespace,
		Name: DefaultConfigmapName}, defaultConfigmap); err != nil {
		log.Error(err, "can't get default configmap")
		s.injectErrorAnnotation(annotation, fmt.Sprintf("get configmap %s from namespace %s error[%s]",
			DefaultConfigmapName, DefaultConfigmapNamespace, err.Error()))
		return false
	}

	// use default configmap's data to create new configmap and update namespace
	injectConfigmap := corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name:      configmapName,
			Namespace: namespace,
		},
		Data: defaultConfigmap.Data,
	}

	// create the configmap in user's namespace
	if err := kubeclient.Create(ctx, &injectConfigmap); err != nil {
		log.Error(err, "create configmap failed")
		s.injectErrorAnnotation(annotation, fmt.Sprintf("create configmap %s in namespace %s error[%s]",
			configmapName, namespace, err.Error()))
		return false
	}
	return true
}