internal/manager/run.go

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package manager

import (
	"context"
	"crypto/tls"
	"os"

	"github.com/go-logr/logr"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
	"k8s.io/utils/ptr"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/healthz"
	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
	"sigs.k8s.io/controller-runtime/pkg/webhook"
	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
	gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
	"sigs.k8s.io/gateway-api/apis/v1beta1"

	"github.com/apache/apisix-ingress-controller/api/v1alpha1"
	apiv2 "github.com/apache/apisix-ingress-controller/api/v2"
	"github.com/apache/apisix-ingress-controller/internal/controller"
	"github.com/apache/apisix-ingress-controller/internal/controller/config"
	"github.com/apache/apisix-ingress-controller/internal/controller/status"
	"github.com/apache/apisix-ingress-controller/internal/manager/readiness"
	"github.com/apache/apisix-ingress-controller/internal/manager/server"
	"github.com/apache/apisix-ingress-controller/internal/provider"
	_ "github.com/apache/apisix-ingress-controller/internal/provider/init"
	_ "github.com/apache/apisix-ingress-controller/pkg/metrics"
)

var (
	scheme = runtime.NewScheme()
)

func init() {
	utilruntime.Must(clientgoscheme.AddToScheme(scheme))

	if err := gatewayv1.Install(scheme); err != nil {
		panic(err)
	}
	if err := gatewayv1alpha2.Install(scheme); err != nil {
		panic(err)
	}
	if err := v1alpha1.AddToScheme(scheme); err != nil {
		panic(err)
	}
	if err := apiv2.AddToScheme(scheme); err != nil {
		panic(err)
	}
	if err := v1beta1.Install(scheme); err != nil {
		panic(err)
	}
	// +kubebuilder:scaffold:scheme
}

func Run(ctx context.Context, logger logr.Logger) error {
	cfg := config.ControllerConfig

	setupLog := ctrl.LoggerFrom(ctx).WithName("setup")

	// if the enable-http2 flag is false (the default), http/2 should be disabled
	// due to its vulnerabilities. More specifically, disabling http/2 will
	// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
	// Rapid Reset CVEs. For more information see:
	// - https://github.com/advisories/GHSA-qppj-fm5r-hxr3
	// - https://github.com/advisories/GHSA-4374-p667-p6c8
	disableHTTP2 := func(c *tls.Config) {
		setupLog.Info("disabling http/2")
		c.NextProtos = []string{"http/1.1"}
	}

	var tlsOpts []func(*tls.Config)

	if !cfg.EnableHTTP2 {
		tlsOpts = append(tlsOpts, disableHTTP2)
	}

	// Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server.
	// More info:
	// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/server
	// - https://book.kubebuilder.io/reference/metrics.html
	metricsServerOptions := metricsserver.Options{
		BindAddress:   cfg.MetricsAddr,
		SecureServing: cfg.SecureMetrics,
		// TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are
		// not provided, self-signed certificates will be generated by default. This option is not recommended for
		// production environments as self-signed certificates do not offer the same level of trust and security
		// as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing
		// unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName
		// to provide certificates, ensuring the server communicates using trusted and secure certificates.
		TLSOpts: tlsOpts,
	}

	if cfg.SecureMetrics {
		// FilterProvider is used to protect the metrics endpoint with authn/authz.
		// These configurations ensure that only authorized users and service accounts
		// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
		// https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/filters#WithAuthenticationAndAuthorization
		metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
	}

	namespace := os.Getenv("POD_NAMESPACE")
	if namespace == "" {
		namespace = "default"
	}

	mgrOptions := ctrl.Options{
		Scheme:                  scheme,
		Metrics:                 metricsServerOptions,
		HealthProbeBindAddress:  cfg.ProbeAddr,
		LeaderElection:          !config.ControllerConfig.LeaderElection.Disable,
		LeaderElectionID:        cfg.LeaderElectionID,
		LeaderElectionNamespace: namespace,
		LeaseDuration:           ptr.To(config.ControllerConfig.LeaderElection.LeaseDuration.Duration),
		RenewDeadline:           ptr.To(config.ControllerConfig.LeaderElection.RenewDeadline.Duration),
		RetryPeriod:             ptr.To(config.ControllerConfig.LeaderElection.RetryPeriod.Duration),
		// LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily
		// when the Manager ends. This requires the binary to immediately end when the
		// Manager is stopped, otherwise, this setting is unsafe. Setting this significantly
		// speeds up voluntary leader transitions as the new leader don't have to wait
		// LeaseDuration time first.
		//
		// In the default scaffold provided, the program ends immediately after
		// the manager stops, so would be fine to enable this option. However,
		// if you are doing or is intended to do any operation such as perform cleanups
		// after the manager stops then its usage might be unsafe.
		// LeaderElectionReleaseOnCancel: true,
	}

	if cfg.Webhook != nil && cfg.Webhook.Enable {
		webhookServer := webhook.NewServer(webhook.Options{
			Port:     cfg.Webhook.Port,
			CertDir:  cfg.Webhook.TLSCertDir,
			CertName: cfg.Webhook.TLSCertFile,
			KeyName:  cfg.Webhook.TLSKeyFile,
		})
		mgrOptions.WebhookServer = webhookServer
	}

	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), mgrOptions)
	if err != nil {
		setupLog.Error(err, "unable to start manager")
		return err
	}

	readier := readiness.NewReadinessManager(mgr.GetClient(), logger)
	registerReadinessGVK(mgr.GetClient(), readier)

	if err := mgr.Add(readier); err != nil {
		setupLog.Error(err, "unable to add readiness manager")
	}

	updater := status.NewStatusUpdateHandler(ctrl.LoggerFrom(ctx).WithName("status").WithName("updater"), mgr.GetClient())
	if err := mgr.Add(updater); err != nil {
		setupLog.Error(err, "unable to add status updater")
		return err
	}

	providerType := string(config.ControllerConfig.ProviderConfig.Type)

	providerOptions := &provider.Options{
		SyncTimeout:   config.ControllerConfig.ExecADCTimeout.Duration,
		SyncPeriod:    config.ControllerConfig.ProviderConfig.SyncPeriod.Duration,
		InitSyncDelay: config.ControllerConfig.ProviderConfig.InitSyncDelay.Duration,
	}
	provider, err := provider.New(providerType, logger, updater.Writer(), readier, providerOptions)
	if err != nil {
		setupLog.Error(err, "unable to create provider")
		return err
	}

	if cfg.EnableServer {
		srv := server.NewServer(config.ControllerConfig.ServerAddr)
		srv.Register("/debug", provider)
		if err := mgr.Add(srv); err != nil {
			setupLog.Error(err, "unable to add debug server to manager")
			return err
		}
	}
	if err := mgr.Add(provider); err != nil {
		setupLog.Error(err, "unable to add provider to manager")
		return err
	}

	setupLog.Info("check ReferenceGrants is enabled")
	_, err = mgr.GetRESTMapper().KindsFor(schema.GroupVersionResource{
		Group:    v1beta1.GroupVersion.Group,
		Version:  v1beta1.GroupVersion.Version,
		Resource: "referencegrants",
	})
	if err != nil {
		setupLog.Info("CRD ReferenceGrants is not installed", "err", err)
	}
	controller.SetEnableReferenceGrant(err == nil)

	setupLog.Info("setting up controllers")
	controllers, err := setupControllers(ctx, mgr, provider, updater.Writer(), readier)
	if err != nil {
		setupLog.Error(err, "unable to set up controllers")
		return err
	}

	for _, c := range controllers {
		if err := c.SetupWithManager(mgr); err != nil {
			return err
		}
	}

	// +kubebuilder:scaffold:builder

	if cfg.Webhook != nil && cfg.Webhook.Enable {
		setupLog.Info("setting up webhooks")
		if err := setupWebhooks(ctx, mgr); err != nil {
			setupLog.Error(err, "unable to create webhook", "webhook", "Ingress")
			return err
		}
	} else {
		setupLog.Info("webhooks disabled, skipping webhook setup")
	}

	setupLog.Info("setting up health checks")
	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
		setupLog.Error(err, "unable to set up health check")
		return err
	}

	setupLog.Info("setting up ready checks")
	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
		setupLog.Error(err, "unable to set up ready check")
		return err
	}

	setupLog.Info("starting controller manager")
	return mgr.Start(ctrl.SetupSignalHandler())
}