| /* |
| Copyright 2016 The Kubernetes Authors. |
| |
| Licensed under the Apache License, Version 2.0 (the "License"); |
| you may not use this file except in compliance with the License. |
| You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, software |
| distributed under the License is distributed on an "AS IS" BASIS, |
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| See the License for the specific language governing permissions and |
| limitations under the License. |
| */ |
| |
| package app |
| |
| import ( |
| "context" |
| "flag" |
| "fmt" |
| "net/http" |
| "os" |
| "time" |
| |
| "github.com/spf13/cobra" |
| |
| "k8s.io/apimachinery/pkg/util/sets" |
| "k8s.io/apimachinery/pkg/util/uuid" |
| "k8s.io/apimachinery/pkg/util/wait" |
| "k8s.io/apiserver/pkg/server" |
| "k8s.io/apiserver/pkg/server/healthz" |
| "k8s.io/apiserver/pkg/util/term" |
| "k8s.io/client-go/tools/leaderelection" |
| "k8s.io/client-go/tools/leaderelection/resourcelock" |
| cloudprovider "k8s.io/cloud-provider" |
| cliflag "k8s.io/component-base/cli/flag" |
| "k8s.io/component-base/cli/globalflag" |
| "k8s.io/klog" |
| cloudcontrollerconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/config" |
| "k8s.io/kubernetes/cmd/cloud-controller-manager/app/options" |
| genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app" |
| "k8s.io/kubernetes/pkg/util/configz" |
| utilflag "k8s.io/kubernetes/pkg/util/flag" |
| "k8s.io/kubernetes/pkg/version" |
| "k8s.io/kubernetes/pkg/version/verflag" |
| ) |
| |
| const ( |
| // ControllerStartJitter is the jitter value used when starting controller managers. |
| ControllerStartJitter = 1.0 |
| // ConfigzName is the name used for register cloud-controller manager /configz, same with GroupName. |
| ConfigzName = "cloudcontrollermanager.config.k8s.io" |
| ) |
| |
| // NewCloudControllerManagerCommand creates a *cobra.Command object with default parameters |
| func NewCloudControllerManagerCommand() *cobra.Command { |
| s, err := options.NewCloudControllerManagerOptions() |
| if err != nil { |
| klog.Fatalf("unable to initialize command options: %v", err) |
| } |
| |
| cmd := &cobra.Command{ |
| Use: "cloud-controller-manager", |
| Long: `The Cloud controller manager is a daemon that embeds |
| the cloud specific control loops shipped with Kubernetes.`, |
| Run: func(cmd *cobra.Command, args []string) { |
| verflag.PrintAndExitIfRequested() |
| utilflag.PrintFlags(cmd.Flags()) |
| |
| c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List()) |
| if err != nil { |
| fmt.Fprintf(os.Stderr, "%v\n", err) |
| os.Exit(1) |
| } |
| |
| if err := Run(c.Complete(), wait.NeverStop); err != nil { |
| fmt.Fprintf(os.Stderr, "%v\n", err) |
| os.Exit(1) |
| } |
| |
| }, |
| } |
| |
| fs := cmd.Flags() |
| namedFlagSets := s.Flags(KnownControllers(), ControllersDisabledByDefault.List()) |
| verflag.AddFlags(namedFlagSets.FlagSet("global")) |
| globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name()) |
| |
| if flag.CommandLine.Lookup("cloud-provider-gce-lb-src-cidrs") != nil { |
| // hoist this flag from the global flagset to preserve the commandline until |
| // the gce cloudprovider is removed. |
| globalflag.Register(namedFlagSets.FlagSet("generic"), "cloud-provider-gce-lb-src-cidrs") |
| } |
| for _, f := range namedFlagSets.FlagSets { |
| fs.AddFlagSet(f) |
| } |
| usageFmt := "Usage:\n %s\n" |
| cols, _, _ := term.TerminalSize(cmd.OutOrStdout()) |
| cmd.SetUsageFunc(func(cmd *cobra.Command) error { |
| fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine()) |
| cliflag.PrintSections(cmd.OutOrStderr(), namedFlagSets, cols) |
| return nil |
| }) |
| cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) { |
| fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine()) |
| cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols) |
| }) |
| |
| return cmd |
| } |
| |
| // Run runs the ExternalCMServer. This should never exit. |
| func Run(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}) error { |
| // To help debugging, immediately log version |
| klog.Infof("Version: %+v", version.Get()) |
| |
| cloud, err := cloudprovider.InitCloudProvider(c.ComponentConfig.KubeCloudShared.CloudProvider.Name, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile) |
| if err != nil { |
| klog.Fatalf("Cloud provider could not be initialized: %v", err) |
| } |
| if cloud == nil { |
| klog.Fatalf("cloud provider is nil") |
| } |
| |
| if !cloud.HasClusterID() { |
| if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud { |
| klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") |
| } else { |
| klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") |
| } |
| } |
| |
| // setup /configz endpoint |
| if cz, err := configz.New(ConfigzName); err == nil { |
| cz.Set(c.ComponentConfig) |
| } else { |
| klog.Errorf("unable to register configz: %c", err) |
| } |
| |
| // Setup any healthz checks we will want to use. |
| var checks []healthz.HealthzChecker |
| var electionChecker *leaderelection.HealthzAdaptor |
| if c.ComponentConfig.Generic.LeaderElection.LeaderElect { |
| electionChecker = leaderelection.NewLeaderHealthzAdaptor(time.Second * 20) |
| checks = append(checks, electionChecker) |
| } |
| |
| // Start the controller manager HTTP server |
| if c.SecureServing != nil { |
| unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...) |
| handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication) |
| // TODO: handle stoppedCh returned by c.SecureServing.Serve |
| if _, err := c.SecureServing.Serve(handler, 0, stopCh); err != nil { |
| return err |
| } |
| } |
| if c.InsecureServing != nil { |
| unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...) |
| insecureSuperuserAuthn := server.AuthenticationInfo{Authenticator: &server.InsecureSuperuser{}} |
| handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, nil, &insecureSuperuserAuthn) |
| if err := c.InsecureServing.Serve(handler, 0, stopCh); err != nil { |
| return err |
| } |
| } |
| |
| run := func(ctx context.Context) { |
| if err := startControllers(c, ctx.Done(), cloud, newControllerInitializers()); err != nil { |
| klog.Fatalf("error running controllers: %v", err) |
| } |
| } |
| |
| if !c.ComponentConfig.Generic.LeaderElection.LeaderElect { |
| run(context.TODO()) |
| panic("unreachable") |
| } |
| |
| // Identity used to distinguish between multiple cloud controller manager instances |
| id, err := os.Hostname() |
| if err != nil { |
| return err |
| } |
| // add a uniquifier so that two processes on the same host don't accidentally both become active |
| id = id + "_" + string(uuid.NewUUID()) |
| |
| // Lock required for leader election |
| rl, err := resourcelock.New(c.ComponentConfig.Generic.LeaderElection.ResourceLock, |
| "kube-system", |
| "cloud-controller-manager", |
| c.LeaderElectionClient.CoreV1(), |
| c.LeaderElectionClient.CoordinationV1(), |
| resourcelock.ResourceLockConfig{ |
| Identity: id, |
| EventRecorder: c.EventRecorder, |
| }) |
| if err != nil { |
| klog.Fatalf("error creating lock: %v", err) |
| } |
| |
| // Try and become the leader and start cloud controller manager loops |
| leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{ |
| Lock: rl, |
| LeaseDuration: c.ComponentConfig.Generic.LeaderElection.LeaseDuration.Duration, |
| RenewDeadline: c.ComponentConfig.Generic.LeaderElection.RenewDeadline.Duration, |
| RetryPeriod: c.ComponentConfig.Generic.LeaderElection.RetryPeriod.Duration, |
| Callbacks: leaderelection.LeaderCallbacks{ |
| OnStartedLeading: run, |
| OnStoppedLeading: func() { |
| klog.Fatalf("leaderelection lost") |
| }, |
| }, |
| WatchDog: electionChecker, |
| Name: "cloud-controller-manager", |
| }) |
| panic("unreachable") |
| } |
| |
| // startControllers starts the cloud specific controller loops. |
| func startControllers(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}, cloud cloudprovider.Interface, controllers map[string]initFunc) error { |
| // Initialize the cloud provider with a reference to the clientBuilder |
| cloud.Initialize(c.ClientBuilder, stopCh) |
| // Set the informer on the user cloud object |
| if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok { |
| informerUserCloud.SetInformers(c.SharedInformers) |
| } |
| |
| for controllerName, initFn := range controllers { |
| if !genericcontrollermanager.IsControllerEnabled(controllerName, ControllersDisabledByDefault, c.ComponentConfig.Generic.Controllers) { |
| klog.Warningf("%q is disabled", controllerName) |
| continue |
| } |
| |
| klog.V(1).Infof("Starting %q", controllerName) |
| _, started, err := initFn(c, cloud, stopCh) |
| if err != nil { |
| klog.Errorf("Error starting %q", controllerName) |
| return err |
| } |
| if !started { |
| klog.Warningf("Skipping %q", controllerName) |
| continue |
| } |
| klog.Infof("Started %q", controllerName) |
| |
| time.Sleep(wait.Jitter(c.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter)) |
| } |
| |
| // If apiserver is not running we should wait for some time and fail only then. This is particularly |
| // important when we start apiserver and controller manager at the same time. |
| if err := genericcontrollermanager.WaitForAPIServer(c.VersionedClient, 10*time.Second); err != nil { |
| klog.Fatalf("Failed to wait for apiserver being healthy: %v", err) |
| } |
| |
| c.SharedInformers.Start(stopCh) |
| |
| select {} |
| } |
| |
| // initFunc is used to launch a particular controller. It may run additional "should I activate checks". |
| // Any error returned will cause the controller process to `Fatal` |
| // The bool indicates whether the controller was enabled. |
| type initFunc func(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stop <-chan struct{}) (debuggingHandler http.Handler, enabled bool, err error) |
| |
| // KnownControllers indicate the default controller we are known. |
| func KnownControllers() []string { |
| ret := sets.StringKeySet(newControllerInitializers()) |
| return ret.List() |
| } |
| |
| // ControllersDisabledByDefault is the controller disabled default when starting cloud-controller managers. |
| var ControllersDisabledByDefault = sets.NewString() |
| |
| // newControllerInitializers is a private map of named controller groups (you can start more than one in an init func) |
| // paired to their initFunc. This allows for structured downstream composition and subdivision. |
| func newControllerInitializers() map[string]initFunc { |
| controllers := map[string]initFunc{} |
| controllers["cloud-node"] = startCloudNodeController |
| controllers["cloud-node-lifecycle"] = startCloudNodeLifecycleController |
| controllers["service"] = startServiceController |
| controllers["route"] = startRouteController |
| return controllers |
| } |