| // +build linux |
| |
| /* |
| Copyright 2017 The Kubernetes Authors. |
| |
| Licensed under the Apache License, Version 2.0 (the "License"); |
| you may not use this file except in compliance with the License. |
| You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, software |
| distributed under the License is distributed on an "AS IS" BASIS, |
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| See the License for the specific language governing permissions and |
| limitations under the License. |
| */ |
| |
| package e2e_node |
| |
| import ( |
| "fmt" |
| "io/ioutil" |
| "path/filepath" |
| "strconv" |
| "strings" |
| "time" |
| |
| "k8s.io/api/core/v1" |
| "k8s.io/apimachinery/pkg/api/resource" |
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" |
| "k8s.io/kubernetes/pkg/kubelet/cm" |
| "k8s.io/kubernetes/test/e2e/framework" |
| |
| . "github.com/onsi/ginkgo" |
| . "github.com/onsi/gomega" |
| ) |
| |
| func setDesiredConfiguration(initialConfig *kubeletconfig.KubeletConfiguration) { |
| initialConfig.EnforceNodeAllocatable = []string{"pods", kubeReservedCgroup, systemReservedCgroup} |
| initialConfig.SystemReserved = map[string]string{ |
| string(v1.ResourceCPU): "100m", |
| string(v1.ResourceMemory): "100Mi", |
| } |
| initialConfig.KubeReserved = map[string]string{ |
| string(v1.ResourceCPU): "100m", |
| string(v1.ResourceMemory): "100Mi", |
| } |
| initialConfig.EvictionHard = map[string]string{"memory.available": "100Mi"} |
| // Necessary for allocatable cgroup creation. |
| initialConfig.CgroupsPerQOS = true |
| initialConfig.KubeReservedCgroup = kubeReservedCgroup |
| initialConfig.SystemReservedCgroup = systemReservedCgroup |
| } |
| |
| var _ = framework.KubeDescribe("Node Container Manager [Serial]", func() { |
| f := framework.NewDefaultFramework("node-container-manager") |
| Describe("Validate Node Allocatable [NodeFeature:NodeAllocatable]", func() { |
| It("set's up the node and runs the test", func() { |
| framework.ExpectNoError(runTest(f)) |
| }) |
| }) |
| |
| }) |
| |
| func expectFileValToEqual(filePath string, expectedValue, delta int64) error { |
| out, err := ioutil.ReadFile(filePath) |
| if err != nil { |
| return fmt.Errorf("failed to read file %q", filePath) |
| } |
| actual, err := strconv.ParseInt(strings.TrimSpace(string(out)), 10, 64) |
| if err != nil { |
| return fmt.Errorf("failed to parse output %v", err) |
| } |
| |
| // Ensure that values are within a delta range to work arounding rounding errors. |
| if (actual < (expectedValue - delta)) || (actual > (expectedValue + delta)) { |
| return fmt.Errorf("Expected value at %q to be between %d and %d. Got %d", filePath, (expectedValue - delta), (expectedValue + delta), actual) |
| } |
| return nil |
| } |
| |
| func getAllocatableLimits(cpu, memory string, capacity v1.ResourceList) (*resource.Quantity, *resource.Quantity) { |
| var allocatableCPU, allocatableMemory *resource.Quantity |
| // Total cpu reservation is 200m. |
| for k, v := range capacity { |
| if k == v1.ResourceCPU { |
| allocatableCPU = v.Copy() |
| allocatableCPU.Sub(resource.MustParse(cpu)) |
| } |
| if k == v1.ResourceMemory { |
| allocatableMemory = v.Copy() |
| allocatableMemory.Sub(resource.MustParse(memory)) |
| } |
| } |
| return allocatableCPU, allocatableMemory |
| } |
| |
| const ( |
| kubeReservedCgroup = "kube-reserved" |
| systemReservedCgroup = "system-reserved" |
| ) |
| |
| func createIfNotExists(cm cm.CgroupManager, cgroupConfig *cm.CgroupConfig) error { |
| if !cm.Exists(cgroupConfig.Name) { |
| if err := cm.Create(cgroupConfig); err != nil { |
| return err |
| } |
| } |
| return nil |
| } |
| |
| func createTemporaryCgroupsForReservation(cgroupManager cm.CgroupManager) error { |
| // Create kube reserved cgroup |
| cgroupConfig := &cm.CgroupConfig{ |
| Name: cm.NewCgroupName(cm.RootCgroupName, kubeReservedCgroup), |
| } |
| if err := createIfNotExists(cgroupManager, cgroupConfig); err != nil { |
| return err |
| } |
| // Create system reserved cgroup |
| cgroupConfig.Name = cm.NewCgroupName(cm.RootCgroupName, systemReservedCgroup) |
| |
| return createIfNotExists(cgroupManager, cgroupConfig) |
| } |
| |
| func destroyTemporaryCgroupsForReservation(cgroupManager cm.CgroupManager) error { |
| // Create kube reserved cgroup |
| cgroupConfig := &cm.CgroupConfig{ |
| Name: cm.NewCgroupName(cm.RootCgroupName, kubeReservedCgroup), |
| } |
| if err := cgroupManager.Destroy(cgroupConfig); err != nil { |
| return err |
| } |
| cgroupConfig.Name = cm.NewCgroupName(cm.RootCgroupName, systemReservedCgroup) |
| return cgroupManager.Destroy(cgroupConfig) |
| } |
| |
| func runTest(f *framework.Framework) error { |
| var oldCfg *kubeletconfig.KubeletConfiguration |
| subsystems, err := cm.GetCgroupSubsystems() |
| if err != nil { |
| return err |
| } |
| // Get current kubelet configuration |
| oldCfg, err = getCurrentKubeletConfig() |
| if err != nil { |
| return err |
| } |
| |
| // Create a cgroup manager object for manipulating cgroups. |
| cgroupManager := cm.NewCgroupManager(subsystems, oldCfg.CgroupDriver) |
| |
| defer destroyTemporaryCgroupsForReservation(cgroupManager) |
| defer func() { |
| if oldCfg != nil { |
| framework.ExpectNoError(setKubeletConfiguration(f, oldCfg)) |
| } |
| }() |
| if err := createTemporaryCgroupsForReservation(cgroupManager); err != nil { |
| return err |
| } |
| newCfg := oldCfg.DeepCopy() |
| // Change existing kubelet configuration |
| setDesiredConfiguration(newCfg) |
| // Set the new kubelet configuration. |
| err = setKubeletConfiguration(f, newCfg) |
| if err != nil { |
| return err |
| } |
| // Set new config and current config. |
| currentConfig := newCfg |
| |
| expectedNAPodCgroup := cm.ParseCgroupfsToCgroupName(currentConfig.CgroupRoot) |
| expectedNAPodCgroup = cm.NewCgroupName(expectedNAPodCgroup, "kubepods") |
| if !cgroupManager.Exists(expectedNAPodCgroup) { |
| return fmt.Errorf("Expected Node Allocatable Cgroup Does not exist") |
| } |
| // TODO: Update cgroupManager to expose a Status interface to get current Cgroup Settings. |
| // The node may not have updated capacity and allocatable yet, so check that it happens eventually. |
| Eventually(func() error { |
| nodeList, err := f.ClientSet.CoreV1().Nodes().List(metav1.ListOptions{}) |
| if err != nil { |
| return err |
| } |
| if len(nodeList.Items) != 1 { |
| return fmt.Errorf("Unexpected number of node objects for node e2e. Expects only one node: %+v", nodeList) |
| } |
| node := nodeList.Items[0] |
| capacity := node.Status.Capacity |
| allocatableCPU, allocatableMemory := getAllocatableLimits("200m", "200Mi", capacity) |
| // Total Memory reservation is 200Mi excluding eviction thresholds. |
| // Expect CPU shares on node allocatable cgroup to equal allocatable. |
| if err := expectFileValToEqual(filepath.Join(subsystems.MountPoints["cpu"], "kubepods", "cpu.shares"), int64(cm.MilliCPUToShares(allocatableCPU.MilliValue())), 10); err != nil { |
| return err |
| } |
| // Expect Memory limit on node allocatable cgroup to equal allocatable. |
| if err := expectFileValToEqual(filepath.Join(subsystems.MountPoints["memory"], "kubepods", "memory.limit_in_bytes"), allocatableMemory.Value(), 0); err != nil { |
| return err |
| } |
| |
| // Check that Allocatable reported to scheduler includes eviction thresholds. |
| schedulerAllocatable := node.Status.Allocatable |
| // Memory allocatable should take into account eviction thresholds. |
| allocatableCPU, allocatableMemory = getAllocatableLimits("200m", "300Mi", capacity) |
| // Expect allocatable to include all resources in capacity. |
| if len(schedulerAllocatable) != len(capacity) { |
| return fmt.Errorf("Expected all resources in capacity to be found in allocatable") |
| } |
| // CPU based evictions are not supported. |
| if allocatableCPU.Cmp(schedulerAllocatable[v1.ResourceCPU]) != 0 { |
| return fmt.Errorf("Unexpected cpu allocatable value exposed by the node. Expected: %v, got: %v, capacity: %v", allocatableCPU, schedulerAllocatable[v1.ResourceCPU], capacity[v1.ResourceCPU]) |
| } |
| if allocatableMemory.Cmp(schedulerAllocatable[v1.ResourceMemory]) != 0 { |
| return fmt.Errorf("Unexpected memory allocatable value exposed by the node. Expected: %v, got: %v, capacity: %v", allocatableMemory, schedulerAllocatable[v1.ResourceMemory], capacity[v1.ResourceMemory]) |
| } |
| return nil |
| }, time.Minute, 5*time.Second).Should(BeNil()) |
| |
| kubeReservedCgroupName := cm.NewCgroupName(cm.RootCgroupName, kubeReservedCgroup) |
| if !cgroupManager.Exists(kubeReservedCgroupName) { |
| return fmt.Errorf("Expected kube reserved cgroup Does not exist") |
| } |
| // Expect CPU shares on kube reserved cgroup to equal it's reservation which is `100m`. |
| kubeReservedCPU := resource.MustParse(currentConfig.KubeReserved[string(v1.ResourceCPU)]) |
| if err := expectFileValToEqual(filepath.Join(subsystems.MountPoints["cpu"], cgroupManager.Name(kubeReservedCgroupName), "cpu.shares"), int64(cm.MilliCPUToShares(kubeReservedCPU.MilliValue())), 10); err != nil { |
| return err |
| } |
| // Expect Memory limit kube reserved cgroup to equal configured value `100Mi`. |
| kubeReservedMemory := resource.MustParse(currentConfig.KubeReserved[string(v1.ResourceMemory)]) |
| if err := expectFileValToEqual(filepath.Join(subsystems.MountPoints["memory"], cgroupManager.Name(kubeReservedCgroupName), "memory.limit_in_bytes"), kubeReservedMemory.Value(), 0); err != nil { |
| return err |
| } |
| systemReservedCgroupName := cm.NewCgroupName(cm.RootCgroupName, systemReservedCgroup) |
| if !cgroupManager.Exists(systemReservedCgroupName) { |
| return fmt.Errorf("Expected system reserved cgroup Does not exist") |
| } |
| // Expect CPU shares on system reserved cgroup to equal it's reservation which is `100m`. |
| systemReservedCPU := resource.MustParse(currentConfig.SystemReserved[string(v1.ResourceCPU)]) |
| if err := expectFileValToEqual(filepath.Join(subsystems.MountPoints["cpu"], cgroupManager.Name(systemReservedCgroupName), "cpu.shares"), int64(cm.MilliCPUToShares(systemReservedCPU.MilliValue())), 10); err != nil { |
| return err |
| } |
| // Expect Memory limit on node allocatable cgroup to equal allocatable. |
| systemReservedMemory := resource.MustParse(currentConfig.SystemReserved[string(v1.ResourceMemory)]) |
| if err := expectFileValToEqual(filepath.Join(subsystems.MountPoints["memory"], cgroupManager.Name(systemReservedCgroupName), "memory.limit_in_bytes"), systemReservedMemory.Value(), 0); err != nil { |
| return err |
| } |
| return nil |
| } |