| /* |
| Copyright 2016 The Kubernetes Authors. |
| |
| Licensed under the Apache License, Version 2.0 (the "License"); |
| you may not use this file except in compliance with the License. |
| You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, software |
| distributed under the License is distributed on an "AS IS" BASIS, |
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| See the License for the specific language governing permissions and |
| limitations under the License. |
| */ |
| |
| package kuberuntime |
| |
| import ( |
| "reflect" |
| "sort" |
| "testing" |
| "time" |
| |
| cadvisorapi "github.com/google/cadvisor/info/v1" |
| "github.com/stretchr/testify/assert" |
| "github.com/stretchr/testify/require" |
| |
| "k8s.io/api/core/v1" |
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| "k8s.io/apimachinery/pkg/types" |
| "k8s.io/apimachinery/pkg/util/sets" |
| "k8s.io/client-go/util/flowcontrol" |
| "k8s.io/kubernetes/pkg/credentialprovider" |
| runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" |
| apitest "k8s.io/kubernetes/pkg/kubelet/apis/cri/testing" |
| kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" |
| containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" |
| ) |
| |
| var ( |
| fakeCreatedAt int64 = 1 |
| ) |
| |
| func createTestRuntimeManager() (*apitest.FakeRuntimeService, *apitest.FakeImageService, *kubeGenericRuntimeManager, error) { |
| return customTestRuntimeManager(&credentialprovider.BasicDockerKeyring{}) |
| } |
| |
| func customTestRuntimeManager(keyring *credentialprovider.BasicDockerKeyring) (*apitest.FakeRuntimeService, *apitest.FakeImageService, *kubeGenericRuntimeManager, error) { |
| fakeRuntimeService := apitest.NewFakeRuntimeService() |
| fakeImageService := apitest.NewFakeImageService() |
| // Only an empty machineInfo is needed here, because in unit test all containers are besteffort, |
| // data in machineInfo is not used. If burstable containers are used in unit test in the future, |
| // we may want to set memory capacity. |
| machineInfo := &cadvisorapi.MachineInfo{} |
| osInterface := &containertest.FakeOS{} |
| manager, err := newFakeKubeRuntimeManager(fakeRuntimeService, fakeImageService, machineInfo, osInterface, &containertest.FakeRuntimeHelper{}, keyring) |
| return fakeRuntimeService, fakeImageService, manager, err |
| } |
| |
| // sandboxTemplate is a sandbox template to create fake sandbox. |
| type sandboxTemplate struct { |
| pod *v1.Pod |
| attempt uint32 |
| createdAt int64 |
| state runtimeapi.PodSandboxState |
| } |
| |
| // containerTemplate is a container template to create fake container. |
| type containerTemplate struct { |
| pod *v1.Pod |
| container *v1.Container |
| containerType kubecontainer.ContainerType |
| sandboxAttempt uint32 |
| attempt int |
| createdAt int64 |
| state runtimeapi.ContainerState |
| } |
| |
| // makeAndSetFakePod is a helper function to create and set one fake sandbox for a pod and |
| // one fake container for each of its container. |
| func makeAndSetFakePod(t *testing.T, m *kubeGenericRuntimeManager, fakeRuntime *apitest.FakeRuntimeService, |
| pod *v1.Pod) (*apitest.FakePodSandbox, []*apitest.FakeContainer) { |
| sandbox := makeFakePodSandbox(t, m, sandboxTemplate{ |
| pod: pod, |
| createdAt: fakeCreatedAt, |
| state: runtimeapi.PodSandboxState_SANDBOX_READY, |
| }) |
| |
| var containers []*apitest.FakeContainer |
| newTemplate := func(c *v1.Container) containerTemplate { |
| return containerTemplate{ |
| pod: pod, |
| container: c, |
| createdAt: fakeCreatedAt, |
| state: runtimeapi.ContainerState_CONTAINER_RUNNING, |
| } |
| } |
| for i := range pod.Spec.Containers { |
| containers = append(containers, makeFakeContainer(t, m, newTemplate(&pod.Spec.Containers[i]))) |
| } |
| for i := range pod.Spec.InitContainers { |
| containers = append(containers, makeFakeContainer(t, m, newTemplate(&pod.Spec.InitContainers[i]))) |
| } |
| |
| fakeRuntime.SetFakeSandboxes([]*apitest.FakePodSandbox{sandbox}) |
| fakeRuntime.SetFakeContainers(containers) |
| return sandbox, containers |
| } |
| |
| // makeFakePodSandbox creates a fake pod sandbox based on a sandbox template. |
| func makeFakePodSandbox(t *testing.T, m *kubeGenericRuntimeManager, template sandboxTemplate) *apitest.FakePodSandbox { |
| config, err := m.generatePodSandboxConfig(template.pod, template.attempt) |
| assert.NoError(t, err, "generatePodSandboxConfig for sandbox template %+v", template) |
| |
| podSandboxID := apitest.BuildSandboxName(config.Metadata) |
| return &apitest.FakePodSandbox{ |
| PodSandboxStatus: runtimeapi.PodSandboxStatus{ |
| Id: podSandboxID, |
| Metadata: config.Metadata, |
| State: template.state, |
| CreatedAt: template.createdAt, |
| Network: &runtimeapi.PodSandboxNetworkStatus{ |
| Ip: apitest.FakePodSandboxIP, |
| }, |
| Labels: config.Labels, |
| }, |
| } |
| } |
| |
| // makeFakePodSandboxes creates a group of fake pod sandboxes based on the sandbox templates. |
| // The function guarantees the order of the fake pod sandboxes is the same with the templates. |
| func makeFakePodSandboxes(t *testing.T, m *kubeGenericRuntimeManager, templates []sandboxTemplate) []*apitest.FakePodSandbox { |
| var fakePodSandboxes []*apitest.FakePodSandbox |
| for _, template := range templates { |
| fakePodSandboxes = append(fakePodSandboxes, makeFakePodSandbox(t, m, template)) |
| } |
| return fakePodSandboxes |
| } |
| |
| // makeFakeContainer creates a fake container based on a container template. |
| func makeFakeContainer(t *testing.T, m *kubeGenericRuntimeManager, template containerTemplate) *apitest.FakeContainer { |
| sandboxConfig, err := m.generatePodSandboxConfig(template.pod, template.sandboxAttempt) |
| assert.NoError(t, err, "generatePodSandboxConfig for container template %+v", template) |
| |
| containerConfig, _, err := m.generateContainerConfig(template.container, template.pod, template.attempt, "", template.container.Image, template.containerType) |
| assert.NoError(t, err, "generateContainerConfig for container template %+v", template) |
| |
| podSandboxID := apitest.BuildSandboxName(sandboxConfig.Metadata) |
| containerID := apitest.BuildContainerName(containerConfig.Metadata, podSandboxID) |
| imageRef := containerConfig.Image.Image |
| return &apitest.FakeContainer{ |
| ContainerStatus: runtimeapi.ContainerStatus{ |
| Id: containerID, |
| Metadata: containerConfig.Metadata, |
| Image: containerConfig.Image, |
| ImageRef: imageRef, |
| CreatedAt: template.createdAt, |
| State: template.state, |
| Labels: containerConfig.Labels, |
| Annotations: containerConfig.Annotations, |
| }, |
| SandboxID: podSandboxID, |
| } |
| } |
| |
| // makeFakeContainers creates a group of fake containers based on the container templates. |
| // The function guarantees the order of the fake containers is the same with the templates. |
| func makeFakeContainers(t *testing.T, m *kubeGenericRuntimeManager, templates []containerTemplate) []*apitest.FakeContainer { |
| var fakeContainers []*apitest.FakeContainer |
| for _, template := range templates { |
| fakeContainers = append(fakeContainers, makeFakeContainer(t, m, template)) |
| } |
| return fakeContainers |
| } |
| |
| // makeTestContainer creates a test api container. |
| func makeTestContainer(name, image string) v1.Container { |
| return v1.Container{ |
| Name: name, |
| Image: image, |
| } |
| } |
| |
| // makeTestPod creates a test api pod. |
| func makeTestPod(podName, podNamespace, podUID string, containers []v1.Container) *v1.Pod { |
| return &v1.Pod{ |
| ObjectMeta: metav1.ObjectMeta{ |
| UID: types.UID(podUID), |
| Name: podName, |
| Namespace: podNamespace, |
| }, |
| Spec: v1.PodSpec{ |
| Containers: containers, |
| }, |
| } |
| } |
| |
| // verifyPods returns true if the two pod slices are equal. |
| func verifyPods(a, b []*kubecontainer.Pod) bool { |
| if len(a) != len(b) { |
| return false |
| } |
| |
| // Sort the containers within a pod. |
| for i := range a { |
| sort.Sort(containersByID(a[i].Containers)) |
| } |
| for i := range b { |
| sort.Sort(containersByID(b[i].Containers)) |
| } |
| |
| // Sort the pods by UID. |
| sort.Sort(podsByID(a)) |
| sort.Sort(podsByID(b)) |
| |
| return reflect.DeepEqual(a, b) |
| } |
| |
| func verifyFakeContainerList(fakeRuntime *apitest.FakeRuntimeService, expected sets.String) (sets.String, bool) { |
| actual := sets.NewString() |
| for _, c := range fakeRuntime.Containers { |
| actual.Insert(c.Id) |
| } |
| return actual, actual.Equal(expected) |
| } |
| |
| // Only extract the fields of interests. |
| type cRecord struct { |
| name string |
| attempt uint32 |
| state runtimeapi.ContainerState |
| } |
| |
| type cRecordList []*cRecord |
| |
| func (b cRecordList) Len() int { return len(b) } |
| func (b cRecordList) Swap(i, j int) { b[i], b[j] = b[j], b[i] } |
| func (b cRecordList) Less(i, j int) bool { |
| if b[i].name != b[j].name { |
| return b[i].name < b[j].name |
| } |
| return b[i].attempt < b[j].attempt |
| } |
| |
| func verifyContainerStatuses(t *testing.T, runtime *apitest.FakeRuntimeService, expected []*cRecord, desc string) { |
| actual := []*cRecord{} |
| for _, cStatus := range runtime.Containers { |
| actual = append(actual, &cRecord{name: cStatus.Metadata.Name, attempt: cStatus.Metadata.Attempt, state: cStatus.State}) |
| } |
| sort.Sort(cRecordList(expected)) |
| sort.Sort(cRecordList(actual)) |
| assert.Equal(t, expected, actual, desc) |
| } |
| |
| func TestNewKubeRuntimeManager(t *testing.T) { |
| _, _, _, err := createTestRuntimeManager() |
| assert.NoError(t, err) |
| } |
| |
| func TestVersion(t *testing.T) { |
| _, _, m, err := createTestRuntimeManager() |
| assert.NoError(t, err) |
| |
| version, err := m.Version() |
| assert.NoError(t, err) |
| assert.Equal(t, kubeRuntimeAPIVersion, version.String()) |
| } |
| |
| func TestContainerRuntimeType(t *testing.T) { |
| _, _, m, err := createTestRuntimeManager() |
| assert.NoError(t, err) |
| |
| runtimeType := m.Type() |
| assert.Equal(t, apitest.FakeRuntimeName, runtimeType) |
| } |
| |
| func TestGetPodStatus(t *testing.T) { |
| fakeRuntime, _, m, err := createTestRuntimeManager() |
| assert.NoError(t, err) |
| |
| containers := []v1.Container{ |
| { |
| Name: "foo1", |
| Image: "busybox", |
| ImagePullPolicy: v1.PullIfNotPresent, |
| }, |
| { |
| Name: "foo2", |
| Image: "busybox", |
| ImagePullPolicy: v1.PullIfNotPresent, |
| }, |
| } |
| pod := &v1.Pod{ |
| ObjectMeta: metav1.ObjectMeta{ |
| UID: "12345678", |
| Name: "foo", |
| Namespace: "new", |
| }, |
| Spec: v1.PodSpec{ |
| Containers: containers, |
| }, |
| } |
| |
| // Set fake sandbox and faked containers to fakeRuntime. |
| makeAndSetFakePod(t, m, fakeRuntime, pod) |
| |
| podStatus, err := m.GetPodStatus(pod.UID, pod.Name, pod.Namespace) |
| assert.NoError(t, err) |
| assert.Equal(t, pod.UID, podStatus.ID) |
| assert.Equal(t, pod.Name, podStatus.Name) |
| assert.Equal(t, pod.Namespace, podStatus.Namespace) |
| assert.Equal(t, apitest.FakePodSandboxIP, podStatus.IP) |
| } |
| |
| func TestGetPods(t *testing.T) { |
| fakeRuntime, _, m, err := createTestRuntimeManager() |
| assert.NoError(t, err) |
| |
| pod := &v1.Pod{ |
| ObjectMeta: metav1.ObjectMeta{ |
| UID: "12345678", |
| Name: "foo", |
| Namespace: "new", |
| }, |
| Spec: v1.PodSpec{ |
| Containers: []v1.Container{ |
| { |
| Name: "foo1", |
| Image: "busybox", |
| }, |
| { |
| Name: "foo2", |
| Image: "busybox", |
| }, |
| }, |
| }, |
| } |
| |
| // Set fake sandbox and fake containers to fakeRuntime. |
| fakeSandbox, fakeContainers := makeAndSetFakePod(t, m, fakeRuntime, pod) |
| |
| // Convert the fakeContainers to kubecontainer.Container |
| containers := make([]*kubecontainer.Container, len(fakeContainers)) |
| for i := range containers { |
| fakeContainer := fakeContainers[i] |
| c, err := m.toKubeContainer(&runtimeapi.Container{ |
| Id: fakeContainer.Id, |
| Metadata: fakeContainer.Metadata, |
| State: fakeContainer.State, |
| Image: fakeContainer.Image, |
| ImageRef: fakeContainer.ImageRef, |
| Labels: fakeContainer.Labels, |
| Annotations: fakeContainer.Annotations, |
| }) |
| if err != nil { |
| t.Fatalf("unexpected error %v", err) |
| } |
| containers[i] = c |
| } |
| // Convert fakeSandbox to kubecontainer.Container |
| sandbox, err := m.sandboxToKubeContainer(&runtimeapi.PodSandbox{ |
| Id: fakeSandbox.Id, |
| Metadata: fakeSandbox.Metadata, |
| State: fakeSandbox.State, |
| CreatedAt: fakeSandbox.CreatedAt, |
| Labels: fakeSandbox.Labels, |
| Annotations: fakeSandbox.Annotations, |
| }) |
| if err != nil { |
| t.Fatalf("unexpected error %v", err) |
| } |
| |
| expected := []*kubecontainer.Pod{ |
| { |
| ID: types.UID("12345678"), |
| Name: "foo", |
| Namespace: "new", |
| Containers: []*kubecontainer.Container{containers[0], containers[1]}, |
| Sandboxes: []*kubecontainer.Container{sandbox}, |
| }, |
| } |
| |
| actual, err := m.GetPods(false) |
| assert.NoError(t, err) |
| |
| if !verifyPods(expected, actual) { |
| t.Errorf("expected %q, got %q", expected, actual) |
| } |
| } |
| |
| func TestGetPodContainerID(t *testing.T) { |
| fakeRuntime, _, m, err := createTestRuntimeManager() |
| assert.NoError(t, err) |
| |
| pod := &v1.Pod{ |
| ObjectMeta: metav1.ObjectMeta{ |
| UID: "12345678", |
| Name: "foo", |
| Namespace: "new", |
| }, |
| Spec: v1.PodSpec{ |
| Containers: []v1.Container{ |
| { |
| Name: "foo1", |
| Image: "busybox", |
| }, |
| { |
| Name: "foo2", |
| Image: "busybox", |
| }, |
| }, |
| }, |
| } |
| // Set fake sandbox and fake containers to fakeRuntime. |
| fakeSandbox, _ := makeAndSetFakePod(t, m, fakeRuntime, pod) |
| |
| // Convert fakeSandbox to kubecontainer.Container |
| sandbox, err := m.sandboxToKubeContainer(&runtimeapi.PodSandbox{ |
| Id: fakeSandbox.Id, |
| Metadata: fakeSandbox.Metadata, |
| State: fakeSandbox.State, |
| CreatedAt: fakeSandbox.CreatedAt, |
| Labels: fakeSandbox.Labels, |
| }) |
| assert.NoError(t, err) |
| |
| expectedPod := &kubecontainer.Pod{ |
| ID: pod.UID, |
| Name: pod.Name, |
| Namespace: pod.Namespace, |
| Containers: []*kubecontainer.Container{}, |
| Sandboxes: []*kubecontainer.Container{sandbox}, |
| } |
| actual, err := m.GetPodContainerID(expectedPod) |
| assert.Equal(t, fakeSandbox.Id, actual.ID) |
| } |
| |
| func TestGetNetNS(t *testing.T) { |
| fakeRuntime, _, m, err := createTestRuntimeManager() |
| assert.NoError(t, err) |
| |
| pod := &v1.Pod{ |
| ObjectMeta: metav1.ObjectMeta{ |
| UID: "12345678", |
| Name: "foo", |
| Namespace: "new", |
| }, |
| Spec: v1.PodSpec{ |
| Containers: []v1.Container{ |
| { |
| Name: "foo1", |
| Image: "busybox", |
| }, |
| { |
| Name: "foo2", |
| Image: "busybox", |
| }, |
| }, |
| }, |
| } |
| |
| // Set fake sandbox and fake containers to fakeRuntime. |
| sandbox, _ := makeAndSetFakePod(t, m, fakeRuntime, pod) |
| |
| actual, err := m.GetNetNS(kubecontainer.ContainerID{ID: sandbox.Id}) |
| assert.Equal(t, "", actual) |
| assert.Equal(t, "not supported", err.Error()) |
| } |
| |
| func TestKillPod(t *testing.T) { |
| fakeRuntime, _, m, err := createTestRuntimeManager() |
| assert.NoError(t, err) |
| |
| pod := &v1.Pod{ |
| ObjectMeta: metav1.ObjectMeta{ |
| UID: "12345678", |
| Name: "foo", |
| Namespace: "new", |
| }, |
| Spec: v1.PodSpec{ |
| Containers: []v1.Container{ |
| { |
| Name: "foo1", |
| Image: "busybox", |
| }, |
| { |
| Name: "foo2", |
| Image: "busybox", |
| }, |
| }, |
| }, |
| } |
| |
| // Set fake sandbox and fake containers to fakeRuntime. |
| fakeSandbox, fakeContainers := makeAndSetFakePod(t, m, fakeRuntime, pod) |
| |
| // Convert the fakeContainers to kubecontainer.Container |
| containers := make([]*kubecontainer.Container, len(fakeContainers)) |
| for i := range containers { |
| fakeContainer := fakeContainers[i] |
| c, err := m.toKubeContainer(&runtimeapi.Container{ |
| Id: fakeContainer.Id, |
| Metadata: fakeContainer.Metadata, |
| State: fakeContainer.State, |
| Image: fakeContainer.Image, |
| ImageRef: fakeContainer.ImageRef, |
| Labels: fakeContainer.Labels, |
| }) |
| if err != nil { |
| t.Fatalf("unexpected error %v", err) |
| } |
| containers[i] = c |
| } |
| runningPod := kubecontainer.Pod{ |
| ID: pod.UID, |
| Name: pod.Name, |
| Namespace: pod.Namespace, |
| Containers: []*kubecontainer.Container{containers[0], containers[1]}, |
| Sandboxes: []*kubecontainer.Container{ |
| { |
| ID: kubecontainer.ContainerID{ |
| ID: fakeSandbox.Id, |
| Type: apitest.FakeRuntimeName, |
| }, |
| }, |
| }, |
| } |
| |
| err = m.KillPod(pod, runningPod, nil) |
| assert.NoError(t, err) |
| assert.Equal(t, 2, len(fakeRuntime.Containers)) |
| assert.Equal(t, 1, len(fakeRuntime.Sandboxes)) |
| for _, sandbox := range fakeRuntime.Sandboxes { |
| assert.Equal(t, runtimeapi.PodSandboxState_SANDBOX_NOTREADY, sandbox.State) |
| } |
| for _, c := range fakeRuntime.Containers { |
| assert.Equal(t, runtimeapi.ContainerState_CONTAINER_EXITED, c.State) |
| } |
| } |
| |
| func TestSyncPod(t *testing.T) { |
| fakeRuntime, fakeImage, m, err := createTestRuntimeManager() |
| assert.NoError(t, err) |
| |
| containers := []v1.Container{ |
| { |
| Name: "foo1", |
| Image: "busybox", |
| ImagePullPolicy: v1.PullIfNotPresent, |
| }, |
| { |
| Name: "foo2", |
| Image: "alpine", |
| ImagePullPolicy: v1.PullIfNotPresent, |
| }, |
| } |
| pod := &v1.Pod{ |
| ObjectMeta: metav1.ObjectMeta{ |
| UID: "12345678", |
| Name: "foo", |
| Namespace: "new", |
| }, |
| Spec: v1.PodSpec{ |
| Containers: containers, |
| }, |
| } |
| |
| backOff := flowcontrol.NewBackOff(time.Second, time.Minute) |
| result := m.SyncPod(pod, v1.PodStatus{}, &kubecontainer.PodStatus{}, []v1.Secret{}, backOff) |
| assert.NoError(t, result.Error()) |
| assert.Equal(t, 2, len(fakeRuntime.Containers)) |
| assert.Equal(t, 2, len(fakeImage.Images)) |
| assert.Equal(t, 1, len(fakeRuntime.Sandboxes)) |
| for _, sandbox := range fakeRuntime.Sandboxes { |
| assert.Equal(t, runtimeapi.PodSandboxState_SANDBOX_READY, sandbox.State) |
| } |
| for _, c := range fakeRuntime.Containers { |
| assert.Equal(t, runtimeapi.ContainerState_CONTAINER_RUNNING, c.State) |
| } |
| } |
| |
| func TestPruneInitContainers(t *testing.T) { |
| fakeRuntime, _, m, err := createTestRuntimeManager() |
| assert.NoError(t, err) |
| |
| init1 := makeTestContainer("init1", "busybox") |
| init2 := makeTestContainer("init2", "busybox") |
| pod := &v1.Pod{ |
| ObjectMeta: metav1.ObjectMeta{ |
| UID: "12345678", |
| Name: "foo", |
| Namespace: "new", |
| }, |
| Spec: v1.PodSpec{ |
| InitContainers: []v1.Container{init1, init2}, |
| }, |
| } |
| |
| templates := []containerTemplate{ |
| {pod: pod, container: &init1, attempt: 2, createdAt: 2, state: runtimeapi.ContainerState_CONTAINER_EXITED}, |
| {pod: pod, container: &init1, attempt: 1, createdAt: 1, state: runtimeapi.ContainerState_CONTAINER_EXITED}, |
| {pod: pod, container: &init2, attempt: 1, createdAt: 1, state: runtimeapi.ContainerState_CONTAINER_EXITED}, |
| {pod: pod, container: &init2, attempt: 0, createdAt: 0, state: runtimeapi.ContainerState_CONTAINER_EXITED}, |
| {pod: pod, container: &init1, attempt: 0, createdAt: 0, state: runtimeapi.ContainerState_CONTAINER_EXITED}, |
| } |
| fakes := makeFakeContainers(t, m, templates) |
| fakeRuntime.SetFakeContainers(fakes) |
| podStatus, err := m.GetPodStatus(pod.UID, pod.Name, pod.Namespace) |
| assert.NoError(t, err) |
| |
| m.pruneInitContainersBeforeStart(pod, podStatus) |
| expectedContainers := sets.NewString(fakes[0].Id, fakes[2].Id) |
| if actual, ok := verifyFakeContainerList(fakeRuntime, expectedContainers); !ok { |
| t.Errorf("expected %v, got %v", expectedContainers, actual) |
| } |
| } |
| |
| func TestSyncPodWithInitContainers(t *testing.T) { |
| fakeRuntime, _, m, err := createTestRuntimeManager() |
| assert.NoError(t, err) |
| |
| initContainers := []v1.Container{ |
| { |
| Name: "init1", |
| Image: "init", |
| ImagePullPolicy: v1.PullIfNotPresent, |
| }, |
| } |
| containers := []v1.Container{ |
| { |
| Name: "foo1", |
| Image: "busybox", |
| ImagePullPolicy: v1.PullIfNotPresent, |
| }, |
| { |
| Name: "foo2", |
| Image: "alpine", |
| ImagePullPolicy: v1.PullIfNotPresent, |
| }, |
| } |
| pod := &v1.Pod{ |
| ObjectMeta: metav1.ObjectMeta{ |
| UID: "12345678", |
| Name: "foo", |
| Namespace: "new", |
| }, |
| Spec: v1.PodSpec{ |
| Containers: containers, |
| InitContainers: initContainers, |
| }, |
| } |
| |
| backOff := flowcontrol.NewBackOff(time.Second, time.Minute) |
| |
| // 1. should only create the init container. |
| podStatus, err := m.GetPodStatus(pod.UID, pod.Name, pod.Namespace) |
| assert.NoError(t, err) |
| result := m.SyncPod(pod, v1.PodStatus{}, podStatus, []v1.Secret{}, backOff) |
| assert.NoError(t, result.Error()) |
| expected := []*cRecord{ |
| {name: initContainers[0].Name, attempt: 0, state: runtimeapi.ContainerState_CONTAINER_RUNNING}, |
| } |
| verifyContainerStatuses(t, fakeRuntime, expected, "start only the init container") |
| |
| // 2. should not create app container because init container is still running. |
| podStatus, err = m.GetPodStatus(pod.UID, pod.Name, pod.Namespace) |
| assert.NoError(t, err) |
| result = m.SyncPod(pod, v1.PodStatus{}, podStatus, []v1.Secret{}, backOff) |
| assert.NoError(t, result.Error()) |
| verifyContainerStatuses(t, fakeRuntime, expected, "init container still running; do nothing") |
| |
| // 3. should create all app containers because init container finished. |
| // Stop init container instance 0. |
| sandboxIDs, err := m.getSandboxIDByPodUID(pod.UID, nil) |
| require.NoError(t, err) |
| sandboxID := sandboxIDs[0] |
| initID0, err := fakeRuntime.GetContainerID(sandboxID, initContainers[0].Name, 0) |
| require.NoError(t, err) |
| fakeRuntime.StopContainer(initID0, 0) |
| // Sync again. |
| podStatus, err = m.GetPodStatus(pod.UID, pod.Name, pod.Namespace) |
| assert.NoError(t, err) |
| result = m.SyncPod(pod, v1.PodStatus{}, podStatus, []v1.Secret{}, backOff) |
| assert.NoError(t, result.Error()) |
| expected = []*cRecord{ |
| {name: initContainers[0].Name, attempt: 0, state: runtimeapi.ContainerState_CONTAINER_EXITED}, |
| {name: containers[0].Name, attempt: 0, state: runtimeapi.ContainerState_CONTAINER_RUNNING}, |
| {name: containers[1].Name, attempt: 0, state: runtimeapi.ContainerState_CONTAINER_RUNNING}, |
| } |
| verifyContainerStatuses(t, fakeRuntime, expected, "init container completed; all app containers should be running") |
| |
| // 4. should restart the init container if needed to create a new podsandbox |
| // Stop the pod sandbox. |
| fakeRuntime.StopPodSandbox(sandboxID) |
| // Sync again. |
| podStatus, err = m.GetPodStatus(pod.UID, pod.Name, pod.Namespace) |
| assert.NoError(t, err) |
| result = m.SyncPod(pod, v1.PodStatus{}, podStatus, []v1.Secret{}, backOff) |
| assert.NoError(t, result.Error()) |
| expected = []*cRecord{ |
| // The first init container instance is purged and no longer visible. |
| // The second (attempt == 1) instance has been started and is running. |
| {name: initContainers[0].Name, attempt: 1, state: runtimeapi.ContainerState_CONTAINER_RUNNING}, |
| // All containers are killed. |
| {name: containers[0].Name, attempt: 0, state: runtimeapi.ContainerState_CONTAINER_EXITED}, |
| {name: containers[1].Name, attempt: 0, state: runtimeapi.ContainerState_CONTAINER_EXITED}, |
| } |
| verifyContainerStatuses(t, fakeRuntime, expected, "kill all app containers, purge the existing init container, and restart a new one") |
| } |
| |
| // A helper function to get a basic pod and its status assuming all sandbox and |
| // containers are running and ready. |
| func makeBasePodAndStatus() (*v1.Pod, *kubecontainer.PodStatus) { |
| pod := &v1.Pod{ |
| ObjectMeta: metav1.ObjectMeta{ |
| UID: "12345678", |
| Name: "foo", |
| Namespace: "foo-ns", |
| }, |
| Spec: v1.PodSpec{ |
| Containers: []v1.Container{ |
| { |
| Name: "foo1", |
| Image: "busybox", |
| }, |
| { |
| Name: "foo2", |
| Image: "busybox", |
| }, |
| { |
| Name: "foo3", |
| Image: "busybox", |
| }, |
| }, |
| }, |
| } |
| status := &kubecontainer.PodStatus{ |
| ID: pod.UID, |
| Name: pod.Name, |
| Namespace: pod.Namespace, |
| SandboxStatuses: []*runtimeapi.PodSandboxStatus{ |
| { |
| Id: "sandboxID", |
| State: runtimeapi.PodSandboxState_SANDBOX_READY, |
| Metadata: &runtimeapi.PodSandboxMetadata{Name: pod.Name, Namespace: pod.Namespace, Uid: "sandboxuid", Attempt: uint32(0)}, |
| Network: &runtimeapi.PodSandboxNetworkStatus{Ip: "10.0.0.1"}, |
| }, |
| }, |
| ContainerStatuses: []*kubecontainer.ContainerStatus{ |
| { |
| ID: kubecontainer.ContainerID{ID: "id1"}, |
| Name: "foo1", State: kubecontainer.ContainerStateRunning, |
| Hash: kubecontainer.HashContainer(&pod.Spec.Containers[0]), |
| }, |
| { |
| ID: kubecontainer.ContainerID{ID: "id2"}, |
| Name: "foo2", State: kubecontainer.ContainerStateRunning, |
| Hash: kubecontainer.HashContainer(&pod.Spec.Containers[1]), |
| }, |
| { |
| ID: kubecontainer.ContainerID{ID: "id3"}, |
| Name: "foo3", State: kubecontainer.ContainerStateRunning, |
| Hash: kubecontainer.HashContainer(&pod.Spec.Containers[2]), |
| }, |
| }, |
| } |
| return pod, status |
| } |
| |
| func TestComputePodActions(t *testing.T) { |
| _, _, m, err := createTestRuntimeManager() |
| require.NoError(t, err) |
| |
| // Createing a pair reference pod and status for the test cases to refer |
| // the specific fields. |
| basePod, baseStatus := makeBasePodAndStatus() |
| noAction := podActions{ |
| SandboxID: baseStatus.SandboxStatuses[0].Id, |
| ContainersToStart: []int{}, |
| ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{}, |
| } |
| |
| for desc, test := range map[string]struct { |
| mutatePodFn func(*v1.Pod) |
| mutateStatusFn func(*kubecontainer.PodStatus) |
| actions podActions |
| }{ |
| "everying is good; do nothing": { |
| actions: noAction, |
| }, |
| "start pod sandbox and all containers for a new pod": { |
| mutateStatusFn: func(status *kubecontainer.PodStatus) { |
| // No container or sandbox exists. |
| status.SandboxStatuses = []*runtimeapi.PodSandboxStatus{} |
| status.ContainerStatuses = []*kubecontainer.ContainerStatus{} |
| }, |
| actions: podActions{ |
| KillPod: true, |
| CreateSandbox: true, |
| Attempt: uint32(0), |
| ContainersToStart: []int{0, 1, 2}, |
| ContainersToKill: getKillMap(basePod, baseStatus, []int{}), |
| }, |
| }, |
| "restart exited containers if RestartPolicy == Always": { |
| mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, |
| mutateStatusFn: func(status *kubecontainer.PodStatus) { |
| // The first container completed, restart it, |
| status.ContainerStatuses[0].State = kubecontainer.ContainerStateExited |
| status.ContainerStatuses[0].ExitCode = 0 |
| |
| // The second container exited with failure, restart it, |
| status.ContainerStatuses[1].State = kubecontainer.ContainerStateExited |
| status.ContainerStatuses[1].ExitCode = 111 |
| }, |
| actions: podActions{ |
| SandboxID: baseStatus.SandboxStatuses[0].Id, |
| ContainersToStart: []int{0, 1}, |
| ContainersToKill: getKillMap(basePod, baseStatus, []int{}), |
| }, |
| }, |
| "restart failed containers if RestartPolicy == OnFailure": { |
| mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure }, |
| mutateStatusFn: func(status *kubecontainer.PodStatus) { |
| // The first container completed, don't restart it, |
| status.ContainerStatuses[0].State = kubecontainer.ContainerStateExited |
| status.ContainerStatuses[0].ExitCode = 0 |
| |
| // The second container exited with failure, restart it, |
| status.ContainerStatuses[1].State = kubecontainer.ContainerStateExited |
| status.ContainerStatuses[1].ExitCode = 111 |
| }, |
| actions: podActions{ |
| SandboxID: baseStatus.SandboxStatuses[0].Id, |
| ContainersToStart: []int{1}, |
| ContainersToKill: getKillMap(basePod, baseStatus, []int{}), |
| }, |
| }, |
| "don't restart containers if RestartPolicy == Never": { |
| mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever }, |
| mutateStatusFn: func(status *kubecontainer.PodStatus) { |
| // Don't restart any containers. |
| status.ContainerStatuses[0].State = kubecontainer.ContainerStateExited |
| status.ContainerStatuses[0].ExitCode = 0 |
| status.ContainerStatuses[1].State = kubecontainer.ContainerStateExited |
| status.ContainerStatuses[1].ExitCode = 111 |
| }, |
| actions: noAction, |
| }, |
| "Kill pod and recreate everything if the pod sandbox is dead, and RestartPolicy == Always": { |
| mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, |
| mutateStatusFn: func(status *kubecontainer.PodStatus) { |
| status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY |
| }, |
| actions: podActions{ |
| KillPod: true, |
| CreateSandbox: true, |
| SandboxID: baseStatus.SandboxStatuses[0].Id, |
| Attempt: uint32(1), |
| ContainersToStart: []int{0, 1, 2}, |
| ContainersToKill: getKillMap(basePod, baseStatus, []int{}), |
| }, |
| }, |
| "Kill pod and recreate all containers (except for the succeeded one) if the pod sandbox is dead, and RestartPolicy == OnFailure": { |
| mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure }, |
| mutateStatusFn: func(status *kubecontainer.PodStatus) { |
| status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY |
| status.ContainerStatuses[1].State = kubecontainer.ContainerStateExited |
| status.ContainerStatuses[1].ExitCode = 0 |
| }, |
| actions: podActions{ |
| KillPod: true, |
| CreateSandbox: true, |
| SandboxID: baseStatus.SandboxStatuses[0].Id, |
| Attempt: uint32(1), |
| ContainersToStart: []int{0, 2}, |
| ContainersToKill: getKillMap(basePod, baseStatus, []int{}), |
| }, |
| }, |
| "Kill pod and recreate all containers if the PodSandbox does not have an IP": { |
| mutateStatusFn: func(status *kubecontainer.PodStatus) { |
| status.SandboxStatuses[0].Network.Ip = "" |
| }, |
| actions: podActions{ |
| KillPod: true, |
| CreateSandbox: true, |
| SandboxID: baseStatus.SandboxStatuses[0].Id, |
| Attempt: uint32(1), |
| ContainersToStart: []int{0, 1, 2}, |
| ContainersToKill: getKillMap(basePod, baseStatus, []int{}), |
| }, |
| }, |
| "Kill and recreate the container if the container's spec changed": { |
| mutatePodFn: func(pod *v1.Pod) { |
| pod.Spec.RestartPolicy = v1.RestartPolicyAlways |
| }, |
| mutateStatusFn: func(status *kubecontainer.PodStatus) { |
| status.ContainerStatuses[1].Hash = uint64(432423432) |
| }, |
| actions: podActions{ |
| SandboxID: baseStatus.SandboxStatuses[0].Id, |
| ContainersToKill: getKillMap(basePod, baseStatus, []int{1}), |
| ContainersToStart: []int{1}, |
| }, |
| // TODO: Add a test case for containers which failed the liveness |
| // check. Will need to fake the livessness check result. |
| }, |
| "Verify we do not create a pod sandbox if no ready sandbox for pod with RestartPolicy=Never and all containers exited": { |
| mutatePodFn: func(pod *v1.Pod) { |
| pod.Spec.RestartPolicy = v1.RestartPolicyNever |
| }, |
| mutateStatusFn: func(status *kubecontainer.PodStatus) { |
| // no ready sandbox |
| status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY |
| status.SandboxStatuses[0].Metadata.Attempt = uint32(1) |
| // all containers exited |
| for i := range status.ContainerStatuses { |
| status.ContainerStatuses[i].State = kubecontainer.ContainerStateExited |
| status.ContainerStatuses[i].ExitCode = 0 |
| } |
| }, |
| actions: podActions{ |
| SandboxID: baseStatus.SandboxStatuses[0].Id, |
| Attempt: uint32(2), |
| CreateSandbox: false, |
| KillPod: true, |
| ContainersToStart: []int{}, |
| ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{}, |
| }, |
| }, |
| } { |
| pod, status := makeBasePodAndStatus() |
| if test.mutatePodFn != nil { |
| test.mutatePodFn(pod) |
| } |
| if test.mutateStatusFn != nil { |
| test.mutateStatusFn(status) |
| } |
| actions := m.computePodActions(pod, status) |
| verifyActions(t, &test.actions, &actions, desc) |
| } |
| } |
| |
| func getKillMap(pod *v1.Pod, status *kubecontainer.PodStatus, cIndexes []int) map[kubecontainer.ContainerID]containerToKillInfo { |
| m := map[kubecontainer.ContainerID]containerToKillInfo{} |
| for _, i := range cIndexes { |
| m[status.ContainerStatuses[i].ID] = containerToKillInfo{ |
| container: &pod.Spec.Containers[i], |
| name: pod.Spec.Containers[i].Name, |
| } |
| } |
| return m |
| } |
| |
| func verifyActions(t *testing.T, expected, actual *podActions, desc string) { |
| if actual.ContainersToKill != nil { |
| // Clear the message field since we don't need to verify the message. |
| for k, info := range actual.ContainersToKill { |
| info.message = "" |
| actual.ContainersToKill[k] = info |
| } |
| } |
| assert.Equal(t, expected, actual, desc) |
| } |
| |
| func TestComputePodActionsWithInitContainers(t *testing.T) { |
| _, _, m, err := createTestRuntimeManager() |
| require.NoError(t, err) |
| |
| // Createing a pair reference pod and status for the test cases to refer |
| // the specific fields. |
| basePod, baseStatus := makeBasePodAndStatusWithInitContainers() |
| noAction := podActions{ |
| SandboxID: baseStatus.SandboxStatuses[0].Id, |
| ContainersToStart: []int{}, |
| ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{}, |
| } |
| |
| for desc, test := range map[string]struct { |
| mutatePodFn func(*v1.Pod) |
| mutateStatusFn func(*kubecontainer.PodStatus) |
| actions podActions |
| }{ |
| "initialization completed; start all containers": { |
| actions: podActions{ |
| SandboxID: baseStatus.SandboxStatuses[0].Id, |
| ContainersToStart: []int{0, 1, 2}, |
| ContainersToKill: getKillMap(basePod, baseStatus, []int{}), |
| }, |
| }, |
| "initialization in progress; do nothing": { |
| mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, |
| mutateStatusFn: func(status *kubecontainer.PodStatus) { |
| status.ContainerStatuses[2].State = kubecontainer.ContainerStateRunning |
| }, |
| actions: noAction, |
| }, |
| "Kill pod and restart the first init container if the pod sandbox is dead": { |
| mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, |
| mutateStatusFn: func(status *kubecontainer.PodStatus) { |
| status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY |
| }, |
| actions: podActions{ |
| KillPod: true, |
| CreateSandbox: true, |
| SandboxID: baseStatus.SandboxStatuses[0].Id, |
| Attempt: uint32(1), |
| NextInitContainerToStart: &basePod.Spec.InitContainers[0], |
| ContainersToStart: []int{}, |
| ContainersToKill: getKillMap(basePod, baseStatus, []int{}), |
| }, |
| }, |
| "initialization failed; restart the last init container if RestartPolicy == Always": { |
| mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, |
| mutateStatusFn: func(status *kubecontainer.PodStatus) { |
| status.ContainerStatuses[2].ExitCode = 137 |
| }, |
| actions: podActions{ |
| SandboxID: baseStatus.SandboxStatuses[0].Id, |
| NextInitContainerToStart: &basePod.Spec.InitContainers[2], |
| ContainersToStart: []int{}, |
| ContainersToKill: getKillMap(basePod, baseStatus, []int{}), |
| }, |
| }, |
| "initialization failed; restart the last init container if RestartPolicy == OnFailure": { |
| mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure }, |
| mutateStatusFn: func(status *kubecontainer.PodStatus) { |
| status.ContainerStatuses[2].ExitCode = 137 |
| }, |
| actions: podActions{ |
| SandboxID: baseStatus.SandboxStatuses[0].Id, |
| NextInitContainerToStart: &basePod.Spec.InitContainers[2], |
| ContainersToStart: []int{}, |
| ContainersToKill: getKillMap(basePod, baseStatus, []int{}), |
| }, |
| }, |
| "initialization failed; kill pod if RestartPolicy == Never": { |
| mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever }, |
| mutateStatusFn: func(status *kubecontainer.PodStatus) { |
| status.ContainerStatuses[2].ExitCode = 137 |
| }, |
| actions: podActions{ |
| KillPod: true, |
| SandboxID: baseStatus.SandboxStatuses[0].Id, |
| ContainersToStart: []int{}, |
| ContainersToKill: getKillMap(basePod, baseStatus, []int{}), |
| }, |
| }, |
| } { |
| pod, status := makeBasePodAndStatusWithInitContainers() |
| if test.mutatePodFn != nil { |
| test.mutatePodFn(pod) |
| } |
| if test.mutateStatusFn != nil { |
| test.mutateStatusFn(status) |
| } |
| actions := m.computePodActions(pod, status) |
| verifyActions(t, &test.actions, &actions, desc) |
| } |
| } |
| |
| func makeBasePodAndStatusWithInitContainers() (*v1.Pod, *kubecontainer.PodStatus) { |
| pod, status := makeBasePodAndStatus() |
| pod.Spec.InitContainers = []v1.Container{ |
| { |
| Name: "init1", |
| Image: "bar-image", |
| }, |
| { |
| Name: "init2", |
| Image: "bar-image", |
| }, |
| { |
| Name: "init3", |
| Image: "bar-image", |
| }, |
| } |
| // Replace the original statuses of the containers with those for the init |
| // containers. |
| status.ContainerStatuses = []*kubecontainer.ContainerStatus{ |
| { |
| ID: kubecontainer.ContainerID{ID: "initid1"}, |
| Name: "init1", State: kubecontainer.ContainerStateExited, |
| Hash: kubecontainer.HashContainer(&pod.Spec.InitContainers[0]), |
| }, |
| { |
| ID: kubecontainer.ContainerID{ID: "initid2"}, |
| Name: "init2", State: kubecontainer.ContainerStateExited, |
| Hash: kubecontainer.HashContainer(&pod.Spec.InitContainers[0]), |
| }, |
| { |
| ID: kubecontainer.ContainerID{ID: "initid3"}, |
| Name: "init3", State: kubecontainer.ContainerStateExited, |
| Hash: kubecontainer.HashContainer(&pod.Spec.InitContainers[0]), |
| }, |
| } |
| return pod, status |
| } |