container_kill.go

Documentation: github.com/chaos-mesh/chaos-mesh/test/e2e/chaos/podchaos

     1  // Copyright 2020 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package podchaos
    15  
    16  import (
    17  	"context"
    18  	"time"
    19  
    20  	. "github.com/onsi/ginkgo"
    21  	corev1 "k8s.io/api/core/v1"
    22  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    23  	"k8s.io/apimachinery/pkg/labels"
    24  	"k8s.io/apimachinery/pkg/types"
    25  	"k8s.io/apimachinery/pkg/util/wait"
    26  	"k8s.io/client-go/kubernetes"
    27  	"k8s.io/kubernetes/test/e2e/framework"
    28  	"k8s.io/utils/pointer"
    29  	"sigs.k8s.io/controller-runtime/pkg/client"
    30  
    31  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    32  	"github.com/chaos-mesh/chaos-mesh/test/e2e/util"
    33  	"github.com/chaos-mesh/chaos-mesh/test/pkg/fixture"
    34  )
    35  
    36  func TestcaseContainerKillOnceThenDelete(ns string, kubeCli kubernetes.Interface, cli client.Client) {
    37  	ctx, cancel := context.WithCancel(context.Background())
    38  	defer cancel()
    39  
    40  	nd := fixture.NewCommonNginxDeployment("nginx", ns, 1)
    41  	_, err := kubeCli.AppsV1().Deployments(ns).Create(nd)
    42  	framework.ExpectNoError(err, "create nginx deployment error")
    43  	err = util.WaitDeploymentReady("nginx", ns, kubeCli)
    44  	framework.ExpectNoError(err, "wait nginx deployment ready error")
    45  
    46  	containerKillChaos := &v1alpha1.PodChaos{
    47  		ObjectMeta: metav1.ObjectMeta{
    48  			Name:      "nginx-container-kill",
    49  			Namespace: ns,
    50  		},
    51  		Spec: v1alpha1.PodChaosSpec{
    52  			Selector: v1alpha1.SelectorSpec{
    53  				Namespaces: []string{
    54  					ns,
    55  				},
    56  				LabelSelectors: map[string]string{
    57  					"app": "nginx",
    58  				},
    59  			},
    60  			Action:        v1alpha1.ContainerKillAction,
    61  			Mode:          v1alpha1.OnePodMode,
    62  			ContainerName: "nginx",
    63  			Scheduler: &v1alpha1.SchedulerSpec{
    64  				Cron: "@every 10s",
    65  			},
    66  		},
    67  	}
    68  	err = cli.Create(ctx, containerKillChaos)
    69  	framework.ExpectNoError(err, "create container kill chaos error")
    70  
    71  	err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (done bool, err error) {
    72  		listOption := metav1.ListOptions{
    73  			LabelSelector: labels.SelectorFromSet(map[string]string{
    74  				"app": "nginx",
    75  			}).String(),
    76  		}
    77  		pods, err := kubeCli.CoreV1().Pods(ns).List(listOption)
    78  		if err != nil {
    79  			return false, nil
    80  		}
    81  		if len(pods.Items) != 1 {
    82  			return false, nil
    83  		}
    84  		pod := pods.Items[0]
    85  		for _, cs := range pod.Status.ContainerStatuses {
    86  			if cs.Name == "nginx" && cs.Ready == false && cs.LastTerminationState.Terminated != nil {
    87  				return true, nil
    88  			}
    89  		}
    90  		return false, nil
    91  	})
    92  
    93  	err = cli.Delete(ctx, containerKillChaos)
    94  	framework.ExpectNoError(err, "failed to delete container kill chaos")
    95  
    96  	By("success to perform container kill")
    97  	err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (done bool, err error) {
    98  		listOption := metav1.ListOptions{
    99  			LabelSelector: labels.SelectorFromSet(map[string]string{
   100  				"app": "nginx",
   101  			}).String(),
   102  		}
   103  		pods, err := kubeCli.CoreV1().Pods(ns).List(listOption)
   104  		if err != nil {
   105  			return false, nil
   106  		}
   107  		if len(pods.Items) != 1 {
   108  			return false, nil
   109  		}
   110  		pod := pods.Items[0]
   111  		for _, cs := range pod.Status.ContainerStatuses {
   112  			if cs.Name == "nginx" && cs.Ready == true && cs.State.Running != nil {
   113  				return true, nil
   114  			}
   115  		}
   116  		return false, nil
   117  	})
   118  	framework.ExpectNoError(err, "container kill recover failed")
   119  
   120  }
   121  
   122  func TestcaseContainerKillPauseThenUnPause(ns string, kubeCli kubernetes.Interface, cli client.Client) {
   123  	ctx, cancel := context.WithCancel(context.Background())
   124  	defer cancel()
   125  	nd := fixture.NewCommonNginxDeployment("nginx", ns, 1)
   126  	_, err := kubeCli.AppsV1().Deployments(ns).Create(nd)
   127  	framework.ExpectNoError(err, "create nginx deployment error")
   128  	err = util.WaitDeploymentReady("nginx", ns, kubeCli)
   129  	framework.ExpectNoError(err, "wait nginx deployment ready error")
   130  
   131  	var pods *corev1.PodList
   132  	var newPods *corev1.PodList
   133  	listOption := metav1.ListOptions{
   134  		LabelSelector: labels.SelectorFromSet(map[string]string{
   135  			"app": "nginx",
   136  		}).String(),
   137  	}
   138  	pods, err = kubeCli.CoreV1().Pods(ns).List(listOption)
   139  	framework.ExpectNoError(err, "get nginx pods error")
   140  
   141  	// Get the running nginx container ID
   142  	containerID := pods.Items[0].Status.ContainerStatuses[0].ContainerID
   143  
   144  	containerKillChaos := &v1alpha1.PodChaos{
   145  		ObjectMeta: metav1.ObjectMeta{
   146  			Name:      "nginx-container-kill",
   147  			Namespace: ns,
   148  		},
   149  		Spec: v1alpha1.PodChaosSpec{
   150  			Selector: v1alpha1.SelectorSpec{
   151  				Namespaces: []string{
   152  					ns,
   153  				},
   154  				LabelSelectors: map[string]string{
   155  					"app": "nginx",
   156  				},
   157  			},
   158  			Action:        v1alpha1.ContainerKillAction,
   159  			Mode:          v1alpha1.OnePodMode,
   160  			ContainerName: "nginx",
   161  			Duration:      pointer.StringPtr("9m"),
   162  			Scheduler: &v1alpha1.SchedulerSpec{
   163  				Cron: "@every 10m",
   164  			},
   165  		},
   166  	}
   167  	err = cli.Create(ctx, containerKillChaos)
   168  	framework.ExpectNoError(err, "create container kill chaos error")
   169  
   170  	chaosKey := types.NamespacedName{
   171  		Namespace: ns,
   172  		Name:      "nginx-container-kill",
   173  	}
   174  
   175  	// nginx container is killed as expected
   176  	err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   177  		newPods, err = kubeCli.CoreV1().Pods(ns).List(listOption)
   178  		framework.ExpectNoError(err, "get nginx pods error")
   179  		return containerID != newPods.Items[0].Status.ContainerStatuses[0].ContainerID, nil
   180  	})
   181  	framework.ExpectNoError(err, "wait container kill failed")
   182  
   183  	// pause experiment
   184  	err = util.PauseChaos(ctx, cli, containerKillChaos)
   185  	framework.ExpectNoError(err, "pause chaos error")
   186  
   187  	err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   188  		chaos := &v1alpha1.PodChaos{}
   189  		err = cli.Get(ctx, chaosKey, chaos)
   190  		framework.ExpectNoError(err, "get pod chaos error")
   191  		if chaos.Status.Experiment.Phase == v1alpha1.ExperimentPhasePaused {
   192  			return true, nil
   193  		}
   194  		return false, err
   195  	})
   196  	framework.ExpectNoError(err, "check paused chaos failed")
   197  
   198  	// wait for 1 minutes and check whether nginx container will be killed or not
   199  	pods, err = kubeCli.CoreV1().Pods(ns).List(listOption)
   200  	framework.ExpectNoError(err, "get nginx pods error")
   201  	containerID = pods.Items[0].Status.ContainerStatuses[0].ContainerID
   202  	err = wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) {
   203  		newPods, err = kubeCli.CoreV1().Pods(ns).List(listOption)
   204  		framework.ExpectNoError(err, "get nginx pods error")
   205  		return containerID != newPods.Items[0].Status.ContainerStatuses[0].ContainerID, nil
   206  	})
   207  	framework.ExpectError(err, "wait container not killed failed")
   208  	framework.ExpectEqual(err.Error(), wait.ErrWaitTimeout.Error())
   209  
   210  	// resume experiment
   211  	err = util.UnPauseChaos(ctx, cli, containerKillChaos)
   212  	framework.ExpectNoError(err, "resume chaos error")
   213  
   214  	err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   215  		chaos := &v1alpha1.PodChaos{}
   216  		err = cli.Get(ctx, chaosKey, chaos)
   217  		framework.ExpectNoError(err, "get pod chaos error")
   218  		if chaos.Status.Experiment.Phase == v1alpha1.ExperimentPhaseRunning {
   219  			return true, nil
   220  		}
   221  		return false, err
   222  	})
   223  	framework.ExpectNoError(err, "check resumed chaos failed")
   224  
   225  	// nginx container is killed by resumed experiment
   226  	pods, err = kubeCli.CoreV1().Pods(ns).List(listOption)
   227  	framework.ExpectNoError(err, "get nginx pods error")
   228  	containerID = pods.Items[0].Status.ContainerStatuses[0].ContainerID
   229  	err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   230  		newPods, err = kubeCli.CoreV1().Pods(ns).List(listOption)
   231  		framework.ExpectNoError(err, "get nginx pods error")
   232  		return containerID != newPods.Items[0].Status.ContainerStatuses[0].ContainerID, nil
   233  	})
   234  	framework.ExpectNoError(err, "wait container killed failed")
   235  
   236  }
   237