pod_failure.go

Documentation: github.com/chaos-mesh/chaos-mesh/test/e2e/chaos/podchaos

     1  // Copyright 2020 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package podchaos
    15  
    16  import (
    17  	"context"
    18  	"time"
    19  
    20  	. "github.com/onsi/ginkgo"
    21  	corev1 "k8s.io/api/core/v1"
    22  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    23  	"k8s.io/apimachinery/pkg/labels"
    24  	"k8s.io/apimachinery/pkg/types"
    25  	"k8s.io/apimachinery/pkg/util/wait"
    26  	"k8s.io/client-go/kubernetes"
    27  	"k8s.io/kubernetes/test/e2e/framework"
    28  	"k8s.io/utils/pointer"
    29  	"sigs.k8s.io/controller-runtime/pkg/client"
    30  
    31  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    32  	"github.com/chaos-mesh/chaos-mesh/test/e2e/e2econst"
    33  	"github.com/chaos-mesh/chaos-mesh/test/e2e/util"
    34  	"github.com/chaos-mesh/chaos-mesh/test/pkg/fixture"
    35  )
    36  
    37  func TestcasePodFailureOnceThenDelete(ns string, kubeCli kubernetes.Interface, cli client.Client) {
    38  	ctx, cancel := context.WithCancel(context.Background())
    39  	defer cancel()
    40  
    41  	By("preparing experiment pods")
    42  	appName := "timer-pod-failure1"
    43  	nd := fixture.NewTimerDeployment(appName, ns)
    44  	_, err := kubeCli.AppsV1().Deployments(ns).Create(nd)
    45  	framework.ExpectNoError(err, "create timer deployment error")
    46  	err = util.WaitDeploymentReady(appName, ns, kubeCli)
    47  	framework.ExpectNoError(err, "wait timer deployment ready error")
    48  
    49  	By("create pod failure chaos CRD objects")
    50  	listOption := metav1.ListOptions{
    51  		LabelSelector: labels.SelectorFromSet(map[string]string{
    52  			"app": appName,
    53  		}).String(),
    54  	}
    55  	podFailureChaos := &v1alpha1.PodChaos{
    56  		ObjectMeta: metav1.ObjectMeta{
    57  			Name:      "timer-failure1",
    58  			Namespace: ns,
    59  		},
    60  		Spec: v1alpha1.PodChaosSpec{
    61  			Selector: v1alpha1.SelectorSpec{
    62  				Namespaces: []string{
    63  					ns,
    64  				},
    65  				LabelSelectors: map[string]string{
    66  					"app": appName,
    67  				},
    68  			},
    69  			Action: v1alpha1.PodFailureAction,
    70  			Mode:   v1alpha1.OnePodMode,
    71  		},
    72  	}
    73  
    74  	err = cli.Create(ctx, podFailureChaos)
    75  	framework.ExpectNoError(err, "create pod failure chaos error")
    76  
    77  	By("waiting for assertion some pod fall into failure")
    78  	err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
    79  		pods, err := kubeCli.CoreV1().Pods(ns).List(listOption)
    80  		if err != nil {
    81  			return false, nil
    82  		}
    83  		if len(pods.Items) != 1 {
    84  			return false, nil
    85  		}
    86  		pod := pods.Items[0]
    87  		for _, c := range pod.Spec.Containers {
    88  			if c.Image == e2econst.PauseImage {
    89  				return true, nil
    90  			}
    91  		}
    92  		return false, nil
    93  	})
    94  	framework.ExpectNoError(err, "failed to verify PodFailure")
    95  
    96  	By("delete pod failure chaos CRD objects")
    97  	err = cli.Delete(ctx, podFailureChaos)
    98  	framework.ExpectNoError(err, "failed to delete pod failure chaos")
    99  
   100  	By("waiting for assertion recovering")
   101  	err = wait.Poll(5*time.Second, 2*time.Minute, func() (done bool, err error) {
   102  		pods, err := kubeCli.CoreV1().Pods(ns).List(listOption)
   103  		if err != nil {
   104  			return false, nil
   105  		}
   106  		if len(pods.Items) != 1 {
   107  			return false, nil
   108  		}
   109  		pod := pods.Items[0]
   110  		for _, c := range pod.Spec.Containers {
   111  			if c.Image == nd.Spec.Template.Spec.Containers[0].Image {
   112  				return true, nil
   113  			}
   114  		}
   115  		return false, nil
   116  	})
   117  	framework.ExpectNoError(err, "pod failure recover failed")
   118  }
   119  
   120  func TestcasePodFailurePauseThenUnPause(ns string, kubeCli kubernetes.Interface, cli client.Client) {
   121  	ctx, cancel := context.WithCancel(context.Background())
   122  	defer cancel()
   123  
   124  	By("preparing experiment pods")
   125  	appName := "timer-pod-failure2"
   126  	nd := fixture.NewTimerDeployment(appName, ns)
   127  	_, err := kubeCli.AppsV1().Deployments(ns).Create(nd)
   128  	framework.ExpectNoError(err, "create timer deployment error")
   129  	err = util.WaitDeploymentReady(appName, ns, kubeCli)
   130  	framework.ExpectNoError(err, "wait timer deployment ready error")
   131  
   132  	By("create pod failure chaos CRD objects")
   133  	var pods *corev1.PodList
   134  	listOption := metav1.ListOptions{
   135  		LabelSelector: labels.SelectorFromSet(map[string]string{
   136  			"app": appName,
   137  		}).String(),
   138  	}
   139  
   140  	podFailureChaos := &v1alpha1.PodChaos{
   141  		ObjectMeta: metav1.ObjectMeta{
   142  			Name:      "timer-failure2",
   143  			Namespace: ns,
   144  		},
   145  		Spec: v1alpha1.PodChaosSpec{
   146  			Selector: v1alpha1.SelectorSpec{
   147  				Namespaces:     []string{ns},
   148  				LabelSelectors: map[string]string{"app": appName},
   149  			},
   150  			Action:   v1alpha1.PodFailureAction,
   151  			Mode:     v1alpha1.OnePodMode,
   152  			Duration: pointer.StringPtr("9m"),
   153  			Scheduler: &v1alpha1.SchedulerSpec{
   154  				Cron: "@every 10m",
   155  			},
   156  		},
   157  	}
   158  	err = cli.Create(ctx, podFailureChaos)
   159  	framework.ExpectNoError(err, "create pod failure chaos error")
   160  	chaosKey := types.NamespacedName{
   161  		Namespace: ns,
   162  		Name:      "timer-failure2",
   163  	}
   164  
   165  	By("waiting for assertion some pod fall into failure")
   166  	// check whether the pod failure chaos succeeded or not
   167  	err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   168  		pods, err := kubeCli.CoreV1().Pods(ns).List(listOption)
   169  		if err != nil {
   170  			return false, nil
   171  		}
   172  		pod := pods.Items[0]
   173  		for _, c := range pod.Spec.Containers {
   174  			if c.Image == e2econst.PauseImage {
   175  				return true, nil
   176  			}
   177  		}
   178  		return false, nil
   179  	})
   180  	framework.ExpectNoError(err, "image not update to pause")
   181  
   182  	// pause experiment
   183  	By("pause pod failure chaos")
   184  	err = util.PauseChaos(ctx, cli, podFailureChaos)
   185  	framework.ExpectNoError(err, "pause chaos error")
   186  
   187  	By("waiting for assertion about chaos experiment paused")
   188  	err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   189  		chaos := &v1alpha1.PodChaos{}
   190  		err = cli.Get(ctx, chaosKey, chaos)
   191  		framework.ExpectNoError(err, "get pod chaos error")
   192  		if chaos.Status.Experiment.Phase == v1alpha1.ExperimentPhasePaused {
   193  			return true, nil
   194  		}
   195  		return false, err
   196  	})
   197  	framework.ExpectNoError(err, "check paused chaos failed")
   198  
   199  	By("wait for 30 seconds and no pod failure")
   200  	pods, err = kubeCli.CoreV1().Pods(ns).List(listOption)
   201  	framework.ExpectNoError(err, "get timer pod error")
   202  	err = wait.Poll(5*time.Second, 30*time.Second, func() (done bool, err error) {
   203  		pods, err = kubeCli.CoreV1().Pods(ns).List(listOption)
   204  		framework.ExpectNoError(err, "get timer pod error")
   205  		pod := pods.Items[0]
   206  		for _, c := range pod.Spec.Containers {
   207  			if c.Image == e2econst.PauseImage {
   208  				return false, nil
   209  			}
   210  		}
   211  
   212  		return true, nil
   213  	})
   214  	framework.ExpectNoError(err, "check paused chaos failed")
   215  
   216  	By("resume paused chaos experiment")
   217  	err = util.UnPauseChaos(ctx, cli, podFailureChaos)
   218  	framework.ExpectNoError(err, "resume chaos error")
   219  
   220  	By("waiting for assertion about pod failure happens again")
   221  	err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   222  		chaos := &v1alpha1.PodChaos{}
   223  		err = cli.Get(ctx, chaosKey, chaos)
   224  		framework.ExpectNoError(err, "get pod chaos error")
   225  		if chaos.Status.Experiment.Phase == v1alpha1.ExperimentPhaseRunning {
   226  			return true, nil
   227  		}
   228  		return false, err
   229  	})
   230  	framework.ExpectNoError(err, "check resumed chaos failed")
   231  
   232  	By("waiting for assert pod failure happens again")
   233  	err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   234  		pods, err = kubeCli.CoreV1().Pods(ns).List(listOption)
   235  		framework.ExpectNoError(err, "get timer pod error")
   236  		pod := pods.Items[0]
   237  		for _, c := range pod.Spec.Containers {
   238  			if c.Image == e2econst.PauseImage {
   239  				return true, nil
   240  			}
   241  		}
   242  		return false, nil
   243  	})
   244  	framework.ExpectNoError(err, "wait pod failure failed")
   245  }
   246