io_delay.go

Documentation: github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/iochaos

     1  // Copyright 2020 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package iochaos
    15  
    16  import (
    17  	"context"
    18  	"net/http"
    19  	"time"
    20  
    21  	. "github.com/onsi/ginkgo"
    22  	corev1 "k8s.io/api/core/v1"
    23  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    24  	"k8s.io/apimachinery/pkg/types"
    25  	"k8s.io/apimachinery/pkg/util/wait"
    26  	"k8s.io/klog"
    27  	"k8s.io/kubernetes/test/e2e/framework"
    28  	"k8s.io/utils/pointer"
    29  	"sigs.k8s.io/controller-runtime/pkg/client"
    30  
    31  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    32  	"github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/util"
    33  )
    34  
    35  func TestcaseIODelayDurationForATimeThenRecover(
    36  	ns string,
    37  	cli client.Client,
    38  	c http.Client,
    39  	port uint16,
    40  ) {
    41  	ctx, cancel := context.WithCancel(context.Background())
    42  	defer cancel()
    43  
    44  	By("waiting on e2e helper ready")
    45  	err := util.WaitE2EHelperReady(c, port)
    46  	framework.ExpectNoError(err, "wait e2e helper ready error")
    47  	By("create IO delay chaos CRD objects")
    48  	ioChaos := &v1alpha1.IOChaos{
    49  		ObjectMeta: metav1.ObjectMeta{
    50  			Name:      "io-chaos",
    51  			Namespace: ns,
    52  		},
    53  		Spec: v1alpha1.IOChaosSpec{
    54  			Action:     v1alpha1.IoLatency,
    55  			VolumePath: "/var/run/data",
    56  			Path:       "/var/run/data/*",
    57  			Delay:      "1s",
    58  			Percent:    100,
    59  			ContainerSelector: v1alpha1.ContainerSelector{
    60  				PodSelector: v1alpha1.PodSelector{
    61  					Selector: v1alpha1.PodSelectorSpec{
    62  						Namespaces:     []string{ns},
    63  						LabelSelectors: map[string]string{"app": "io"},
    64  					},
    65  					Mode: v1alpha1.OnePodMode,
    66  				},
    67  			},
    68  		},
    69  	}
    70  	err = cli.Create(ctx, ioChaos)
    71  	framework.ExpectNoError(err, "create io chaos error")
    72  	By("waiting for assertion IO delay")
    73  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
    74  		dur, _ := getPodIODelay(c, port)
    75  		second := dur.Seconds()
    76  		klog.Infof("get io delay %fs", second)
    77  		// IO Delay >= 1s
    78  		if second >= 1 {
    79  			return true, nil
    80  		}
    81  		return false, nil
    82  	})
    83  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
    84  	By("apply io chaos successfully")
    85  
    86  	By("delete chaos CRD objects")
    87  	// delete chaos CRD
    88  	err = cli.Delete(ctx, ioChaos)
    89  	framework.ExpectNoError(err, "failed to delete io chaos")
    90  	By("waiting for assertion recovering")
    91  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
    92  		dur, _ := getPodIODelay(c, port)
    93  		second := dur.Seconds()
    94  		klog.Infof("get io delay %fs", second)
    95  		// IO Delay shouldn't longer than 1s
    96  		if second >= 1 {
    97  			return false, nil
    98  		}
    99  		return true, nil
   100  	})
   101  	framework.ExpectNoError(err, "fail to recover io chaos")
   102  }
   103  
   104  func TestcaseIODelayDurationForATimePauseAndUnPause(
   105  	ns string,
   106  	cli client.Client,
   107  	c http.Client,
   108  	port uint16,
   109  ) {
   110  	ctx, cancel := context.WithCancel(context.Background())
   111  	defer cancel()
   112  	By("waiting for e2e helper ready")
   113  	err := util.WaitE2EHelperReady(c, port)
   114  	framework.ExpectNoError(err, "wait e2e helper ready error")
   115  
   116  	By("create io chaos crd object")
   117  	ioChaos := &v1alpha1.IOChaos{
   118  		ObjectMeta: metav1.ObjectMeta{
   119  			Name:      "io-chaos",
   120  			Namespace: ns,
   121  		},
   122  		Spec: v1alpha1.IOChaosSpec{
   123  			Action:     v1alpha1.IoLatency,
   124  			VolumePath: "/var/run/data",
   125  			Path:       "/var/run/data/*",
   126  			Delay:      "10ms",
   127  			Percent:    100,
   128  			ContainerSelector: v1alpha1.ContainerSelector{
   129  				PodSelector: v1alpha1.PodSelector{
   130  					Selector: v1alpha1.PodSelectorSpec{
   131  						Namespaces:     []string{ns},
   132  						LabelSelectors: map[string]string{"app": "io"},
   133  					},
   134  					Mode: v1alpha1.OnePodMode,
   135  				},
   136  			},
   137  		},
   138  	}
   139  	err = cli.Create(ctx, ioChaos)
   140  	framework.ExpectNoError(err, "error occurs while applying io chaos")
   141  
   142  	chaosKey := types.NamespacedName{
   143  		Namespace: ns,
   144  		Name:      "io-chaos",
   145  	}
   146  
   147  	By("waiting for assertion io chaos")
   148  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   149  		chaos := &v1alpha1.IOChaos{}
   150  		err = cli.Get(ctx, chaosKey, chaos)
   151  		framework.ExpectNoError(err, "get io chaos error")
   152  
   153  		for _, c := range chaos.GetStatus().Conditions {
   154  			if c.Type == v1alpha1.ConditionAllInjected {
   155  				if c.Status != corev1.ConditionTrue {
   156  					return false, nil
   157  				}
   158  			} else if c.Type == v1alpha1.ConditionSelected {
   159  				if c.Status != corev1.ConditionTrue {
   160  					return false, nil
   161  				}
   162  			}
   163  		}
   164  
   165  		dur, _ := getPodIODelay(c, port)
   166  
   167  		ms := dur.Milliseconds()
   168  		klog.Infof("get io delay %dms", ms)
   169  		// IO Delay >= 500ms
   170  		if ms >= 10 {
   171  			return true, nil
   172  		}
   173  		return false, nil
   174  	})
   175  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
   176  
   177  	By("pause io delay chaos experiment")
   178  	// pause experiment
   179  	err = util.PauseChaos(ctx, cli, ioChaos)
   180  	framework.ExpectNoError(err, "pause chaos error")
   181  
   182  	By("waiting for assertion about pause")
   183  	err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   184  		chaos := &v1alpha1.IOChaos{}
   185  		err = cli.Get(ctx, chaosKey, chaos)
   186  		framework.ExpectNoError(err, "get io chaos error")
   187  
   188  		for _, c := range chaos.GetStatus().Conditions {
   189  			if c.Type == v1alpha1.ConditionAllRecovered {
   190  				if c.Status != corev1.ConditionTrue {
   191  					return false, nil
   192  				}
   193  			} else if c.Type == v1alpha1.ConditionSelected {
   194  				if c.Status != corev1.ConditionTrue {
   195  					return false, nil
   196  				}
   197  			}
   198  		}
   199  
   200  		return true, err
   201  	})
   202  	framework.ExpectNoError(err, "check paused chaos failed")
   203  
   204  	// wait 1 min to check whether io delay still exists
   205  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   206  		dur, _ := getPodIODelay(c, port)
   207  
   208  		ms := dur.Milliseconds()
   209  		klog.Infof("get io delay %dms", ms)
   210  		// IO Delay shouldn't longer than 10ms
   211  		if ms > 10 {
   212  			return false, nil
   213  		}
   214  		return true, nil
   215  	})
   216  	framework.ExpectNoError(err, "fail to recover io chaos")
   217  
   218  	By("resume io delay chaos experiment")
   219  	// resume experiment
   220  	err = util.UnPauseChaos(ctx, cli, ioChaos)
   221  	framework.ExpectNoError(err, "resume chaos error")
   222  
   223  	By("assert that io delay is effective again")
   224  	err = wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) {
   225  		chaos := &v1alpha1.IOChaos{}
   226  		err = cli.Get(ctx, chaosKey, chaos)
   227  		framework.ExpectNoError(err, "get io chaos error")
   228  
   229  		for _, c := range chaos.GetStatus().Conditions {
   230  			if c.Type == v1alpha1.ConditionAllInjected {
   231  				if c.Status != corev1.ConditionTrue {
   232  					return false, nil
   233  				}
   234  			} else if c.Type == v1alpha1.ConditionSelected {
   235  				if c.Status != corev1.ConditionTrue {
   236  					return false, nil
   237  				}
   238  			}
   239  		}
   240  
   241  		return true, err
   242  	})
   243  	framework.ExpectNoError(err, "check resumed chaos failed")
   244  
   245  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   246  		dur, _ := getPodIODelay(c, port)
   247  
   248  		ms := dur.Milliseconds()
   249  		klog.Infof("get io delay %dms", ms)
   250  		// IO Delay >= 10ms
   251  		if ms >= 10 {
   252  			return true, nil
   253  		}
   254  		return false, nil
   255  	})
   256  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
   257  
   258  	By("cleanup")
   259  	// cleanup
   260  	cli.Delete(ctx, ioChaos)
   261  }
   262  
   263  func TestcaseIODelayWithSpecifiedContainer(
   264  	ns string,
   265  	cli client.Client,
   266  	c http.Client,
   267  	port uint16) {
   268  	ctx, cancel := context.WithCancel(context.Background())
   269  	defer cancel()
   270  	err := util.WaitE2EHelperReady(c, port)
   271  	framework.ExpectNoError(err, "wait e2e helper ready error")
   272  
   273  	containerName := "io"
   274  	ioChaos := &v1alpha1.IOChaos{
   275  		ObjectMeta: metav1.ObjectMeta{
   276  			Name:      "io-chaos",
   277  			Namespace: ns,
   278  		},
   279  		Spec: v1alpha1.IOChaosSpec{
   280  			Action:     v1alpha1.IoLatency,
   281  			VolumePath: "/var/run/data",
   282  			Path:       "/var/run/data/*",
   283  			Delay:      "10ms",
   284  			Percent:    100,
   285  			ContainerSelector: v1alpha1.ContainerSelector{
   286  				PodSelector: v1alpha1.PodSelector{
   287  					Selector: v1alpha1.PodSelectorSpec{
   288  						Namespaces:     []string{ns},
   289  						LabelSelectors: map[string]string{"app": "io"},
   290  					},
   291  					Mode: v1alpha1.OnePodMode,
   292  				},
   293  				ContainerNames: []string{containerName},
   294  			},
   295  		},
   296  	}
   297  	err = cli.Create(ctx, ioChaos)
   298  	framework.ExpectNoError(err, "create io chaos error")
   299  
   300  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   301  		dur, _ := getPodIODelay(c, port)
   302  
   303  		ms := dur.Milliseconds()
   304  		klog.Infof("get io delay %dms", ms)
   305  		// IO Delay >= 10ms
   306  		if ms >= 10 {
   307  			return true, nil
   308  		}
   309  		return false, nil
   310  	})
   311  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
   312  	klog.Infof("apply io chaos successfully")
   313  
   314  	err = cli.Delete(ctx, ioChaos)
   315  	framework.ExpectNoError(err, "failed to delete io chaos")
   316  
   317  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   318  		dur, _ := getPodIODelay(c, port)
   319  
   320  		ms := dur.Milliseconds()
   321  		klog.Infof("get io delay %dms", ms)
   322  		// IO Delay shouldn't longer than 10ms
   323  		if ms >= 10 {
   324  			return false, nil
   325  		}
   326  		return true, nil
   327  	})
   328  	framework.ExpectNoError(err, "fail to recover io chaos")
   329  }
   330  
   331  func TestcaseIODelayWithWrongSpec(
   332  	ns string,
   333  	cli client.Client,
   334  	c http.Client,
   335  	port uint16,
   336  ) {
   337  	ctx, cancel := context.WithCancel(context.Background())
   338  	defer cancel()
   339  
   340  	By("waiting on e2e helper ready")
   341  	err := util.WaitE2EHelperReady(c, port)
   342  	framework.ExpectNoError(err, "wait e2e helper ready error")
   343  	By("create IO delay chaos CRD objects")
   344  	ioChaos := &v1alpha1.IOChaos{
   345  		ObjectMeta: metav1.ObjectMeta{
   346  			Name:      "io-chaos",
   347  			Namespace: ns,
   348  		},
   349  		Spec: v1alpha1.IOChaosSpec{
   350  			ContainerSelector: v1alpha1.ContainerSelector{
   351  				PodSelector: v1alpha1.PodSelector{
   352  					Selector: v1alpha1.PodSelectorSpec{
   353  						Namespaces:     []string{ns},
   354  						LabelSelectors: map[string]string{"app": "io"},
   355  					},
   356  					Mode: v1alpha1.OnePodMode,
   357  				},
   358  			},
   359  			Action:     v1alpha1.IoLatency,
   360  			VolumePath: "/var/run/data/123",
   361  			Path:       "/var/run/data/*",
   362  			Delay:      "1s",
   363  			Percent:    100,
   364  			Duration:   pointer.StringPtr("9m"),
   365  		},
   366  	}
   367  	err = cli.Create(ctx, ioChaos)
   368  	framework.ExpectNoError(err, "create io chaos error")
   369  	// TODO: resume the e2e test
   370  	// err = wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) {
   371  	// 	err := cli.Get(ctx, types.NamespacedName{Namespace: ioChaos.ObjectMeta.Namespace, Name: ioChaos.ObjectMeta.Name}, ioChaos)
   372  	// 	if err != nil {
   373  	// 		return false, err
   374  	// 	}
   375  	// 	errStr := ioChaos.Status.ChaosStatus.FailedMessage
   376  	// 	klog.Infof("get chaos err: %s", errStr)
   377  	// 	if strings.Contains(errStr, "Toda startup takes too long or an error occurs") {
   378  	// 		return true, nil
   379  	// 	}
   380  	// 	return false, nil
   381  	// })
   382  	// framework.ExpectNoError(err, "A wrong chaos spec should raise an error")
   383  }
   384