io_delay.go

Documentation: github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/iochaos

     1  // Copyright 2021 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  // http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  
    16  package iochaos
    17  
    18  import (
    19  	"context"
    20  	"net/http"
    21  	"time"
    22  
    23  	. "github.com/onsi/ginkgo/v2"
    24  	corev1 "k8s.io/api/core/v1"
    25  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    26  	"k8s.io/apimachinery/pkg/types"
    27  	"k8s.io/apimachinery/pkg/util/wait"
    28  	"k8s.io/klog/v2"
    29  	"k8s.io/kubernetes/test/e2e/framework"
    30  	"k8s.io/utils/pointer"
    31  	"sigs.k8s.io/controller-runtime/pkg/client"
    32  
    33  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    34  	"github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/util"
    35  )
    36  
    37  func TestcaseIODelayDurationForATimeThenRecover(
    38  	ns string,
    39  	cli client.Client,
    40  	c http.Client,
    41  	port uint16,
    42  ) {
    43  	ctx, cancel := context.WithCancel(context.Background())
    44  	defer cancel()
    45  
    46  	By("waiting on e2e helper ready")
    47  	err := util.WaitE2EHelperReady(c, port)
    48  	framework.ExpectNoError(err, "wait e2e helper ready error")
    49  	By("create IO delay chaos CRD objects")
    50  	ioChaos := &v1alpha1.IOChaos{
    51  		ObjectMeta: metav1.ObjectMeta{
    52  			Name:      "io-chaos",
    53  			Namespace: ns,
    54  		},
    55  		Spec: v1alpha1.IOChaosSpec{
    56  			Action:     v1alpha1.IoLatency,
    57  			VolumePath: "/var/run/data",
    58  			Path:       "/var/run/data/*",
    59  			Delay:      "1s",
    60  			Percent:    100,
    61  			ContainerSelector: v1alpha1.ContainerSelector{
    62  				PodSelector: v1alpha1.PodSelector{
    63  					Selector: v1alpha1.PodSelectorSpec{
    64  						GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
    65  							Namespaces:     []string{ns},
    66  							LabelSelectors: map[string]string{"app": "io"},
    67  						},
    68  					},
    69  					Mode: v1alpha1.OneMode,
    70  				},
    71  			},
    72  		},
    73  	}
    74  	err = cli.Create(ctx, ioChaos)
    75  	framework.ExpectNoError(err, "create io chaos error")
    76  	By("waiting for assertion IO delay")
    77  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
    78  		dur, _ := getPodIODelay(c, port)
    79  		second := dur.Seconds()
    80  		klog.Infof("get io delay %fs", second)
    81  		// IO Delay >= 1s
    82  		if second >= 1 {
    83  			return true, nil
    84  		}
    85  		return false, nil
    86  	})
    87  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
    88  	By("apply io chaos successfully")
    89  
    90  	By("delete chaos CRD objects")
    91  	// delete chaos CRD
    92  	err = cli.Delete(ctx, ioChaos)
    93  	framework.ExpectNoError(err, "failed to delete io chaos")
    94  	By("waiting for assertion recovering")
    95  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
    96  		dur, _ := getPodIODelay(c, port)
    97  		second := dur.Seconds()
    98  		klog.Infof("get io delay %fs", second)
    99  		// IO Delay shouldn't longer than 1s
   100  		if second >= 1 {
   101  			return false, nil
   102  		}
   103  		return true, nil
   104  	})
   105  	framework.ExpectNoError(err, "fail to recover io chaos")
   106  }
   107  
   108  func TestcaseIODelayDurationForATimePauseAndUnPause(
   109  	ns string,
   110  	cli client.Client,
   111  	c http.Client,
   112  	port uint16,
   113  ) {
   114  	ctx, cancel := context.WithCancel(context.Background())
   115  	defer cancel()
   116  	By("waiting for e2e helper ready")
   117  	err := util.WaitE2EHelperReady(c, port)
   118  	framework.ExpectNoError(err, "wait e2e helper ready error")
   119  
   120  	By("create io chaos crd object")
   121  	ioChaos := &v1alpha1.IOChaos{
   122  		ObjectMeta: metav1.ObjectMeta{
   123  			Name:      "io-chaos",
   124  			Namespace: ns,
   125  		},
   126  		Spec: v1alpha1.IOChaosSpec{
   127  			Action:     v1alpha1.IoLatency,
   128  			VolumePath: "/var/run/data",
   129  			Path:       "/var/run/data/*",
   130  			Delay:      "10ms",
   131  			Percent:    100,
   132  			ContainerSelector: v1alpha1.ContainerSelector{
   133  				PodSelector: v1alpha1.PodSelector{
   134  					Selector: v1alpha1.PodSelectorSpec{
   135  						GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
   136  							Namespaces:     []string{ns},
   137  							LabelSelectors: map[string]string{"app": "io"},
   138  						},
   139  					},
   140  					Mode: v1alpha1.OneMode,
   141  				},
   142  			},
   143  		},
   144  	}
   145  	err = cli.Create(ctx, ioChaos)
   146  	framework.ExpectNoError(err, "error occurs while applying io chaos")
   147  
   148  	chaosKey := types.NamespacedName{
   149  		Namespace: ns,
   150  		Name:      "io-chaos",
   151  	}
   152  
   153  	By("waiting for assertion io chaos")
   154  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   155  		chaos := &v1alpha1.IOChaos{}
   156  		err = cli.Get(ctx, chaosKey, chaos)
   157  		framework.ExpectNoError(err, "get io chaos error")
   158  
   159  		for _, c := range chaos.GetStatus().Conditions {
   160  			if c.Type == v1alpha1.ConditionAllInjected {
   161  				if c.Status != corev1.ConditionTrue {
   162  					return false, nil
   163  				}
   164  			} else if c.Type == v1alpha1.ConditionSelected {
   165  				if c.Status != corev1.ConditionTrue {
   166  					return false, nil
   167  				}
   168  			}
   169  		}
   170  
   171  		dur, _ := getPodIODelay(c, port)
   172  
   173  		ms := dur.Milliseconds()
   174  		klog.Infof("get io delay %dms", ms)
   175  		// IO Delay >= 500ms
   176  		if ms >= 10 {
   177  			return true, nil
   178  		}
   179  		return false, nil
   180  	})
   181  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
   182  
   183  	By("pause io delay chaos experiment")
   184  	// pause experiment
   185  	err = util.PauseChaos(ctx, cli, ioChaos)
   186  	framework.ExpectNoError(err, "pause chaos error")
   187  
   188  	By("waiting for assertion about pause")
   189  	err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   190  		chaos := &v1alpha1.IOChaos{}
   191  		err = cli.Get(ctx, chaosKey, chaos)
   192  		framework.ExpectNoError(err, "get io chaos error")
   193  
   194  		for _, c := range chaos.GetStatus().Conditions {
   195  			if c.Type == v1alpha1.ConditionAllRecovered {
   196  				if c.Status != corev1.ConditionTrue {
   197  					return false, nil
   198  				}
   199  			} else if c.Type == v1alpha1.ConditionSelected {
   200  				if c.Status != corev1.ConditionTrue {
   201  					return false, nil
   202  				}
   203  			}
   204  		}
   205  
   206  		return true, err
   207  	})
   208  	framework.ExpectNoError(err, "check paused chaos failed")
   209  
   210  	// wait 1 min to check whether io delay still exists
   211  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   212  		dur, _ := getPodIODelay(c, port)
   213  
   214  		ms := dur.Milliseconds()
   215  		klog.Infof("get io delay %dms", ms)
   216  		// IO Delay shouldn't longer than 10ms
   217  		if ms > 10 {
   218  			return false, nil
   219  		}
   220  		return true, nil
   221  	})
   222  	framework.ExpectNoError(err, "fail to recover io chaos")
   223  
   224  	By("resume io delay chaos experiment")
   225  	// resume experiment
   226  	err = util.UnPauseChaos(ctx, cli, ioChaos)
   227  	framework.ExpectNoError(err, "resume chaos error")
   228  
   229  	By("assert that io delay is effective again")
   230  	err = wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) {
   231  		chaos := &v1alpha1.IOChaos{}
   232  		err = cli.Get(ctx, chaosKey, chaos)
   233  		framework.ExpectNoError(err, "get io chaos error")
   234  
   235  		for _, c := range chaos.GetStatus().Conditions {
   236  			if c.Type == v1alpha1.ConditionAllInjected {
   237  				if c.Status != corev1.ConditionTrue {
   238  					return false, nil
   239  				}
   240  			} else if c.Type == v1alpha1.ConditionSelected {
   241  				if c.Status != corev1.ConditionTrue {
   242  					return false, nil
   243  				}
   244  			}
   245  		}
   246  
   247  		return true, err
   248  	})
   249  	framework.ExpectNoError(err, "check resumed chaos failed")
   250  
   251  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   252  		dur, _ := getPodIODelay(c, port)
   253  
   254  		ms := dur.Milliseconds()
   255  		klog.Infof("get io delay %dms", ms)
   256  		// IO Delay >= 10ms
   257  		if ms >= 10 {
   258  			return true, nil
   259  		}
   260  		return false, nil
   261  	})
   262  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
   263  
   264  	By("cleanup")
   265  	// cleanup
   266  	cli.Delete(ctx, ioChaos)
   267  }
   268  
   269  func TestcaseIODelayWithSpecifiedContainer(
   270  	ns string,
   271  	cli client.Client,
   272  	c http.Client,
   273  	port uint16) {
   274  	ctx, cancel := context.WithCancel(context.Background())
   275  	defer cancel()
   276  	err := util.WaitE2EHelperReady(c, port)
   277  	framework.ExpectNoError(err, "wait e2e helper ready error")
   278  
   279  	containerName := "io"
   280  	ioChaos := &v1alpha1.IOChaos{
   281  		ObjectMeta: metav1.ObjectMeta{
   282  			Name:      "io-chaos",
   283  			Namespace: ns,
   284  		},
   285  		Spec: v1alpha1.IOChaosSpec{
   286  			Action:     v1alpha1.IoLatency,
   287  			VolumePath: "/var/run/data",
   288  			Path:       "/var/run/data/*",
   289  			Delay:      "10ms",
   290  			Percent:    100,
   291  			ContainerSelector: v1alpha1.ContainerSelector{
   292  				PodSelector: v1alpha1.PodSelector{
   293  					Selector: v1alpha1.PodSelectorSpec{
   294  						GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
   295  							Namespaces:     []string{ns},
   296  							LabelSelectors: map[string]string{"app": "io"},
   297  						},
   298  					},
   299  					Mode: v1alpha1.OneMode,
   300  				},
   301  				ContainerNames: []string{containerName},
   302  			},
   303  		},
   304  	}
   305  	err = cli.Create(ctx, ioChaos)
   306  	framework.ExpectNoError(err, "create io chaos error")
   307  
   308  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   309  		dur, _ := getPodIODelay(c, port)
   310  
   311  		ms := dur.Milliseconds()
   312  		klog.Infof("get io delay %dms", ms)
   313  		// IO Delay >= 10ms
   314  		if ms >= 10 {
   315  			return true, nil
   316  		}
   317  		return false, nil
   318  	})
   319  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
   320  	klog.Infof("apply io chaos successfully")
   321  
   322  	err = cli.Delete(ctx, ioChaos)
   323  	framework.ExpectNoError(err, "failed to delete io chaos")
   324  
   325  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   326  		dur, _ := getPodIODelay(c, port)
   327  
   328  		ms := dur.Milliseconds()
   329  		klog.Infof("get io delay %dms", ms)
   330  		// IO Delay shouldn't longer than 10ms
   331  		if ms >= 10 {
   332  			return false, nil
   333  		}
   334  		return true, nil
   335  	})
   336  	framework.ExpectNoError(err, "fail to recover io chaos")
   337  }
   338  
   339  func TestcaseIODelayWithWrongSpec(
   340  	ns string,
   341  	cli client.Client,
   342  	c http.Client,
   343  	port uint16,
   344  ) {
   345  	ctx, cancel := context.WithCancel(context.Background())
   346  	defer cancel()
   347  
   348  	By("waiting on e2e helper ready")
   349  	err := util.WaitE2EHelperReady(c, port)
   350  	framework.ExpectNoError(err, "wait e2e helper ready error")
   351  	By("create IO delay chaos CRD objects")
   352  	ioChaos := &v1alpha1.IOChaos{
   353  		ObjectMeta: metav1.ObjectMeta{
   354  			Name:      "io-chaos",
   355  			Namespace: ns,
   356  		},
   357  		Spec: v1alpha1.IOChaosSpec{
   358  			ContainerSelector: v1alpha1.ContainerSelector{
   359  				PodSelector: v1alpha1.PodSelector{
   360  					Selector: v1alpha1.PodSelectorSpec{
   361  						GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
   362  							Namespaces:     []string{ns},
   363  							LabelSelectors: map[string]string{"app": "io"},
   364  						},
   365  					},
   366  					Mode: v1alpha1.OneMode,
   367  				},
   368  			},
   369  			Action:     v1alpha1.IoLatency,
   370  			VolumePath: "/var/run/data/123",
   371  			Path:       "/var/run/data/*",
   372  			Delay:      "1s",
   373  			Percent:    100,
   374  			Duration:   pointer.StringPtr("9m"),
   375  		},
   376  	}
   377  	err = cli.Create(ctx, ioChaos)
   378  	framework.ExpectNoError(err, "create io chaos error")
   379  	// TODO: resume the e2e test
   380  	// err = wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) {
   381  	// 	err := cli.Get(ctx, types.NamespacedName{Namespace: ioChaos.ObjectMeta.Namespace, Name: ioChaos.ObjectMeta.Name}, ioChaos)
   382  	// 	if err != nil {
   383  	// 		return false, err
   384  	// 	}
   385  	// 	errStr := ioChaos.Status.ChaosStatus.FailedMessage
   386  	// 	klog.Infof("get chaos err: %s", errStr)
   387  	// 	if strings.Contains(errStr, "Toda startup takes too long or an error occurs") {
   388  	// 		return true, nil
   389  	// 	}
   390  	// 	return false, nil
   391  	// })
   392  	// framework.ExpectNoError(err, "A wrong chaos spec should raise an error")
   393  }
   394