io_error.go

Documentation: github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/iochaos

     1  // Copyright 2020 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package iochaos
    15  
    16  import (
    17  	"context"
    18  	"net/http"
    19  	"strings"
    20  	"time"
    21  
    22  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    23  	"k8s.io/apimachinery/pkg/types"
    24  	"k8s.io/apimachinery/pkg/util/wait"
    25  	"k8s.io/klog"
    26  	"k8s.io/kubernetes/test/e2e/framework"
    27  	"sigs.k8s.io/controller-runtime/pkg/client"
    28  
    29  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    30  	"github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/util"
    31  )
    32  
    33  func TestcaseIOErrorDurationForATimeThenRecover(
    34  	ns string,
    35  	cli client.Client,
    36  	c http.Client,
    37  	port uint16,
    38  ) {
    39  	ctx, cancel := context.WithCancel(context.Background())
    40  	err := util.WaitE2EHelperReady(c, port)
    41  	framework.ExpectNoError(err, "wait e2e helper ready error")
    42  
    43  	ioChaos := &v1alpha1.IOChaos{
    44  		ObjectMeta: metav1.ObjectMeta{
    45  			Name:      "io-chaos",
    46  			Namespace: ns,
    47  		},
    48  		Spec: v1alpha1.IOChaosSpec{
    49  			Action:     v1alpha1.IoFaults,
    50  			VolumePath: "/var/run/data",
    51  			Path:       "/var/run/data/*",
    52  			Percent:    100,
    53  			// errno 5 is EIO -> I/O error
    54  			Errno: 5,
    55  			// only inject write method
    56  			Methods: []v1alpha1.IoMethod{v1alpha1.Write},
    57  			ContainerSelector: v1alpha1.ContainerSelector{
    58  				PodSelector: v1alpha1.PodSelector{
    59  					Selector: v1alpha1.PodSelectorSpec{
    60  						Namespaces:     []string{ns},
    61  						LabelSelectors: map[string]string{"app": "io"},
    62  					},
    63  					Mode: v1alpha1.OnePodMode,
    64  				},
    65  			},
    66  		},
    67  	}
    68  	err = cli.Create(ctx, ioChaos)
    69  	framework.ExpectNoError(err, "create io chaos")
    70  
    71  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
    72  		_, err = getPodIODelay(c, port)
    73  		// input/output error is errno 5
    74  		if err != nil && strings.Contains(err.Error(), "input/output error") {
    75  			return true, nil
    76  		}
    77  		return false, nil
    78  	})
    79  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
    80  
    81  	err = cli.Delete(ctx, ioChaos)
    82  	framework.ExpectNoError(err, "failed to delete io chaos")
    83  
    84  	klog.Infof("success to perform io chaos")
    85  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
    86  		_, err = getPodIODelay(c, port)
    87  
    88  		if err == nil {
    89  			return true, nil
    90  		}
    91  		return false, nil
    92  	})
    93  	framework.ExpectNoError(err, "fail to recover io chaos")
    94  
    95  	cancel()
    96  }
    97  
    98  func TestcaseIOErrorDurationForATimePauseAndUnPause(
    99  	ns string,
   100  	cli client.Client,
   101  	c http.Client,
   102  	port uint16,
   103  ) {
   104  	ctx, cancel := context.WithCancel(context.Background())
   105  	err := util.WaitE2EHelperReady(c, port)
   106  	framework.ExpectNoError(err, "wait e2e helper ready error")
   107  
   108  	ioChaos := &v1alpha1.IOChaos{
   109  		ObjectMeta: metav1.ObjectMeta{
   110  			Name:      "io-chaos",
   111  			Namespace: ns,
   112  		},
   113  		Spec: v1alpha1.IOChaosSpec{
   114  			Action:     v1alpha1.IoFaults,
   115  			VolumePath: "/var/run/data",
   116  			Path:       "/var/run/data/*",
   117  			Percent:    100,
   118  			// errno 5 is EIO -> I/O error
   119  			Errno: 5,
   120  			// only inject write method
   121  			Methods: []v1alpha1.IoMethod{v1alpha1.Write},
   122  			ContainerSelector: v1alpha1.ContainerSelector{
   123  				PodSelector: v1alpha1.PodSelector{
   124  					Selector: v1alpha1.PodSelectorSpec{
   125  						Namespaces:     []string{ns},
   126  						LabelSelectors: map[string]string{"app": "io"},
   127  					},
   128  					Mode: v1alpha1.OnePodMode,
   129  				},
   130  			},
   131  		},
   132  	}
   133  	err = cli.Create(ctx, ioChaos)
   134  	framework.ExpectNoError(err, "create io chaos error")
   135  
   136  	klog.Info("create iochaos successfully")
   137  
   138  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   139  		_, err = getPodIODelay(c, port)
   140  		// input/output error is errno 5
   141  		if err != nil && strings.Contains(err.Error(), "input/output error") {
   142  			return true, nil
   143  		}
   144  		return false, nil
   145  	})
   146  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
   147  
   148  	chaosKey := types.NamespacedName{
   149  		Namespace: ns,
   150  		Name:      "io-chaos",
   151  	}
   152  
   153  	// pause experiment
   154  	err = util.PauseChaos(ctx, cli, ioChaos)
   155  	framework.ExpectNoError(err, "pause chaos error")
   156  
   157  	klog.Info("pause iochaos")
   158  
   159  	err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   160  		chaos := &v1alpha1.IOChaos{}
   161  		err = cli.Get(ctx, chaosKey, chaos)
   162  		framework.ExpectNoError(err, "get io chaos error")
   163  		if chaos.Status.Experiment.DesiredPhase == v1alpha1.StoppedPhase {
   164  			return true, nil
   165  		}
   166  		return false, err
   167  	})
   168  	framework.ExpectNoError(err, "check paused chaos failed")
   169  
   170  	// wait 1 min to check whether io delay still exists
   171  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   172  		_, err = getPodIODelay(c, port)
   173  
   174  		if err == nil {
   175  			return true, nil
   176  		}
   177  		return false, nil
   178  	})
   179  	framework.ExpectNoError(err, "fail to recover io chaos")
   180  
   181  	// resume experiment
   182  	err = util.UnPauseChaos(ctx, cli, ioChaos)
   183  	framework.ExpectNoError(err, "resume chaos error")
   184  
   185  	err = wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) {
   186  		chaos := &v1alpha1.IOChaos{}
   187  		err = cli.Get(ctx, chaosKey, chaos)
   188  		framework.ExpectNoError(err, "get io chaos error")
   189  		if chaos.Status.Experiment.DesiredPhase == v1alpha1.RunningPhase {
   190  			return true, nil
   191  		}
   192  		return false, err
   193  	})
   194  	framework.ExpectNoError(err, "check resumed chaos failed")
   195  
   196  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   197  		_, err = getPodIODelay(c, port)
   198  		// input/output error is errno 5
   199  		if err != nil && strings.Contains(err.Error(), "input/output error") {
   200  			return true, nil
   201  		}
   202  		return false, nil
   203  	})
   204  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
   205  
   206  	// cleanup
   207  	cli.Delete(ctx, ioChaos)
   208  	cancel()
   209  }
   210  
   211  func TestcaseIOErrorWithSpecifiedContainer(
   212  	ns string,
   213  	cli client.Client,
   214  	c http.Client,
   215  	port uint16) {
   216  	ctx, cancel := context.WithCancel(context.Background())
   217  	defer cancel()
   218  	err := util.WaitE2EHelperReady(c, port)
   219  	framework.ExpectNoError(err, "wait e2e helper ready error")
   220  
   221  	containerName := "io"
   222  	ioChaos := &v1alpha1.IOChaos{
   223  		ObjectMeta: metav1.ObjectMeta{
   224  			Name:      "io-chaos",
   225  			Namespace: ns,
   226  		},
   227  		Spec: v1alpha1.IOChaosSpec{
   228  			Action:     v1alpha1.IoFaults,
   229  			VolumePath: "/var/run/data",
   230  			Path:       "/var/run/data/*",
   231  			Percent:    100,
   232  			// errno 5 is EIO -> I/O error
   233  			Errno: 5,
   234  			// only inject write method
   235  			Methods: []v1alpha1.IoMethod{v1alpha1.Write},
   236  			ContainerSelector: v1alpha1.ContainerSelector{
   237  				PodSelector: v1alpha1.PodSelector{
   238  					Selector: v1alpha1.PodSelectorSpec{
   239  						Namespaces:     []string{ns},
   240  						LabelSelectors: map[string]string{"app": "io"},
   241  					},
   242  					Mode: v1alpha1.OnePodMode,
   243  				},
   244  				ContainerNames: []string{containerName},
   245  			},
   246  		},
   247  	}
   248  	err = cli.Create(ctx, ioChaos)
   249  	framework.ExpectNoError(err, "create io chaos")
   250  
   251  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   252  		_, err = getPodIODelay(c, port)
   253  		// input/output error is errno 5
   254  		if err != nil && strings.Contains(err.Error(), "input/output error") {
   255  			return true, nil
   256  		}
   257  		return false, nil
   258  	})
   259  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
   260  
   261  	err = cli.Delete(ctx, ioChaos)
   262  	framework.ExpectNoError(err, "failed to delete io chaos")
   263  
   264  	klog.Infof("success to perform io chaos")
   265  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   266  		_, err = getPodIODelay(c, port)
   267  
   268  		if err == nil {
   269  			return true, nil
   270  		}
   271  		return false, nil
   272  	})
   273  	framework.ExpectNoError(err, "fail to recover io chaos")
   274  }
   275