io_error.go

Documentation: github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/iochaos

     1  // Copyright 2021 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  // http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  
    16  package iochaos
    17  
    18  import (
    19  	"context"
    20  	"net/http"
    21  	"strings"
    22  	"time"
    23  
    24  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    25  	"k8s.io/apimachinery/pkg/types"
    26  	"k8s.io/apimachinery/pkg/util/wait"
    27  	"k8s.io/klog/v2"
    28  	"k8s.io/kubernetes/test/e2e/framework"
    29  	"sigs.k8s.io/controller-runtime/pkg/client"
    30  
    31  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    32  	"github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/util"
    33  )
    34  
    35  func TestcaseIOErrorDurationForATimeThenRecover(
    36  	ns string,
    37  	cli client.Client,
    38  	c http.Client,
    39  	port uint16,
    40  ) {
    41  	ctx, cancel := context.WithCancel(context.Background())
    42  	err := util.WaitE2EHelperReady(c, port)
    43  	framework.ExpectNoError(err, "wait e2e helper ready error")
    44  
    45  	ioChaos := &v1alpha1.IOChaos{
    46  		ObjectMeta: metav1.ObjectMeta{
    47  			Name:      "io-chaos",
    48  			Namespace: ns,
    49  		},
    50  		Spec: v1alpha1.IOChaosSpec{
    51  			Action:     v1alpha1.IoFaults,
    52  			VolumePath: "/var/run/data",
    53  			Path:       "/var/run/data/*",
    54  			Percent:    100,
    55  			// errno 5 is EIO -> I/O error
    56  			Errno: 5,
    57  			// only inject write method
    58  			Methods: []v1alpha1.IoMethod{v1alpha1.Write},
    59  			ContainerSelector: v1alpha1.ContainerSelector{
    60  				PodSelector: v1alpha1.PodSelector{
    61  					Selector: v1alpha1.PodSelectorSpec{
    62  						GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
    63  							Namespaces:     []string{ns},
    64  							LabelSelectors: map[string]string{"app": "io"},
    65  						},
    66  					},
    67  					Mode: v1alpha1.OneMode,
    68  				},
    69  			},
    70  		},
    71  	}
    72  	err = cli.Create(ctx, ioChaos)
    73  	framework.ExpectNoError(err, "create io chaos")
    74  
    75  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
    76  		_, err = getPodIODelay(c, port)
    77  		// input/output error is errno 5
    78  		if err != nil && strings.Contains(err.Error(), "input/output error") {
    79  			return true, nil
    80  		}
    81  		return false, nil
    82  	})
    83  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
    84  
    85  	err = cli.Delete(ctx, ioChaos)
    86  	framework.ExpectNoError(err, "failed to delete io chaos")
    87  
    88  	klog.Infof("success to perform io chaos")
    89  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
    90  		_, err = getPodIODelay(c, port)
    91  
    92  		if err == nil {
    93  			return true, nil
    94  		}
    95  		return false, nil
    96  	})
    97  	framework.ExpectNoError(err, "fail to recover io chaos")
    98  
    99  	cancel()
   100  }
   101  
   102  func TestcaseIOErrorDurationForATimePauseAndUnPause(
   103  	ns string,
   104  	cli client.Client,
   105  	c http.Client,
   106  	port uint16,
   107  ) {
   108  	ctx, cancel := context.WithCancel(context.Background())
   109  	err := util.WaitE2EHelperReady(c, port)
   110  	framework.ExpectNoError(err, "wait e2e helper ready error")
   111  
   112  	ioChaos := &v1alpha1.IOChaos{
   113  		ObjectMeta: metav1.ObjectMeta{
   114  			Name:      "io-chaos",
   115  			Namespace: ns,
   116  		},
   117  		Spec: v1alpha1.IOChaosSpec{
   118  			Action:     v1alpha1.IoFaults,
   119  			VolumePath: "/var/run/data",
   120  			Path:       "/var/run/data/*",
   121  			Percent:    100,
   122  			// errno 5 is EIO -> I/O error
   123  			Errno: 5,
   124  			// only inject write method
   125  			Methods: []v1alpha1.IoMethod{v1alpha1.Write},
   126  			ContainerSelector: v1alpha1.ContainerSelector{
   127  				PodSelector: v1alpha1.PodSelector{
   128  					Selector: v1alpha1.PodSelectorSpec{
   129  						GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
   130  							Namespaces:     []string{ns},
   131  							LabelSelectors: map[string]string{"app": "io"},
   132  						},
   133  					},
   134  					Mode: v1alpha1.OneMode,
   135  				},
   136  			},
   137  		},
   138  	}
   139  	err = cli.Create(ctx, ioChaos)
   140  	framework.ExpectNoError(err, "create io chaos error")
   141  
   142  	klog.Info("create iochaos successfully")
   143  
   144  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   145  		_, err = getPodIODelay(c, port)
   146  		// input/output error is errno 5
   147  		if err != nil && strings.Contains(err.Error(), "input/output error") {
   148  			return true, nil
   149  		}
   150  		return false, nil
   151  	})
   152  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
   153  
   154  	chaosKey := types.NamespacedName{
   155  		Namespace: ns,
   156  		Name:      "io-chaos",
   157  	}
   158  
   159  	// pause experiment
   160  	err = util.PauseChaos(ctx, cli, ioChaos)
   161  	framework.ExpectNoError(err, "pause chaos error")
   162  
   163  	klog.Info("pause iochaos")
   164  
   165  	err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   166  		chaos := &v1alpha1.IOChaos{}
   167  		err = cli.Get(ctx, chaosKey, chaos)
   168  		framework.ExpectNoError(err, "get io chaos error")
   169  		if chaos.Status.Experiment.DesiredPhase == v1alpha1.StoppedPhase {
   170  			return true, nil
   171  		}
   172  		return false, err
   173  	})
   174  	framework.ExpectNoError(err, "check paused chaos failed")
   175  
   176  	// wait 1 min to check whether io delay still exists
   177  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   178  		_, err = getPodIODelay(c, port)
   179  
   180  		if err == nil {
   181  			return true, nil
   182  		}
   183  		return false, nil
   184  	})
   185  	framework.ExpectNoError(err, "fail to recover io chaos")
   186  
   187  	// resume experiment
   188  	err = util.UnPauseChaos(ctx, cli, ioChaos)
   189  	framework.ExpectNoError(err, "resume chaos error")
   190  
   191  	err = wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) {
   192  		chaos := &v1alpha1.IOChaos{}
   193  		err = cli.Get(ctx, chaosKey, chaos)
   194  		framework.ExpectNoError(err, "get io chaos error")
   195  		if chaos.Status.Experiment.DesiredPhase == v1alpha1.RunningPhase {
   196  			return true, nil
   197  		}
   198  		return false, err
   199  	})
   200  	framework.ExpectNoError(err, "check resumed chaos failed")
   201  
   202  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   203  		_, err = getPodIODelay(c, port)
   204  		// input/output error is errno 5
   205  		if err != nil && strings.Contains(err.Error(), "input/output error") {
   206  			return true, nil
   207  		}
   208  		return false, nil
   209  	})
   210  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
   211  
   212  	// cleanup
   213  	cli.Delete(ctx, ioChaos)
   214  	cancel()
   215  }
   216  
   217  func TestcaseIOErrorWithSpecifiedContainer(
   218  	ns string,
   219  	cli client.Client,
   220  	c http.Client,
   221  	port uint16) {
   222  	ctx, cancel := context.WithCancel(context.Background())
   223  	defer cancel()
   224  	err := util.WaitE2EHelperReady(c, port)
   225  	framework.ExpectNoError(err, "wait e2e helper ready error")
   226  
   227  	containerName := "io"
   228  	ioChaos := &v1alpha1.IOChaos{
   229  		ObjectMeta: metav1.ObjectMeta{
   230  			Name:      "io-chaos",
   231  			Namespace: ns,
   232  		},
   233  		Spec: v1alpha1.IOChaosSpec{
   234  			Action:     v1alpha1.IoFaults,
   235  			VolumePath: "/var/run/data",
   236  			Path:       "/var/run/data/*",
   237  			Percent:    100,
   238  			// errno 5 is EIO -> I/O error
   239  			Errno: 5,
   240  			// only inject write method
   241  			Methods: []v1alpha1.IoMethod{v1alpha1.Write},
   242  			ContainerSelector: v1alpha1.ContainerSelector{
   243  				PodSelector: v1alpha1.PodSelector{
   244  					Selector: v1alpha1.PodSelectorSpec{
   245  						GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
   246  							Namespaces:     []string{ns},
   247  							LabelSelectors: map[string]string{"app": "io"},
   248  						},
   249  					},
   250  					Mode: v1alpha1.OneMode,
   251  				},
   252  				ContainerNames: []string{containerName},
   253  			},
   254  		},
   255  	}
   256  	err = cli.Create(ctx, ioChaos)
   257  	framework.ExpectNoError(err, "create io chaos")
   258  
   259  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   260  		_, err = getPodIODelay(c, port)
   261  		// input/output error is errno 5
   262  		if err != nil && strings.Contains(err.Error(), "input/output error") {
   263  			return true, nil
   264  		}
   265  		return false, nil
   266  	})
   267  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
   268  
   269  	err = cli.Delete(ctx, ioChaos)
   270  	framework.ExpectNoError(err, "failed to delete io chaos")
   271  
   272  	klog.Infof("success to perform io chaos")
   273  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   274  		_, err = getPodIODelay(c, port)
   275  
   276  		if err == nil {
   277  			return true, nil
   278  		}
   279  		return false, nil
   280  	})
   281  	framework.ExpectNoError(err, "fail to recover io chaos")
   282  }
   283