io_mistake.go

Documentation: github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/iochaos

     1  // Copyright 2021 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  // http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  
    16  package iochaos
    17  
    18  import (
    19  	"context"
    20  	"net/http"
    21  	"time"
    22  
    23  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    24  	"k8s.io/apimachinery/pkg/types"
    25  	"k8s.io/apimachinery/pkg/util/wait"
    26  	"k8s.io/klog/v2"
    27  	"k8s.io/kubernetes/test/e2e/framework"
    28  	"k8s.io/utils/pointer"
    29  	"sigs.k8s.io/controller-runtime/pkg/client"
    30  
    31  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    32  	"github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/util"
    33  )
    34  
    35  func TestcaseIOMistakeDurationForATimeThenRecover(
    36  	ns string,
    37  	cli client.Client,
    38  	c http.Client,
    39  	port uint16,
    40  ) {
    41  	ctx, cancel := context.WithCancel(context.Background())
    42  	defer cancel()
    43  	err := util.WaitE2EHelperReady(c, port)
    44  	framework.ExpectNoError(err, "wait e2e helper ready error")
    45  
    46  	ioChaos := &v1alpha1.IOChaos{
    47  		ObjectMeta: metav1.ObjectMeta{
    48  			Name:      "io-chaos",
    49  			Namespace: ns,
    50  		},
    51  		Spec: v1alpha1.IOChaosSpec{
    52  			ContainerSelector: v1alpha1.ContainerSelector{
    53  				PodSelector: v1alpha1.PodSelector{
    54  					Selector: v1alpha1.PodSelectorSpec{
    55  						GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
    56  							Namespaces:     []string{ns},
    57  							LabelSelectors: map[string]string{"app": "io"},
    58  						},
    59  					},
    60  					Mode: v1alpha1.OneMode,
    61  				},
    62  			},
    63  			Action:     v1alpha1.IoMistake,
    64  			VolumePath: "/var/run/data",
    65  			Path:       "/var/run/data/*",
    66  			Percent:    100,
    67  			Mistake: &v1alpha1.MistakeSpec{
    68  				MaxOccurrences: 1,
    69  				MaxLength:      10000,
    70  				Filling:        v1alpha1.Zero,
    71  			},
    72  			// only inject read or write method. Other method may or may not run properly, but is not recommended
    73  			Methods:  []v1alpha1.IoMethod{v1alpha1.Read, v1alpha1.Write},
    74  			Duration: pointer.String("9m"),
    75  		},
    76  	}
    77  	err = cli.Create(ctx, ioChaos)
    78  	framework.ExpectNoError(err, "create io chaos")
    79  
    80  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
    81  		res, err := getPodIoMistake(c, port)
    82  		if err != nil {
    83  			return false, nil
    84  		}
    85  		if err == nil && res {
    86  			return true, nil
    87  		}
    88  		return false, nil
    89  	})
    90  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
    91  
    92  	err = cli.Delete(ctx, ioChaos)
    93  	framework.ExpectNoError(err, "failed to delete io chaos")
    94  
    95  	klog.Infof("success to perform io chaos")
    96  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
    97  		res, err := getPodIoMistake(c, port)
    98  		if err != nil {
    99  			return false, nil
   100  		}
   101  		if err == nil && !res {
   102  			return true, nil
   103  		}
   104  		return false, nil
   105  	})
   106  	framework.ExpectNoError(err, "fail to recover io chaos")
   107  }
   108  
   109  func TestcaseIOMistakeDurationForATimePauseAndUnPause(
   110  	ns string,
   111  	cli client.Client,
   112  	c http.Client,
   113  	port uint16,
   114  ) {
   115  	ctx, cancel := context.WithCancel(context.Background())
   116  	defer cancel()
   117  	err := util.WaitE2EHelperReady(c, port)
   118  	framework.ExpectNoError(err, "wait e2e helper ready error")
   119  
   120  	ioChaos := &v1alpha1.IOChaos{
   121  		ObjectMeta: metav1.ObjectMeta{
   122  			Name:      "io-chaos",
   123  			Namespace: ns,
   124  		},
   125  		Spec: v1alpha1.IOChaosSpec{
   126  			ContainerSelector: v1alpha1.ContainerSelector{
   127  				PodSelector: v1alpha1.PodSelector{
   128  					Selector: v1alpha1.PodSelectorSpec{
   129  						GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
   130  							Namespaces:     []string{ns},
   131  							LabelSelectors: map[string]string{"app": "io"},
   132  						},
   133  					},
   134  					Mode: v1alpha1.OneMode,
   135  				},
   136  			},
   137  			Action:     v1alpha1.IoMistake,
   138  			VolumePath: "/var/run/data",
   139  			Path:       "/var/run/data/*",
   140  			Percent:    100,
   141  			Mistake: &v1alpha1.MistakeSpec{
   142  				MaxOccurrences: 1,
   143  				MaxLength:      10000,
   144  				Filling:        v1alpha1.Zero,
   145  			},
   146  			// only inject read or write method. Other method may or may not run properly, but is not recommended
   147  			Methods:  []v1alpha1.IoMethod{v1alpha1.Read, v1alpha1.Write},
   148  			Duration: pointer.String("9m"),
   149  		},
   150  	}
   151  	err = cli.Create(ctx, ioChaos)
   152  	framework.ExpectNoError(err, "create io chaos error")
   153  
   154  	klog.Info("create iochaos successfully")
   155  
   156  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   157  		res, err := getPodIoMistake(c, port)
   158  		if err != nil {
   159  			return false, nil
   160  		}
   161  		if err == nil && res {
   162  			return true, nil
   163  		}
   164  		return false, nil
   165  	})
   166  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
   167  
   168  	chaosKey := types.NamespacedName{
   169  		Namespace: ns,
   170  		Name:      "io-chaos",
   171  	}
   172  
   173  	// pause experiment
   174  	err = util.PauseChaos(ctx, cli, ioChaos)
   175  	framework.ExpectNoError(err, "pause chaos error")
   176  
   177  	klog.Info("pause iochaos")
   178  
   179  	err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   180  		chaos := &v1alpha1.IOChaos{}
   181  		err = cli.Get(ctx, chaosKey, chaos)
   182  		framework.ExpectNoError(err, "get io chaos error")
   183  		if chaos.Status.Experiment.DesiredPhase == v1alpha1.StoppedPhase {
   184  			return true, nil
   185  		}
   186  		return false, err
   187  	})
   188  	framework.ExpectNoError(err, "check paused chaos failed")
   189  
   190  	// wait 1 min to check whether io delay still exists
   191  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   192  		res, err := getPodIoMistake(c, port)
   193  		if err != nil {
   194  			return false, nil
   195  		}
   196  		if err == nil && !res {
   197  			return true, nil
   198  		}
   199  		return false, nil
   200  	})
   201  	framework.ExpectNoError(err, "fail to recover io chaos")
   202  
   203  	// resume experiment
   204  	err = util.UnPauseChaos(ctx, cli, ioChaos)
   205  	framework.ExpectNoError(err, "resume chaos error")
   206  
   207  	err = wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) {
   208  		chaos := &v1alpha1.IOChaos{}
   209  		err = cli.Get(ctx, chaosKey, chaos)
   210  		framework.ExpectNoError(err, "get io chaos error")
   211  		if chaos.Status.Experiment.DesiredPhase == v1alpha1.RunningPhase {
   212  			return true, nil
   213  		}
   214  		return false, err
   215  	})
   216  	framework.ExpectNoError(err, "check resumed chaos failed")
   217  
   218  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   219  		res, err := getPodIoMistake(c, port)
   220  		if err != nil {
   221  			return false, nil
   222  		}
   223  		if err == nil && res {
   224  			return true, nil
   225  		}
   226  		return false, nil
   227  	})
   228  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
   229  
   230  	// cleanup
   231  	cli.Delete(ctx, ioChaos)
   232  }
   233  
   234  func TestcaseIOMistakeWithSpecifiedContainer(
   235  	ns string,
   236  	cli client.Client,
   237  	c http.Client,
   238  	port uint16) {
   239  	ctx, cancel := context.WithCancel(context.Background())
   240  	defer cancel()
   241  	err := util.WaitE2EHelperReady(c, port)
   242  	framework.ExpectNoError(err, "wait e2e helper ready error")
   243  
   244  	containerName := "io"
   245  	ioChaos := &v1alpha1.IOChaos{
   246  		ObjectMeta: metav1.ObjectMeta{
   247  			Name:      "io-chaos",
   248  			Namespace: ns,
   249  		},
   250  		Spec: v1alpha1.IOChaosSpec{
   251  			ContainerSelector: v1alpha1.ContainerSelector{
   252  				PodSelector: v1alpha1.PodSelector{
   253  					Selector: v1alpha1.PodSelectorSpec{
   254  						GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
   255  							Namespaces:     []string{ns},
   256  							LabelSelectors: map[string]string{"app": "io"},
   257  						},
   258  					},
   259  					Mode: v1alpha1.OneMode,
   260  				},
   261  				ContainerNames: []string{containerName},
   262  			},
   263  			Action:     v1alpha1.IoMistake,
   264  			VolumePath: "/var/run/data",
   265  			Path:       "/var/run/data/*",
   266  			Percent:    100,
   267  			Mistake: &v1alpha1.MistakeSpec{
   268  				MaxOccurrences: 1,
   269  				MaxLength:      10000,
   270  				Filling:        v1alpha1.Zero,
   271  			},
   272  			// only inject read or write method. Other method may or may not run properly, but is not recommended
   273  			Methods:  []v1alpha1.IoMethod{v1alpha1.Read, v1alpha1.Write},
   274  			Duration: pointer.String("9m"),
   275  		},
   276  	}
   277  	err = cli.Create(ctx, ioChaos)
   278  	framework.ExpectNoError(err, "create io chaos")
   279  
   280  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   281  		res, err := getPodIoMistake(c, port)
   282  		if err != nil {
   283  			return false, nil
   284  		}
   285  		if err == nil && res {
   286  			return true, nil
   287  		}
   288  		return false, nil
   289  	})
   290  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
   291  
   292  	err = cli.Delete(ctx, ioChaos)
   293  	framework.ExpectNoError(err, "failed to delete io chaos")
   294  
   295  	klog.Infof("success to perform io chaos")
   296  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
   297  		res, err := getPodIoMistake(c, port)
   298  		if err != nil {
   299  			return false, nil
   300  		}
   301  		if err == nil && !res {
   302  			return true, nil
   303  		}
   304  		return false, nil
   305  	})
   306  	framework.ExpectNoError(err, "fail to recover io chaos")
   307  }
   308