io_graceful_shutdown.go

Documentation: github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/iochaos

     1  // Copyright 2021 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  // http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  
    16  package iochaos
    17  
    18  import (
    19  	"context"
    20  	"net/http"
    21  	"strings"
    22  	"time"
    23  
    24  	. "github.com/onsi/ginkgo/v2"
    25  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    26  	"k8s.io/apimachinery/pkg/util/wait"
    27  	"k8s.io/kubernetes/test/e2e/framework"
    28  	"sigs.k8s.io/controller-runtime/pkg/client"
    29  
    30  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    31  	test "github.com/chaos-mesh/chaos-mesh/e2e-test"
    32  	e2econfig "github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/config"
    33  	"github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/util"
    34  )
    35  
    36  func TestcaseIOErrorGracefulShutdown(
    37  	ns string,
    38  	cli client.Client,
    39  	c http.Client,
    40  	port uint16,
    41  ) {
    42  	ctx, cancel := context.WithCancel(context.Background())
    43  	defer cancel()
    44  	err := util.WaitE2EHelperReady(c, port)
    45  	framework.ExpectNoError(err, "wait e2e helper ready error")
    46  
    47  	ioChaos := &v1alpha1.IOChaos{
    48  		ObjectMeta: metav1.ObjectMeta{
    49  			Name:      "io-chaos",
    50  			Namespace: ns,
    51  		},
    52  		Spec: v1alpha1.IOChaosSpec{
    53  			Action:     v1alpha1.IoFaults,
    54  			VolumePath: "/var/run/data",
    55  			Path:       "/var/run/data/*",
    56  			Percent:    100,
    57  			// errno 5 is EIO -> I/O error
    58  			Errno: 5,
    59  			// only inject write method
    60  			Methods: []v1alpha1.IoMethod{v1alpha1.Write},
    61  			ContainerSelector: v1alpha1.ContainerSelector{
    62  				PodSelector: v1alpha1.PodSelector{
    63  					Selector: v1alpha1.PodSelectorSpec{
    64  						GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
    65  							Namespaces:     []string{ns},
    66  							LabelSelectors: map[string]string{"app": "io"},
    67  						},
    68  					},
    69  					Mode: v1alpha1.OneMode,
    70  				},
    71  			},
    72  		},
    73  	}
    74  	err = cli.Create(ctx, ioChaos)
    75  	framework.ExpectNoError(err, "create io chaos")
    76  
    77  	defer func() {
    78  		err = cli.Delete(ctx, ioChaos)
    79  		framework.ExpectNoError(err, "delete io chaos")
    80  	}()
    81  
    82  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
    83  		_, err = getPodIODelay(c, port)
    84  		// input/output error is errno 5
    85  		if err != nil && strings.Contains(err.Error(), "input/output error") {
    86  			return true, nil
    87  		}
    88  		return false, nil
    89  	})
    90  	framework.ExpectNoError(err, "io chaos doesn't work as expected")
    91  
    92  	By("upgrade chaos mesh")
    93  	// Get clients
    94  	oa, ocfg, err := test.BuildOperatorActionAndCfg(e2econfig.TestConfig)
    95  	framework.ExpectNoError(err, "failed to create operator action")
    96  	err = oa.RestartDaemon(ocfg)
    97  	framework.ExpectNoError(err, "failed to restart chaos daemon")
    98  
    99  	By("waiting for assertion IO error recovery")
   100  	err = wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) {
   101  		_, err = getPodIODelay(c, port)
   102  		// recovered
   103  		if err == nil {
   104  			return true, nil
   105  		}
   106  		return false, nil
   107  	})
   108  	framework.ExpectNoError(err, "io chaos doesn't gracefully shutdown as expected")
   109  	By("io chaos shutdown successfully")
   110  }
   111