network_delay.go

Documentation: github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/networkchaos

     1  // Copyright 2021 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  // http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  
    16  package networkchaos
    17  
    18  import (
    19  	"context"
    20  	"net/http"
    21  	"time"
    22  
    23  	. "github.com/onsi/ginkgo/v2"
    24  	corev1 "k8s.io/api/core/v1"
    25  	"k8s.io/apimachinery/pkg/util/wait"
    26  	"k8s.io/kubernetes/test/e2e/framework"
    27  	"k8s.io/utils/pointer"
    28  	"sigs.k8s.io/controller-runtime/pkg/client"
    29  
    30  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    31  	"github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/util"
    32  )
    33  
    34  func TestcaseNetworkDelay(
    35  	ns string,
    36  	cli client.Client,
    37  	networkPeers []*corev1.Pod,
    38  	ports []uint16,
    39  	c http.Client,
    40  ) {
    41  	ctx, cancel := context.WithCancel(context.Background())
    42  	defer cancel()
    43  
    44  	By("prepare experiment playground")
    45  	for index := range networkPeers {
    46  		err := util.WaitE2EHelperReady(c, ports[index])
    47  
    48  		framework.ExpectNoError(err, "wait e2e helper ready error")
    49  	}
    50  
    51  	result := probeNetworkCondition(c, networkPeers, ports, false)
    52  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
    53  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
    54  
    55  	var (
    56  		testDelayTcParam = v1alpha1.TcParameter{
    57  			Delay: &v1alpha1.DelaySpec{
    58  				Latency:     "200ms",
    59  				Correlation: "25",
    60  				Jitter:      "0ms",
    61  			},
    62  		}
    63  		testDelayTcParamEvenMoreComplicate = v1alpha1.TcParameter{
    64  			Delay: &v1alpha1.DelaySpec{
    65  				Latency:     "200ms",
    66  				Correlation: "25",
    67  				Jitter:      "0ms",
    68  			},
    69  			Loss: &v1alpha1.LossSpec{
    70  				Loss:        "25",
    71  				Correlation: "25",
    72  			},
    73  			Duplicate: &v1alpha1.DuplicateSpec{
    74  				Duplicate:   "25",
    75  				Correlation: "25",
    76  			},
    77  			Corrupt: &v1alpha1.CorruptSpec{
    78  				Corrupt:     "25",
    79  				Correlation: "25",
    80  			},
    81  		}
    82  		testDelayDuration = pointer.String("9m")
    83  	)
    84  
    85  	By("normal delay chaos")
    86  	networkDelay := makeNetworkDelayChaos(
    87  		ns, "network-chaos-1",
    88  		map[string]string{"app": "network-peer-0"},
    89  		nil, // no target specified
    90  		v1alpha1.OneMode,
    91  		v1alpha1.OneMode,
    92  		v1alpha1.To,
    93  		testDelayTcParam,
    94  		testDelayDuration,
    95  	)
    96  	By("Injecting delay for 0")
    97  	err := cli.Create(ctx, networkDelay.DeepCopy())
    98  	framework.ExpectNoError(err, "create network chaos error")
    99  
   100  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   101  		result = probeNetworkCondition(c, networkPeers, ports, false)
   102  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 3 {
   103  			return false, nil
   104  		}
   105  		return true, nil
   106  	})
   107  
   108  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   109  	framework.ExpectEqual(result[networkConditionSlow], [][]int{{0, 1}, {0, 2}, {0, 3}})
   110  
   111  	By("recover")
   112  	err = cli.Delete(ctx, networkDelay.DeepCopy())
   113  	framework.ExpectNoError(err, "delete network chaos error")
   114  
   115  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   116  		result = probeNetworkCondition(c, networkPeers, ports, false)
   117  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   118  			return false, nil
   119  		}
   120  		return true, nil
   121  	})
   122  
   123  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   124  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   125  
   126  	networkDelayWithTarget := makeNetworkDelayChaos(
   127  		ns, "network-chaos-1",
   128  		map[string]string{"app": "network-peer-0"},
   129  		map[string]string{"app": "network-peer-1"}, // 0 -> 1 add delays
   130  		v1alpha1.OneMode,
   131  		v1alpha1.OneMode,
   132  		v1alpha1.To,
   133  		testDelayTcParam,
   134  		testDelayDuration,
   135  	)
   136  
   137  	By("Injecting delay for 0 -> 1")
   138  	err = cli.Create(ctx, networkDelayWithTarget.DeepCopy())
   139  	framework.ExpectNoError(err, "create network chaos error")
   140  
   141  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   142  		result = probeNetworkCondition(c, networkPeers, ports, false)
   143  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 1 {
   144  			return false, nil
   145  		}
   146  		return true, nil
   147  	})
   148  
   149  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   150  	framework.ExpectEqual(result[networkConditionSlow], [][]int{{0, 1}})
   151  
   152  	err = cli.Delete(ctx, networkDelayWithTarget.DeepCopy())
   153  	framework.ExpectNoError(err, "delete network chaos error")
   154  
   155  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   156  		result = probeNetworkCondition(c, networkPeers, ports, false)
   157  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   158  			return false, nil
   159  		}
   160  		return true, nil
   161  	})
   162  
   163  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   164  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   165  
   166  	evenNetworkDelay := makeNetworkDelayChaos(
   167  		ns, "network-chaos-2",
   168  		map[string]string{"app": "network-peer-0"},
   169  		map[string]string{"partition": "0"}, // 0 -> even its partition (idx % 2)
   170  		v1alpha1.OneMode,
   171  		v1alpha1.AllMode,
   172  		v1alpha1.To,
   173  		testDelayTcParam,
   174  		testDelayDuration,
   175  	)
   176  	By("Injecting delay for 0 -> even partition")
   177  	err = cli.Create(ctx, evenNetworkDelay.DeepCopy())
   178  	framework.ExpectNoError(err, "create network chaos error")
   179  
   180  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   181  		result = probeNetworkCondition(c, networkPeers, ports, false)
   182  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 1 {
   183  			return false, nil
   184  		}
   185  		return true, nil
   186  	})
   187  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   188  	framework.ExpectEqual(result[networkConditionSlow], [][]int{{0, 2}})
   189  
   190  	By("Injecting delay for 0 -> 1")
   191  	err = cli.Create(ctx, networkDelayWithTarget.DeepCopy())
   192  	framework.ExpectNoError(err, "create network chaos error")
   193  
   194  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   195  		result = probeNetworkCondition(c, networkPeers, ports, false)
   196  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 2 {
   197  			return false, nil
   198  		}
   199  		return true, nil
   200  	})
   201  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   202  	framework.ExpectEqual(result[networkConditionSlow], [][]int{{0, 1}, {0, 2}})
   203  
   204  	err = cli.Delete(ctx, networkDelayWithTarget.DeepCopy())
   205  	framework.ExpectNoError(err, "delete network chaos error")
   206  
   207  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   208  		result = probeNetworkCondition(c, networkPeers, ports, false)
   209  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 1 {
   210  			return false, nil
   211  		}
   212  		return true, nil
   213  	})
   214  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   215  	framework.ExpectEqual(result[networkConditionSlow], [][]int{{0, 2}})
   216  
   217  	err = cli.Delete(ctx, evenNetworkDelay.DeepCopy())
   218  	framework.ExpectNoError(err, "delete network chaos error")
   219  
   220  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   221  		result = probeNetworkCondition(c, networkPeers, ports, false)
   222  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   223  			return false, nil
   224  		}
   225  		return true, nil
   226  	})
   227  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   228  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   229  
   230  	complicateNetem := makeNetworkDelayChaos(
   231  		ns, "network-chaos-3",
   232  		map[string]string{"app": "network-peer-0"},
   233  		nil, // no target specified
   234  		v1alpha1.OneMode,
   235  		v1alpha1.OneMode,
   236  		v1alpha1.To,
   237  		testDelayTcParamEvenMoreComplicate,
   238  		testDelayDuration,
   239  	)
   240  	By("Injecting complicate chaos for 0")
   241  	err = cli.Create(ctx, complicateNetem.DeepCopy())
   242  	framework.ExpectNoError(err, "create network chaos error")
   243  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   244  		result = probeNetworkCondition(c, networkPeers, ports, false)
   245  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 3 {
   246  			return false, nil
   247  		}
   248  		return true, nil
   249  	})
   250  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   251  	framework.ExpectEqual(result[networkConditionSlow], [][]int{{0, 1}, {0, 2}, {0, 3}})
   252  
   253  	By("recover")
   254  	err = cli.Delete(ctx, complicateNetem.DeepCopy())
   255  	framework.ExpectNoError(err, "delete network chaos error")
   256  
   257  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   258  		result = probeNetworkCondition(c, networkPeers, ports, false)
   259  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   260  			return false, nil
   261  		}
   262  		return true, nil
   263  	})
   264  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   265  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   266  
   267  	bothDirectionNetem := makeNetworkDelayChaos(
   268  		ns, "network-chaos-4",
   269  		map[string]string{"app": "network-peer-0"},
   270  		map[string]string{"partition": "0"}, // 0 -> even its partition (idx % 2)
   271  		v1alpha1.OneMode,
   272  		v1alpha1.AllMode,
   273  		v1alpha1.Both,
   274  		testDelayTcParam,
   275  		testDelayDuration,
   276  	)
   277  	By("Injecting both direction chaos for 0")
   278  	err = cli.Create(ctx, bothDirectionNetem.DeepCopy())
   279  	framework.ExpectNoError(err, "create network chaos error")
   280  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   281  		result = probeNetworkCondition(c, networkPeers, ports, true)
   282  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 2 {
   283  			return false, nil
   284  		}
   285  		return true, nil
   286  	})
   287  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   288  	framework.ExpectEqual(result[networkConditionSlow], [][]int{{0, 2}, {2, 0}})
   289  
   290  	By("recover")
   291  	err = cli.Delete(ctx, bothDirectionNetem.DeepCopy())
   292  	framework.ExpectNoError(err, "delete network chaos error")
   293  
   294  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   295  		result = probeNetworkCondition(c, networkPeers, ports, true)
   296  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   297  			return false, nil
   298  		}
   299  		return true, nil
   300  	})
   301  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   302  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   303  }
   304