network_partition.go

Documentation: github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/networkchaos

     1  // Copyright 2020 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package networkchaos
    15  
    16  import (
    17  	"context"
    18  	"net/http"
    19  	"strings"
    20  	"time"
    21  
    22  	. "github.com/onsi/ginkgo"
    23  	corev1 "k8s.io/api/core/v1"
    24  	"k8s.io/apimachinery/pkg/types"
    25  	"k8s.io/apimachinery/pkg/util/wait"
    26  	"k8s.io/client-go/kubernetes"
    27  	"k8s.io/klog"
    28  	"k8s.io/kubernetes/test/e2e/framework"
    29  	"k8s.io/utils/pointer"
    30  	"sigs.k8s.io/controller-runtime/pkg/client"
    31  
    32  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    33  	"github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/util"
    34  	"github.com/chaos-mesh/chaos-mesh/e2e-test/pkg/fixture"
    35  )
    36  
    37  // TestcaseForbidHostNetwork We do NOT allow that inject chaos on a pod which uses hostNetwork
    38  func TestcaseForbidHostNetwork(
    39  	ns string,
    40  	kubeCli kubernetes.Interface,
    41  	cli client.Client,
    42  ) {
    43  	ctx, cancel := context.WithCancel(context.Background())
    44  	defer cancel()
    45  
    46  	By("preparing experiment pods")
    47  	name := "network-peer-4"
    48  	nd := fixture.NewNetworkTestDeployment(name, ns, map[string]string{"partition": "0"})
    49  	nd.Spec.Template.Spec.HostNetwork = true
    50  	_, err := kubeCli.AppsV1().Deployments(ns).Create(nd)
    51  	framework.ExpectNoError(err, "create network-peer deployment error")
    52  	err = util.WaitDeploymentReady(name, ns, kubeCli)
    53  	framework.ExpectNoError(err, "wait network-peer deployment ready error")
    54  
    55  	By("create network partition chaos CRD objects")
    56  	networkPartition := makeNetworkPartitionChaos(
    57  		ns, "network-chaos-1",
    58  		map[string]string{"app": "network-peer-4"},
    59  		map[string]string{"app": "network-peer-1"},
    60  		v1alpha1.OnePodMode,
    61  		v1alpha1.OnePodMode,
    62  		v1alpha1.To,
    63  		pointer.StringPtr("9m"),
    64  	)
    65  
    66  	err = cli.Create(ctx, networkPartition.DeepCopy())
    67  	framework.ExpectNoError(err, "create network chaos error")
    68  
    69  	By("waiting for rejecting for network chaos with hostNetwork")
    70  	err = wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) {
    71  		err = cli.Get(ctx, types.NamespacedName{
    72  			Namespace: ns,
    73  			Name:      "network-chaos-1",
    74  		}, networkPartition)
    75  		if err != nil {
    76  			return false, err
    77  		}
    78  
    79  		failed := true
    80  		for _, record := range networkPartition.Status.ChaosStatus.Experiment.Records {
    81  			klog.Infof("current chaos record %s phase: %s", record.Id, record.Phase)
    82  			if strings.Contains(record.Id, "network-peer-4") && record.Phase == v1alpha1.Injected {
    83  				failed = false
    84  			}
    85  		}
    86  		return failed, nil
    87  	})
    88  
    89  	framework.ExpectNoError(err, "failed to waiting on not injected state with chaos")
    90  	framework.ExpectEqual(networkPartition.Status.ChaosStatus.Experiment.DesiredPhase, v1alpha1.RunningPhase)
    91  	// TODO: add failed event check
    92  	//framework.ExpectEqual(strings.Contains(networkPartition.Status.ChaosStatus.FailedMessage, "it's dangerous to inject network chaos on a pod"), true)
    93  }
    94  
    95  func TestcaseNetworkPartition(
    96  	ns string,
    97  	cli client.Client,
    98  	networkPeers []*corev1.Pod,
    99  	ports []uint16,
   100  	c http.Client,
   101  ) {
   102  	ctx, cancel := context.WithCancel(context.Background())
   103  	defer cancel()
   104  	By("prepare experiment playground")
   105  	for index := range networkPeers {
   106  		err := util.WaitE2EHelperReady(c, ports[index])
   107  
   108  		framework.ExpectNoError(err, "wait e2e helper ready error")
   109  	}
   110  
   111  	var result map[string][][]int
   112  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   113  		result = probeNetworkCondition(c, networkPeers, ports, false)
   114  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   115  			return false, nil
   116  		}
   117  		return true, nil
   118  	})
   119  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   120  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   121  
   122  	var (
   123  		testDelayDuration = pointer.StringPtr("9m")
   124  	)
   125  
   126  	baseNetworkPartition := makeNetworkPartitionChaos(
   127  		ns, "network-chaos-1",
   128  		map[string]string{"app": "network-peer-0"},
   129  		map[string]string{"app": "network-peer-1"},
   130  		v1alpha1.OnePodMode,
   131  		v1alpha1.OnePodMode,
   132  		v1alpha1.To,
   133  		testDelayDuration,
   134  	)
   135  
   136  	By("block from peer-0 to peer-1")
   137  	err := cli.Create(ctx, baseNetworkPartition.DeepCopy())
   138  	framework.ExpectNoError(err, "create network chaos error")
   139  
   140  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   141  		result = probeNetworkCondition(c, networkPeers, ports, false)
   142  		if len(result[networkConditionBlocked]) != 1 || len(result[networkConditionSlow]) != 0 {
   143  			return false, nil
   144  		}
   145  		return true, nil
   146  	})
   147  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}})
   148  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   149  
   150  	By("recover")
   151  	err = cli.Delete(ctx, baseNetworkPartition.DeepCopy())
   152  	framework.ExpectNoError(err, "delete network chaos error")
   153  
   154  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   155  		result = probeNetworkCondition(c, networkPeers, ports, false)
   156  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   157  			return false, nil
   158  		}
   159  		return true, nil
   160  	})
   161  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   162  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   163  
   164  	By("block both from peer-0 to peer-1 and from peer-1 to peer-0")
   165  	bothDirectionNetworkPartition := makeNetworkPartitionChaos(
   166  		ns, "network-chaos-1",
   167  		map[string]string{"app": "network-peer-0"},
   168  		map[string]string{"app": "network-peer-1"},
   169  		v1alpha1.OnePodMode,
   170  		v1alpha1.OnePodMode,
   171  		v1alpha1.Both,
   172  		testDelayDuration,
   173  	)
   174  	err = cli.Create(ctx, bothDirectionNetworkPartition.DeepCopy())
   175  	framework.ExpectNoError(err, "create network chaos error")
   176  
   177  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   178  		result = probeNetworkCondition(c, networkPeers, ports, false)
   179  		if len(result[networkConditionBlocked]) != 2 || len(result[networkConditionSlow]) != 0 {
   180  			return false, nil
   181  		}
   182  		return true, nil
   183  	})
   184  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}, {1, 0}})
   185  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   186  
   187  	By("recover")
   188  	err = cli.Delete(ctx, bothDirectionNetworkPartition.DeepCopy())
   189  	framework.ExpectNoError(err, "delete network chaos error")
   190  
   191  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   192  		result = probeNetworkCondition(c, networkPeers, ports, false)
   193  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   194  			return false, nil
   195  		}
   196  		return true, nil
   197  	})
   198  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   199  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   200  
   201  	By("block from peer-1 to peer-0")
   202  	fromDirectionNetworkPartition := makeNetworkPartitionChaos(
   203  		ns, "network-chaos-1",
   204  		map[string]string{"app": "network-peer-0"},
   205  		map[string]string{"app": "network-peer-1"},
   206  		v1alpha1.OnePodMode,
   207  		v1alpha1.OnePodMode,
   208  		v1alpha1.From,
   209  		testDelayDuration,
   210  	)
   211  
   212  	err = cli.Create(ctx, fromDirectionNetworkPartition.DeepCopy())
   213  	framework.ExpectNoError(err, "create network chaos error")
   214  
   215  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   216  		result = probeNetworkCondition(c, networkPeers, ports, false)
   217  		if len(result[networkConditionBlocked]) != 1 || len(result[networkConditionSlow]) != 0 {
   218  			return false, nil
   219  		}
   220  		return true, nil
   221  	})
   222  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{1, 0}})
   223  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   224  
   225  	By("recover")
   226  	err = cli.Delete(ctx, fromDirectionNetworkPartition.DeepCopy())
   227  	framework.ExpectNoError(err, "delete network chaos error")
   228  
   229  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   230  		result = probeNetworkCondition(c, networkPeers, ports, false)
   231  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   232  			return false, nil
   233  		}
   234  		return true, nil
   235  	})
   236  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   237  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   238  
   239  	By("network partition 1")
   240  
   241  	bothDirectionWithPartitionNetworkPartition := makeNetworkPartitionChaos(
   242  		ns, "network-chaos-1",
   243  		map[string]string{"app": "network-peer-0"},
   244  		map[string]string{"partition": "1"},
   245  		v1alpha1.OnePodMode,
   246  		v1alpha1.AllPodMode,
   247  		v1alpha1.Both,
   248  		testDelayDuration,
   249  	)
   250  	err = cli.Create(ctx, bothDirectionWithPartitionNetworkPartition.DeepCopy())
   251  	framework.ExpectNoError(err, "create network chaos error")
   252  
   253  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   254  		result = probeNetworkCondition(c, networkPeers, ports, false)
   255  		if len(result[networkConditionBlocked]) != 4 || len(result[networkConditionSlow]) != 0 {
   256  			return false, nil
   257  		}
   258  		return true, nil
   259  	})
   260  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}, {1, 0}, {0, 3}, {3, 0}})
   261  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   262  
   263  	By("recover")
   264  	err = cli.Delete(ctx, bothDirectionWithPartitionNetworkPartition.DeepCopy())
   265  	framework.ExpectNoError(err, "delete network chaos error")
   266  
   267  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   268  		result = probeNetworkCondition(c, networkPeers, ports, false)
   269  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   270  			return false, nil
   271  		}
   272  		return true, nil
   273  	})
   274  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   275  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   276  
   277  	By("multiple network partition chaos on peer-0")
   278  	anotherNetworkPartition := makeNetworkPartitionChaos(
   279  		ns, "network-chaos-2",
   280  		map[string]string{"app": "network-peer-0"},
   281  		map[string]string{"partition": "0"},
   282  		v1alpha1.OnePodMode,
   283  		v1alpha1.AllPodMode,
   284  		v1alpha1.To,
   285  		testDelayDuration,
   286  	)
   287  	err = cli.Create(ctx, bothDirectionWithPartitionNetworkPartition.DeepCopy())
   288  	framework.ExpectNoError(err, "create network chaos error")
   289  	err = cli.Create(ctx, anotherNetworkPartition.DeepCopy())
   290  	framework.ExpectNoError(err, "create network chaos error")
   291  
   292  	wait.Poll(time.Second, 30*time.Second, func() (done bool, err error) {
   293  		result = probeNetworkCondition(c, networkPeers, ports, false)
   294  		if len(result[networkConditionBlocked]) != 5 || len(result[networkConditionSlow]) != 0 {
   295  			return false, nil
   296  		}
   297  		return true, nil
   298  	})
   299  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}, {1, 0}, {0, 2}, {0, 3}, {3, 0}})
   300  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   301  
   302  	By("recover")
   303  	err = cli.Delete(ctx, bothDirectionWithPartitionNetworkPartition.DeepCopy())
   304  	framework.ExpectNoError(err, "delete network chaos error")
   305  	err = cli.Delete(ctx, anotherNetworkPartition.DeepCopy())
   306  	framework.ExpectNoError(err, "delete network chaos error")
   307  
   308  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   309  		klog.Info("retry probeNetworkCondition")
   310  		result = probeNetworkCondition(c, networkPeers, ports, false)
   311  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   312  			return false, nil
   313  		}
   314  		return true, nil
   315  	})
   316  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   317  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   318  
   319  	By("block from peer-0 to all")
   320  	networkPartitionWithoutTarget := makeNetworkPartitionChaos(
   321  		ns, "network-chaos-without-target",
   322  		map[string]string{"app": "network-peer-0"},
   323  		nil,
   324  		v1alpha1.OnePodMode,
   325  		v1alpha1.AllPodMode,
   326  		v1alpha1.To,
   327  		testDelayDuration,
   328  	)
   329  	err = cli.Create(ctx, networkPartitionWithoutTarget.DeepCopy())
   330  	framework.ExpectNoError(err, "create network chaos error")
   331  
   332  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   333  		result = probeNetworkCondition(c, networkPeers, ports, false)
   334  		if len(result[networkConditionBlocked]) != 5 || len(result[networkConditionSlow]) != 0 {
   335  			return false, nil
   336  		}
   337  		return true, nil
   338  	})
   339  	// The expected behavior is to block only 0 -> 1, 0 -> 2 and 0 -> 3
   340  	// Actually, 1 -> 0, 2 -> 0, 3 -> 0 are not blocked, but the `recvUDP`
   341  	// request failed due to partition.
   342  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}, {1, 0}, {0, 2}, {2, 0}, {0, 3}, {3, 0}})
   343  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   344  
   345  	By("recover")
   346  	err = cli.Delete(ctx, networkPartitionWithoutTarget.DeepCopy())
   347  	framework.ExpectNoError(err, "delete network chaos error")
   348  
   349  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   350  		klog.Info("retry probeNetworkCondition")
   351  		result = probeNetworkCondition(c, networkPeers, ports, false)
   352  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   353  			return false, nil
   354  		}
   355  		return true, nil
   356  	})
   357  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   358  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   359  
   360  	By("block from peer-0 from all")
   361  	networkPartitionWithoutTarget = makeNetworkPartitionChaos(
   362  		ns, "network-chaos-without-target",
   363  		map[string]string{"app": "network-peer-0"},
   364  		nil,
   365  		v1alpha1.OnePodMode,
   366  		v1alpha1.AllPodMode,
   367  		v1alpha1.From,
   368  		testDelayDuration,
   369  	)
   370  	err = cli.Create(ctx, networkPartitionWithoutTarget.DeepCopy())
   371  	framework.ExpectNoError(err, "create network chaos error")
   372  
   373  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   374  		result = probeNetworkCondition(c, networkPeers, ports, false)
   375  		if len(result[networkConditionBlocked]) != 5 || len(result[networkConditionSlow]) != 0 {
   376  			return false, nil
   377  		}
   378  		return true, nil
   379  	})
   380  	// The expected behavior is to block only 0 -> 1, 0 -> 2 and 0 -> 3
   381  	// Actually, 1 -> 0, 2 -> 0, 3 -> 0 are not blocked, but the `recvUDP`
   382  	// request failed due to partition.
   383  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}, {1, 0}, {0, 2}, {2, 0}, {0, 3}, {3, 0}})
   384  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   385  
   386  	By("recover")
   387  	err = cli.Delete(ctx, networkPartitionWithoutTarget.DeepCopy())
   388  	framework.ExpectNoError(err, "delete network chaos error")
   389  
   390  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   391  		klog.Info("retry probeNetworkCondition")
   392  		result = probeNetworkCondition(c, networkPeers, ports, false)
   393  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   394  			return false, nil
   395  		}
   396  		return true, nil
   397  	})
   398  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   399  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   400  }
   401