network_partition.go

Documentation: github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/networkchaos

     1  // Copyright 2021 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  // http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  
    16  package networkchaos
    17  
    18  import (
    19  	"context"
    20  	"net/http"
    21  	"strings"
    22  	"time"
    23  
    24  	. "github.com/onsi/ginkgo/v2"
    25  	corev1 "k8s.io/api/core/v1"
    26  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    27  	"k8s.io/apimachinery/pkg/types"
    28  	"k8s.io/apimachinery/pkg/util/wait"
    29  	"k8s.io/client-go/kubernetes"
    30  	"k8s.io/klog/v2"
    31  	"k8s.io/kubernetes/test/e2e/framework"
    32  	"k8s.io/utils/pointer"
    33  	"sigs.k8s.io/controller-runtime/pkg/client"
    34  
    35  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    36  	"github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/util"
    37  	"github.com/chaos-mesh/chaos-mesh/e2e-test/pkg/fixture"
    38  )
    39  
    40  // TestcaseForbidHostNetwork We do NOT allow that inject chaos on a pod which uses hostNetwork
    41  func TestcaseForbidHostNetwork(
    42  	ns string,
    43  	kubeCli kubernetes.Interface,
    44  	cli client.Client,
    45  ) {
    46  	ctx, cancel := context.WithCancel(context.Background())
    47  	defer cancel()
    48  
    49  	By("preparing experiment pods")
    50  	name := "network-peer-4"
    51  	nd := fixture.NewNetworkTestDeployment(name, ns, map[string]string{"partition": "0"})
    52  	nd.Spec.Template.Spec.HostNetwork = true
    53  	_, err := kubeCli.AppsV1().Deployments(ns).Create(context.TODO(), nd, metav1.CreateOptions{})
    54  	framework.ExpectNoError(err, "create network-peer deployment error")
    55  	err = util.WaitDeploymentReady(name, ns, kubeCli)
    56  	framework.ExpectNoError(err, "wait network-peer deployment ready error")
    57  
    58  	By("create network partition chaos CRD objects")
    59  	networkPartition := makeNetworkPartitionChaos(
    60  		ns, "network-chaos-1",
    61  		map[string]string{"app": "network-peer-4"},
    62  		map[string]string{"app": "network-peer-1"},
    63  		v1alpha1.OneMode,
    64  		v1alpha1.OneMode,
    65  		v1alpha1.To,
    66  		pointer.StringPtr("9m"),
    67  	)
    68  
    69  	err = cli.Create(ctx, networkPartition.DeepCopy())
    70  	framework.ExpectNoError(err, "create network chaos error")
    71  
    72  	By("waiting for rejecting for network chaos with hostNetwork")
    73  	err = wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) {
    74  		err = cli.Get(ctx, types.NamespacedName{
    75  			Namespace: ns,
    76  			Name:      "network-chaos-1",
    77  		}, networkPartition)
    78  		if err != nil {
    79  			return false, err
    80  		}
    81  
    82  		failed := true
    83  		for _, record := range networkPartition.Status.ChaosStatus.Experiment.Records {
    84  			klog.Infof("current chaos record %s phase: %s", record.Id, record.Phase)
    85  			if strings.Contains(record.Id, "network-peer-4") && record.Phase == v1alpha1.Injected {
    86  				failed = false
    87  			}
    88  		}
    89  		return failed, nil
    90  	})
    91  
    92  	framework.ExpectNoError(err, "failed to waiting on not injected state with chaos")
    93  	framework.ExpectEqual(networkPartition.Status.ChaosStatus.Experiment.DesiredPhase, v1alpha1.RunningPhase)
    94  	// TODO: add failed event check
    95  	//framework.ExpectEqual(strings.Contains(networkPartition.Status.ChaosStatus.FailedMessage, "it's dangerous to inject network chaos on a pod"), true)
    96  }
    97  
    98  func TestcaseNetworkPartition(
    99  	ns string,
   100  	cli client.Client,
   101  	networkPeers []*corev1.Pod,
   102  	ports []uint16,
   103  	c http.Client,
   104  ) {
   105  	ctx, cancel := context.WithCancel(context.Background())
   106  	defer cancel()
   107  	By("prepare experiment playground")
   108  	for index := range networkPeers {
   109  		err := util.WaitE2EHelperReady(c, ports[index])
   110  
   111  		framework.ExpectNoError(err, "wait e2e helper ready error")
   112  	}
   113  
   114  	var result map[string][][]int
   115  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   116  		result = probeNetworkCondition(c, networkPeers, ports, false)
   117  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   118  			return false, nil
   119  		}
   120  		return true, nil
   121  	})
   122  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   123  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   124  
   125  	var (
   126  		testDelayDuration = pointer.StringPtr("9m")
   127  	)
   128  
   129  	baseNetworkPartition := makeNetworkPartitionChaos(
   130  		ns, "network-chaos-1",
   131  		map[string]string{"app": "network-peer-0"},
   132  		map[string]string{"app": "network-peer-1"},
   133  		v1alpha1.OneMode,
   134  		v1alpha1.OneMode,
   135  		v1alpha1.To,
   136  		testDelayDuration,
   137  	)
   138  
   139  	By("block from peer-0 to peer-1")
   140  	err := cli.Create(ctx, baseNetworkPartition.DeepCopy())
   141  	framework.ExpectNoError(err, "create network chaos error")
   142  
   143  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   144  		result = probeNetworkCondition(c, networkPeers, ports, false)
   145  		if len(result[networkConditionBlocked]) != 1 || len(result[networkConditionSlow]) != 0 {
   146  			return false, nil
   147  		}
   148  		return true, nil
   149  	})
   150  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}})
   151  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   152  
   153  	By("recover")
   154  	err = cli.Delete(ctx, baseNetworkPartition.DeepCopy())
   155  	framework.ExpectNoError(err, "delete network chaos error")
   156  
   157  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   158  		result = probeNetworkCondition(c, networkPeers, ports, false)
   159  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   160  			return false, nil
   161  		}
   162  		return true, nil
   163  	})
   164  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   165  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   166  
   167  	By("block both from peer-0 to peer-1 and from peer-1 to peer-0")
   168  	bothDirectionNetworkPartition := makeNetworkPartitionChaos(
   169  		ns, "network-chaos-1",
   170  		map[string]string{"app": "network-peer-0"},
   171  		map[string]string{"app": "network-peer-1"},
   172  		v1alpha1.OneMode,
   173  		v1alpha1.OneMode,
   174  		v1alpha1.Both,
   175  		testDelayDuration,
   176  	)
   177  	err = cli.Create(ctx, bothDirectionNetworkPartition.DeepCopy())
   178  	framework.ExpectNoError(err, "create network chaos error")
   179  
   180  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   181  		result = probeNetworkCondition(c, networkPeers, ports, false)
   182  		if len(result[networkConditionBlocked]) != 2 || len(result[networkConditionSlow]) != 0 {
   183  			return false, nil
   184  		}
   185  		return true, nil
   186  	})
   187  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}, {1, 0}})
   188  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   189  
   190  	By("recover")
   191  	err = cli.Delete(ctx, bothDirectionNetworkPartition.DeepCopy())
   192  	framework.ExpectNoError(err, "delete network chaos error")
   193  
   194  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   195  		result = probeNetworkCondition(c, networkPeers, ports, false)
   196  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   197  			return false, nil
   198  		}
   199  		return true, nil
   200  	})
   201  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   202  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   203  
   204  	By("block from peer-1 to peer-0")
   205  	fromDirectionNetworkPartition := makeNetworkPartitionChaos(
   206  		ns, "network-chaos-1",
   207  		map[string]string{"app": "network-peer-0"},
   208  		map[string]string{"app": "network-peer-1"},
   209  		v1alpha1.OneMode,
   210  		v1alpha1.OneMode,
   211  		v1alpha1.From,
   212  		testDelayDuration,
   213  	)
   214  
   215  	err = cli.Create(ctx, fromDirectionNetworkPartition.DeepCopy())
   216  	framework.ExpectNoError(err, "create network chaos error")
   217  
   218  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   219  		result = probeNetworkCondition(c, networkPeers, ports, false)
   220  		if len(result[networkConditionBlocked]) != 1 || len(result[networkConditionSlow]) != 0 {
   221  			return false, nil
   222  		}
   223  		return true, nil
   224  	})
   225  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{1, 0}})
   226  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   227  
   228  	By("recover")
   229  	err = cli.Delete(ctx, fromDirectionNetworkPartition.DeepCopy())
   230  	framework.ExpectNoError(err, "delete network chaos error")
   231  
   232  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   233  		result = probeNetworkCondition(c, networkPeers, ports, false)
   234  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   235  			return false, nil
   236  		}
   237  		return true, nil
   238  	})
   239  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   240  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   241  
   242  	By("network partition 1")
   243  
   244  	bothDirectionWithPartitionNetworkPartition := makeNetworkPartitionChaos(
   245  		ns, "network-chaos-1",
   246  		map[string]string{"app": "network-peer-0"},
   247  		map[string]string{"partition": "1"},
   248  		v1alpha1.OneMode,
   249  		v1alpha1.AllMode,
   250  		v1alpha1.Both,
   251  		testDelayDuration,
   252  	)
   253  	err = cli.Create(ctx, bothDirectionWithPartitionNetworkPartition.DeepCopy())
   254  	framework.ExpectNoError(err, "create network chaos error")
   255  
   256  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   257  		result = probeNetworkCondition(c, networkPeers, ports, false)
   258  		if len(result[networkConditionBlocked]) != 4 || len(result[networkConditionSlow]) != 0 {
   259  			return false, nil
   260  		}
   261  		return true, nil
   262  	})
   263  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}, {1, 0}, {0, 3}, {3, 0}})
   264  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   265  
   266  	By("recover")
   267  	err = cli.Delete(ctx, bothDirectionWithPartitionNetworkPartition.DeepCopy())
   268  	framework.ExpectNoError(err, "delete network chaos error")
   269  
   270  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   271  		result = probeNetworkCondition(c, networkPeers, ports, false)
   272  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   273  			return false, nil
   274  		}
   275  		return true, nil
   276  	})
   277  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   278  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   279  
   280  	By("multiple network partition chaos on peer-0")
   281  	anotherNetworkPartition := makeNetworkPartitionChaos(
   282  		ns, "network-chaos-2",
   283  		map[string]string{"app": "network-peer-0"},
   284  		map[string]string{"partition": "0"},
   285  		v1alpha1.OneMode,
   286  		v1alpha1.AllMode,
   287  		v1alpha1.To,
   288  		testDelayDuration,
   289  	)
   290  	err = cli.Create(ctx, bothDirectionWithPartitionNetworkPartition.DeepCopy())
   291  	framework.ExpectNoError(err, "create network chaos error")
   292  	err = cli.Create(ctx, anotherNetworkPartition.DeepCopy())
   293  	framework.ExpectNoError(err, "create network chaos error")
   294  
   295  	wait.Poll(time.Second, 30*time.Second, func() (done bool, err error) {
   296  		result = probeNetworkCondition(c, networkPeers, ports, false)
   297  		if len(result[networkConditionBlocked]) != 5 || len(result[networkConditionSlow]) != 0 {
   298  			return false, nil
   299  		}
   300  		return true, nil
   301  	})
   302  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}, {1, 0}, {0, 2}, {0, 3}, {3, 0}})
   303  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   304  
   305  	By("recover")
   306  	err = cli.Delete(ctx, bothDirectionWithPartitionNetworkPartition.DeepCopy())
   307  	framework.ExpectNoError(err, "delete network chaos error")
   308  	err = cli.Delete(ctx, anotherNetworkPartition.DeepCopy())
   309  	framework.ExpectNoError(err, "delete network chaos error")
   310  
   311  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   312  		klog.Info("retry probeNetworkCondition")
   313  		result = probeNetworkCondition(c, networkPeers, ports, false)
   314  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   315  			return false, nil
   316  		}
   317  		return true, nil
   318  	})
   319  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   320  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   321  
   322  	By("block from peer-0 to all")
   323  	networkPartitionWithoutTarget := makeNetworkPartitionChaos(
   324  		ns, "network-chaos-without-target",
   325  		map[string]string{"app": "network-peer-0"},
   326  		nil,
   327  		v1alpha1.OneMode,
   328  		v1alpha1.AllMode,
   329  		v1alpha1.To,
   330  		testDelayDuration,
   331  	)
   332  	err = cli.Create(ctx, networkPartitionWithoutTarget.DeepCopy())
   333  	framework.ExpectNoError(err, "create network chaos error")
   334  
   335  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   336  		result = probeNetworkCondition(c, networkPeers, ports, false)
   337  		if len(result[networkConditionBlocked]) != 3 || len(result[networkConditionSlow]) != 0 {
   338  			return false, nil
   339  		}
   340  		return true, nil
   341  	})
   342  	// The expected behavior is to block only 0 -> 1, 0 -> 2 and 0 -> 3
   343  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}, {0, 2}, {0, 3}})
   344  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   345  
   346  	By("recover")
   347  	err = cli.Delete(ctx, networkPartitionWithoutTarget.DeepCopy())
   348  	framework.ExpectNoError(err, "delete network chaos error")
   349  
   350  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   351  		klog.Info("retry probeNetworkCondition")
   352  		result = probeNetworkCondition(c, networkPeers, ports, false)
   353  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   354  			return false, nil
   355  		}
   356  		return true, nil
   357  	})
   358  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   359  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   360  
   361  	By("block from peer-0 from all")
   362  	networkPartitionWithoutTarget = makeNetworkPartitionChaos(
   363  		ns, "network-chaos-without-target",
   364  		map[string]string{"app": "network-peer-0"},
   365  		nil,
   366  		v1alpha1.OneMode,
   367  		v1alpha1.AllMode,
   368  		v1alpha1.From,
   369  		testDelayDuration,
   370  	)
   371  	err = cli.Create(ctx, networkPartitionWithoutTarget.DeepCopy())
   372  	framework.ExpectNoError(err, "create network chaos error")
   373  
   374  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   375  		result = probeNetworkCondition(c, networkPeers, ports, true)
   376  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   377  			return false, nil
   378  		}
   379  		return true, nil
   380  	})
   381  	// The expected behavior is to block only 0 -> 1, 0 -> 2 and 0 -> 3
   382  	// but the dropped packet will not throw an error
   383  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   384  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   385  
   386  	By("recover")
   387  	err = cli.Delete(ctx, networkPartitionWithoutTarget.DeepCopy())
   388  	framework.ExpectNoError(err, "delete network chaos error")
   389  
   390  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   391  		klog.Info("retry probeNetworkCondition")
   392  		result = probeNetworkCondition(c, networkPeers, ports, false)
   393  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   394  			return false, nil
   395  		}
   396  		return true, nil
   397  	})
   398  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   399  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   400  }
   401