network_partition.go

Documentation: github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/networkchaos

     1  // Copyright 2021 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  // http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  
    16  package networkchaos
    17  
    18  import (
    19  	"context"
    20  	"net/http"
    21  	"strings"
    22  	"time"
    23  
    24  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    25  
    26  	. "github.com/onsi/ginkgo"
    27  	corev1 "k8s.io/api/core/v1"
    28  	"k8s.io/apimachinery/pkg/types"
    29  	"k8s.io/apimachinery/pkg/util/wait"
    30  	"k8s.io/client-go/kubernetes"
    31  	"k8s.io/klog/v2"
    32  	"k8s.io/kubernetes/test/e2e/framework"
    33  	"k8s.io/utils/pointer"
    34  	"sigs.k8s.io/controller-runtime/pkg/client"
    35  
    36  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    37  	"github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/util"
    38  	"github.com/chaos-mesh/chaos-mesh/e2e-test/pkg/fixture"
    39  )
    40  
    41  // TestcaseForbidHostNetwork We do NOT allow that inject chaos on a pod which uses hostNetwork
    42  func TestcaseForbidHostNetwork(
    43  	ns string,
    44  	kubeCli kubernetes.Interface,
    45  	cli client.Client,
    46  ) {
    47  	ctx, cancel := context.WithCancel(context.Background())
    48  	defer cancel()
    49  
    50  	By("preparing experiment pods")
    51  	name := "network-peer-4"
    52  	nd := fixture.NewNetworkTestDeployment(name, ns, map[string]string{"partition": "0"})
    53  	nd.Spec.Template.Spec.HostNetwork = true
    54  	_, err := kubeCli.AppsV1().Deployments(ns).Create(context.TODO(), nd, metav1.CreateOptions{})
    55  	framework.ExpectNoError(err, "create network-peer deployment error")
    56  	err = util.WaitDeploymentReady(name, ns, kubeCli)
    57  	framework.ExpectNoError(err, "wait network-peer deployment ready error")
    58  
    59  	By("create network partition chaos CRD objects")
    60  	networkPartition := makeNetworkPartitionChaos(
    61  		ns, "network-chaos-1",
    62  		map[string]string{"app": "network-peer-4"},
    63  		map[string]string{"app": "network-peer-1"},
    64  		v1alpha1.OneMode,
    65  		v1alpha1.OneMode,
    66  		v1alpha1.To,
    67  		pointer.StringPtr("9m"),
    68  	)
    69  
    70  	err = cli.Create(ctx, networkPartition.DeepCopy())
    71  	framework.ExpectNoError(err, "create network chaos error")
    72  
    73  	By("waiting for rejecting for network chaos with hostNetwork")
    74  	err = wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) {
    75  		err = cli.Get(ctx, types.NamespacedName{
    76  			Namespace: ns,
    77  			Name:      "network-chaos-1",
    78  		}, networkPartition)
    79  		if err != nil {
    80  			return false, err
    81  		}
    82  
    83  		failed := true
    84  		for _, record := range networkPartition.Status.ChaosStatus.Experiment.Records {
    85  			klog.Infof("current chaos record %s phase: %s", record.Id, record.Phase)
    86  			if strings.Contains(record.Id, "network-peer-4") && record.Phase == v1alpha1.Injected {
    87  				failed = false
    88  			}
    89  		}
    90  		return failed, nil
    91  	})
    92  
    93  	framework.ExpectNoError(err, "failed to waiting on not injected state with chaos")
    94  	framework.ExpectEqual(networkPartition.Status.ChaosStatus.Experiment.DesiredPhase, v1alpha1.RunningPhase)
    95  	// TODO: add failed event check
    96  	//framework.ExpectEqual(strings.Contains(networkPartition.Status.ChaosStatus.FailedMessage, "it's dangerous to inject network chaos on a pod"), true)
    97  }
    98  
    99  func TestcaseNetworkPartition(
   100  	ns string,
   101  	cli client.Client,
   102  	networkPeers []*corev1.Pod,
   103  	ports []uint16,
   104  	c http.Client,
   105  ) {
   106  	ctx, cancel := context.WithCancel(context.Background())
   107  	defer cancel()
   108  	By("prepare experiment playground")
   109  	for index := range networkPeers {
   110  		err := util.WaitE2EHelperReady(c, ports[index])
   111  
   112  		framework.ExpectNoError(err, "wait e2e helper ready error")
   113  	}
   114  
   115  	var result map[string][][]int
   116  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   117  		result = probeNetworkCondition(c, networkPeers, ports, false)
   118  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   119  			return false, nil
   120  		}
   121  		return true, nil
   122  	})
   123  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   124  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   125  
   126  	var (
   127  		testDelayDuration = pointer.StringPtr("9m")
   128  	)
   129  
   130  	baseNetworkPartition := makeNetworkPartitionChaos(
   131  		ns, "network-chaos-1",
   132  		map[string]string{"app": "network-peer-0"},
   133  		map[string]string{"app": "network-peer-1"},
   134  		v1alpha1.OneMode,
   135  		v1alpha1.OneMode,
   136  		v1alpha1.To,
   137  		testDelayDuration,
   138  	)
   139  
   140  	By("block from peer-0 to peer-1")
   141  	err := cli.Create(ctx, baseNetworkPartition.DeepCopy())
   142  	framework.ExpectNoError(err, "create network chaos error")
   143  
   144  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   145  		result = probeNetworkCondition(c, networkPeers, ports, false)
   146  		if len(result[networkConditionBlocked]) != 1 || len(result[networkConditionSlow]) != 0 {
   147  			return false, nil
   148  		}
   149  		return true, nil
   150  	})
   151  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}})
   152  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   153  
   154  	By("recover")
   155  	err = cli.Delete(ctx, baseNetworkPartition.DeepCopy())
   156  	framework.ExpectNoError(err, "delete network chaos error")
   157  
   158  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   159  		result = probeNetworkCondition(c, networkPeers, ports, false)
   160  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   161  			return false, nil
   162  		}
   163  		return true, nil
   164  	})
   165  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   166  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   167  
   168  	By("block both from peer-0 to peer-1 and from peer-1 to peer-0")
   169  	bothDirectionNetworkPartition := makeNetworkPartitionChaos(
   170  		ns, "network-chaos-1",
   171  		map[string]string{"app": "network-peer-0"},
   172  		map[string]string{"app": "network-peer-1"},
   173  		v1alpha1.OneMode,
   174  		v1alpha1.OneMode,
   175  		v1alpha1.Both,
   176  		testDelayDuration,
   177  	)
   178  	err = cli.Create(ctx, bothDirectionNetworkPartition.DeepCopy())
   179  	framework.ExpectNoError(err, "create network chaos error")
   180  
   181  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   182  		result = probeNetworkCondition(c, networkPeers, ports, false)
   183  		if len(result[networkConditionBlocked]) != 2 || len(result[networkConditionSlow]) != 0 {
   184  			return false, nil
   185  		}
   186  		return true, nil
   187  	})
   188  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}, {1, 0}})
   189  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   190  
   191  	By("recover")
   192  	err = cli.Delete(ctx, bothDirectionNetworkPartition.DeepCopy())
   193  	framework.ExpectNoError(err, "delete network chaos error")
   194  
   195  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   196  		result = probeNetworkCondition(c, networkPeers, ports, false)
   197  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   198  			return false, nil
   199  		}
   200  		return true, nil
   201  	})
   202  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   203  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   204  
   205  	By("block from peer-1 to peer-0")
   206  	fromDirectionNetworkPartition := makeNetworkPartitionChaos(
   207  		ns, "network-chaos-1",
   208  		map[string]string{"app": "network-peer-0"},
   209  		map[string]string{"app": "network-peer-1"},
   210  		v1alpha1.OneMode,
   211  		v1alpha1.OneMode,
   212  		v1alpha1.From,
   213  		testDelayDuration,
   214  	)
   215  
   216  	err = cli.Create(ctx, fromDirectionNetworkPartition.DeepCopy())
   217  	framework.ExpectNoError(err, "create network chaos error")
   218  
   219  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   220  		result = probeNetworkCondition(c, networkPeers, ports, false)
   221  		if len(result[networkConditionBlocked]) != 1 || len(result[networkConditionSlow]) != 0 {
   222  			return false, nil
   223  		}
   224  		return true, nil
   225  	})
   226  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{1, 0}})
   227  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   228  
   229  	By("recover")
   230  	err = cli.Delete(ctx, fromDirectionNetworkPartition.DeepCopy())
   231  	framework.ExpectNoError(err, "delete network chaos error")
   232  
   233  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   234  		result = probeNetworkCondition(c, networkPeers, ports, false)
   235  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   236  			return false, nil
   237  		}
   238  		return true, nil
   239  	})
   240  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   241  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   242  
   243  	By("network partition 1")
   244  
   245  	bothDirectionWithPartitionNetworkPartition := makeNetworkPartitionChaos(
   246  		ns, "network-chaos-1",
   247  		map[string]string{"app": "network-peer-0"},
   248  		map[string]string{"partition": "1"},
   249  		v1alpha1.OneMode,
   250  		v1alpha1.AllMode,
   251  		v1alpha1.Both,
   252  		testDelayDuration,
   253  	)
   254  	err = cli.Create(ctx, bothDirectionWithPartitionNetworkPartition.DeepCopy())
   255  	framework.ExpectNoError(err, "create network chaos error")
   256  
   257  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   258  		result = probeNetworkCondition(c, networkPeers, ports, false)
   259  		if len(result[networkConditionBlocked]) != 4 || len(result[networkConditionSlow]) != 0 {
   260  			return false, nil
   261  		}
   262  		return true, nil
   263  	})
   264  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}, {1, 0}, {0, 3}, {3, 0}})
   265  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   266  
   267  	By("recover")
   268  	err = cli.Delete(ctx, bothDirectionWithPartitionNetworkPartition.DeepCopy())
   269  	framework.ExpectNoError(err, "delete network chaos error")
   270  
   271  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   272  		result = probeNetworkCondition(c, networkPeers, ports, false)
   273  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   274  			return false, nil
   275  		}
   276  		return true, nil
   277  	})
   278  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   279  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   280  
   281  	By("multiple network partition chaos on peer-0")
   282  	anotherNetworkPartition := makeNetworkPartitionChaos(
   283  		ns, "network-chaos-2",
   284  		map[string]string{"app": "network-peer-0"},
   285  		map[string]string{"partition": "0"},
   286  		v1alpha1.OneMode,
   287  		v1alpha1.AllMode,
   288  		v1alpha1.To,
   289  		testDelayDuration,
   290  	)
   291  	err = cli.Create(ctx, bothDirectionWithPartitionNetworkPartition.DeepCopy())
   292  	framework.ExpectNoError(err, "create network chaos error")
   293  	err = cli.Create(ctx, anotherNetworkPartition.DeepCopy())
   294  	framework.ExpectNoError(err, "create network chaos error")
   295  
   296  	wait.Poll(time.Second, 30*time.Second, func() (done bool, err error) {
   297  		result = probeNetworkCondition(c, networkPeers, ports, false)
   298  		if len(result[networkConditionBlocked]) != 5 || len(result[networkConditionSlow]) != 0 {
   299  			return false, nil
   300  		}
   301  		return true, nil
   302  	})
   303  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}, {1, 0}, {0, 2}, {0, 3}, {3, 0}})
   304  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   305  
   306  	By("recover")
   307  	err = cli.Delete(ctx, bothDirectionWithPartitionNetworkPartition.DeepCopy())
   308  	framework.ExpectNoError(err, "delete network chaos error")
   309  	err = cli.Delete(ctx, anotherNetworkPartition.DeepCopy())
   310  	framework.ExpectNoError(err, "delete network chaos error")
   311  
   312  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   313  		klog.Info("retry probeNetworkCondition")
   314  		result = probeNetworkCondition(c, networkPeers, ports, false)
   315  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   316  			return false, nil
   317  		}
   318  		return true, nil
   319  	})
   320  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   321  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   322  
   323  	By("block from peer-0 to all")
   324  	networkPartitionWithoutTarget := makeNetworkPartitionChaos(
   325  		ns, "network-chaos-without-target",
   326  		map[string]string{"app": "network-peer-0"},
   327  		nil,
   328  		v1alpha1.OneMode,
   329  		v1alpha1.AllMode,
   330  		v1alpha1.To,
   331  		testDelayDuration,
   332  	)
   333  	err = cli.Create(ctx, networkPartitionWithoutTarget.DeepCopy())
   334  	framework.ExpectNoError(err, "create network chaos error")
   335  
   336  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   337  		result = probeNetworkCondition(c, networkPeers, ports, false)
   338  		if len(result[networkConditionBlocked]) != 3 || len(result[networkConditionSlow]) != 0 {
   339  			return false, nil
   340  		}
   341  		return true, nil
   342  	})
   343  	// The expected behavior is to block only 0 -> 1, 0 -> 2 and 0 -> 3
   344  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}, {0, 2}, {0, 3}})
   345  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   346  
   347  	By("recover")
   348  	err = cli.Delete(ctx, networkPartitionWithoutTarget.DeepCopy())
   349  	framework.ExpectNoError(err, "delete network chaos error")
   350  
   351  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   352  		klog.Info("retry probeNetworkCondition")
   353  		result = probeNetworkCondition(c, networkPeers, ports, false)
   354  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   355  			return false, nil
   356  		}
   357  		return true, nil
   358  	})
   359  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   360  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   361  
   362  	By("block from peer-0 from all")
   363  	networkPartitionWithoutTarget = makeNetworkPartitionChaos(
   364  		ns, "network-chaos-without-target",
   365  		map[string]string{"app": "network-peer-0"},
   366  		nil,
   367  		v1alpha1.OneMode,
   368  		v1alpha1.AllMode,
   369  		v1alpha1.From,
   370  		testDelayDuration,
   371  	)
   372  	err = cli.Create(ctx, networkPartitionWithoutTarget.DeepCopy())
   373  	framework.ExpectNoError(err, "create network chaos error")
   374  
   375  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   376  		result = probeNetworkCondition(c, networkPeers, ports, true)
   377  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   378  			return false, nil
   379  		}
   380  		return true, nil
   381  	})
   382  	// The expected behavior is to block only 0 -> 1, 0 -> 2 and 0 -> 3
   383  	// but the dropped packet will not throw an error
   384  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   385  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   386  
   387  	By("recover")
   388  	err = cli.Delete(ctx, networkPartitionWithoutTarget.DeepCopy())
   389  	framework.ExpectNoError(err, "delete network chaos error")
   390  
   391  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   392  		klog.Info("retry probeNetworkCondition")
   393  		result = probeNetworkCondition(c, networkPeers, ports, false)
   394  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   395  			return false, nil
   396  		}
   397  		return true, nil
   398  	})
   399  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   400  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   401  }
   402