network_partition.go

Documentation: github.com/chaos-mesh/chaos-mesh/test/e2e/chaos/networkchaos

     1  // Copyright 2020 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package networkchaos
    15  
    16  import (
    17  	"context"
    18  	"net/http"
    19  	"strings"
    20  	"time"
    21  
    22  	. "github.com/onsi/ginkgo"
    23  	corev1 "k8s.io/api/core/v1"
    24  	"k8s.io/apimachinery/pkg/types"
    25  	"k8s.io/apimachinery/pkg/util/wait"
    26  	"k8s.io/client-go/kubernetes"
    27  	"k8s.io/klog"
    28  	"k8s.io/kubernetes/test/e2e/framework"
    29  	"k8s.io/utils/pointer"
    30  	"sigs.k8s.io/controller-runtime/pkg/client"
    31  
    32  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    33  	"github.com/chaos-mesh/chaos-mesh/test/e2e/util"
    34  	"github.com/chaos-mesh/chaos-mesh/test/pkg/fixture"
    35  )
    36  
    37  // TestcaseForbidHostNetwork We do NOT allow that inject chaos on a pod which uses hostNetwork
    38  func TestcaseForbidHostNetwork(
    39  	ns string,
    40  	kubeCli kubernetes.Interface,
    41  	cli client.Client,
    42  ) {
    43  	ctx, cancel := context.WithCancel(context.Background())
    44  	defer cancel()
    45  
    46  	By("preparing experiment pods")
    47  	name := "network-peer-4"
    48  	nd := fixture.NewNetworkTestDeployment(name, ns, map[string]string{"partition": "0"})
    49  	nd.Spec.Template.Spec.HostNetwork = true
    50  	_, err := kubeCli.AppsV1().Deployments(ns).Create(nd)
    51  	framework.ExpectNoError(err, "create network-peer deployment error")
    52  	err = util.WaitDeploymentReady(name, ns, kubeCli)
    53  	framework.ExpectNoError(err, "wait network-peer deployment ready error")
    54  
    55  	By("create network partition chaos CRD objects")
    56  	networkPartition := makeNetworkPartitionChaos(
    57  		ns, "network-chaos-1",
    58  		map[string]string{"app": "network-peer-4"},
    59  		map[string]string{"app": "network-peer-1"},
    60  		v1alpha1.OnePodMode,
    61  		v1alpha1.OnePodMode,
    62  		v1alpha1.To,
    63  		pointer.StringPtr("9m"),
    64  		&v1alpha1.SchedulerSpec{
    65  			Cron: "@every 10m",
    66  		},
    67  	)
    68  
    69  	err = cli.Create(ctx, networkPartition.DeepCopy())
    70  	framework.ExpectNoError(err, "create network chaos error")
    71  
    72  	By("waiting for rejecting for network chaos with hostNetwork")
    73  	err = wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) {
    74  		err = cli.Get(ctx, types.NamespacedName{
    75  			Namespace: ns,
    76  			Name:      "network-chaos-1",
    77  		}, networkPartition)
    78  		if err != nil {
    79  			return false, err
    80  		}
    81  		experimentPhase := networkPartition.Status.ChaosStatus.Experiment.Phase
    82  		klog.Infof("current chaos phase: %s", experimentPhase)
    83  		if experimentPhase == v1alpha1.ExperimentPhaseFailed {
    84  			return true, nil
    85  		}
    86  		return false, nil
    87  	})
    88  
    89  	framework.ExpectNoError(err, "failed to waiting on ExperimentPhaseFailed state with chaos")
    90  	framework.ExpectEqual(networkPartition.Status.ChaosStatus.Experiment.Phase, v1alpha1.ExperimentPhaseFailed)
    91  	framework.ExpectEqual(strings.Contains(networkPartition.Status.ChaosStatus.FailedMessage, "it's dangerous to inject network chaos on a pod"), true)
    92  }
    93  
    94  func TestcaseNetworkPartition(
    95  	ns string,
    96  	cli client.Client,
    97  	networkPeers []*corev1.Pod,
    98  	ports []uint16,
    99  	c http.Client,
   100  ) {
   101  	ctx, cancel := context.WithCancel(context.Background())
   102  	defer cancel()
   103  	By("prepare experiment playground")
   104  	for index := range networkPeers {
   105  		err := util.WaitE2EHelperReady(c, ports[index])
   106  
   107  		framework.ExpectNoError(err, "wait e2e helper ready error")
   108  	}
   109  
   110  	var result map[string][][]int
   111  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   112  		result = probeNetworkCondition(c, networkPeers, ports, false)
   113  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   114  			return false, nil
   115  		}
   116  		return true, nil
   117  	})
   118  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   119  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   120  
   121  	var (
   122  		testDelayDuration      = pointer.StringPtr("9m")
   123  		testDelaySchedulerSpec = &v1alpha1.SchedulerSpec{
   124  			Cron: "@every 10m",
   125  		}
   126  	)
   127  
   128  	baseNetworkPartition := makeNetworkPartitionChaos(
   129  		ns, "network-chaos-1",
   130  		map[string]string{"app": "network-peer-0"},
   131  		map[string]string{"app": "network-peer-1"},
   132  		v1alpha1.OnePodMode,
   133  		v1alpha1.OnePodMode,
   134  		v1alpha1.To,
   135  		testDelayDuration,
   136  		testDelaySchedulerSpec,
   137  	)
   138  
   139  	By("block from peer-0 to peer-1")
   140  	err := cli.Create(ctx, baseNetworkPartition.DeepCopy())
   141  	framework.ExpectNoError(err, "create network chaos error")
   142  
   143  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   144  		result = probeNetworkCondition(c, networkPeers, ports, false)
   145  		if len(result[networkConditionBlocked]) != 1 || len(result[networkConditionSlow]) != 0 {
   146  			return false, nil
   147  		}
   148  		return true, nil
   149  	})
   150  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}})
   151  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   152  
   153  	By("recover")
   154  	err = cli.Delete(ctx, baseNetworkPartition.DeepCopy())
   155  	framework.ExpectNoError(err, "delete network chaos error")
   156  
   157  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   158  		result = probeNetworkCondition(c, networkPeers, ports, false)
   159  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   160  			return false, nil
   161  		}
   162  		return true, nil
   163  	})
   164  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   165  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   166  
   167  	By("block both from peer-0 to peer-1 and from peer-1 to peer-0")
   168  	bothDirectionNetworkPartition := makeNetworkPartitionChaos(
   169  		ns, "network-chaos-1",
   170  		map[string]string{"app": "network-peer-0"},
   171  		map[string]string{"app": "network-peer-1"},
   172  		v1alpha1.OnePodMode,
   173  		v1alpha1.OnePodMode,
   174  		v1alpha1.Both,
   175  		testDelayDuration,
   176  		testDelaySchedulerSpec,
   177  	)
   178  	err = cli.Create(ctx, bothDirectionNetworkPartition.DeepCopy())
   179  	framework.ExpectNoError(err, "create network chaos error")
   180  
   181  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   182  		result = probeNetworkCondition(c, networkPeers, ports, false)
   183  		if len(result[networkConditionBlocked]) != 2 || len(result[networkConditionSlow]) != 0 {
   184  			return false, nil
   185  		}
   186  		return true, nil
   187  	})
   188  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}, {1, 0}})
   189  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   190  
   191  	By("recover")
   192  	err = cli.Delete(ctx, bothDirectionNetworkPartition.DeepCopy())
   193  	framework.ExpectNoError(err, "delete network chaos error")
   194  
   195  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   196  		result = probeNetworkCondition(c, networkPeers, ports, false)
   197  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   198  			return false, nil
   199  		}
   200  		return true, nil
   201  	})
   202  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   203  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   204  
   205  	By("block from peer-1 to peer-0")
   206  	fromDirectionNetworkPartition := makeNetworkPartitionChaos(
   207  		ns, "network-chaos-1",
   208  		map[string]string{"app": "network-peer-0"},
   209  		map[string]string{"app": "network-peer-1"},
   210  		v1alpha1.OnePodMode,
   211  		v1alpha1.OnePodMode,
   212  		v1alpha1.From,
   213  		testDelayDuration,
   214  		testDelaySchedulerSpec,
   215  	)
   216  
   217  	err = cli.Create(ctx, fromDirectionNetworkPartition.DeepCopy())
   218  	framework.ExpectNoError(err, "create network chaos error")
   219  
   220  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   221  		result = probeNetworkCondition(c, networkPeers, ports, false)
   222  		if len(result[networkConditionBlocked]) != 1 || len(result[networkConditionSlow]) != 0 {
   223  			return false, nil
   224  		}
   225  		return true, nil
   226  	})
   227  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{1, 0}})
   228  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   229  
   230  	By("recover")
   231  	err = cli.Delete(ctx, fromDirectionNetworkPartition.DeepCopy())
   232  	framework.ExpectNoError(err, "delete network chaos error")
   233  
   234  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   235  		result = probeNetworkCondition(c, networkPeers, ports, false)
   236  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   237  			return false, nil
   238  		}
   239  		return true, nil
   240  	})
   241  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   242  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   243  
   244  	By("network partition 1")
   245  
   246  	bothDirectionWithPartitionNetworkPartition := makeNetworkPartitionChaos(
   247  		ns, "network-chaos-1",
   248  		map[string]string{"app": "network-peer-0"},
   249  		map[string]string{"partition": "1"},
   250  		v1alpha1.OnePodMode,
   251  		v1alpha1.AllPodMode,
   252  		v1alpha1.Both,
   253  		testDelayDuration,
   254  		testDelaySchedulerSpec,
   255  	)
   256  	err = cli.Create(ctx, bothDirectionWithPartitionNetworkPartition.DeepCopy())
   257  	framework.ExpectNoError(err, "create network chaos error")
   258  
   259  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   260  		result = probeNetworkCondition(c, networkPeers, ports, false)
   261  		if len(result[networkConditionBlocked]) != 4 || len(result[networkConditionSlow]) != 0 {
   262  			return false, nil
   263  		}
   264  		return true, nil
   265  	})
   266  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}, {1, 0}, {0, 3}, {3, 0}})
   267  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   268  
   269  	By("recover")
   270  	err = cli.Delete(ctx, bothDirectionWithPartitionNetworkPartition.DeepCopy())
   271  	framework.ExpectNoError(err, "delete network chaos error")
   272  
   273  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   274  		result = probeNetworkCondition(c, networkPeers, ports, false)
   275  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   276  			return false, nil
   277  		}
   278  		return true, nil
   279  	})
   280  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   281  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   282  
   283  	By("multiple network partition chaos on peer-0")
   284  	anotherNetworkPartition := makeNetworkPartitionChaos(
   285  		ns, "network-chaos-2",
   286  		map[string]string{"app": "network-peer-0"},
   287  		map[string]string{"partition": "0"},
   288  		v1alpha1.OnePodMode,
   289  		v1alpha1.AllPodMode,
   290  		v1alpha1.To,
   291  		testDelayDuration,
   292  		testDelaySchedulerSpec,
   293  	)
   294  	err = cli.Create(ctx, bothDirectionWithPartitionNetworkPartition.DeepCopy())
   295  	framework.ExpectNoError(err, "create network chaos error")
   296  	err = cli.Create(ctx, anotherNetworkPartition.DeepCopy())
   297  	framework.ExpectNoError(err, "create network chaos error")
   298  
   299  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   300  		result = probeNetworkCondition(c, networkPeers, ports, false)
   301  		if len(result[networkConditionBlocked]) != 5 || len(result[networkConditionSlow]) != 0 {
   302  			return false, nil
   303  		}
   304  		return true, nil
   305  	})
   306  	framework.ExpectEqual(result[networkConditionBlocked], [][]int{{0, 1}, {1, 0}, {0, 2}, {0, 3}, {3, 0}})
   307  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   308  
   309  	By("recover")
   310  	err = cli.Delete(ctx, bothDirectionWithPartitionNetworkPartition.DeepCopy())
   311  	framework.ExpectNoError(err, "delete network chaos error")
   312  	err = cli.Delete(ctx, anotherNetworkPartition.DeepCopy())
   313  	framework.ExpectNoError(err, "delete network chaos error")
   314  
   315  	wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
   316  		klog.Info("retry probeNetworkCondition")
   317  		result = probeNetworkCondition(c, networkPeers, ports, false)
   318  		if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
   319  			return false, nil
   320  		}
   321  		return true, nil
   322  	})
   323  	framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
   324  	framework.ExpectEqual(len(result[networkConditionSlow]), 0)
   325  }
   326