basic.go

Documentation: github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos

     1  // Copyright 2021 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  // http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  
    16  package chaos
    17  
    18  import (
    19  	"context"
    20  	"fmt"
    21  	"net/http"
    22  	"strconv"
    23  	"strings"
    24  	"time"
    25  
    26  	"github.com/onsi/ginkgo/v2"
    27  	"github.com/pkg/errors"
    28  	v1 "k8s.io/api/core/v1"
    29  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    30  	"k8s.io/apimachinery/pkg/labels"
    31  	"k8s.io/apimachinery/pkg/runtime"
    32  	"k8s.io/client-go/kubernetes"
    33  	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
    34  	restClient "k8s.io/client-go/rest"
    35  	"k8s.io/kubernetes/test/e2e/framework"
    36  	"k8s.io/pod-security-admission/api"
    37  	"sigs.k8s.io/controller-runtime/pkg/client"
    38  
    39  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    40  	dnschaostestcases "github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/dnschaos"
    41  	httpchaostestcases "github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/httpchaos"
    42  	iochaostestcases "github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/iochaos"
    43  	networkchaostestcases "github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/networkchaos"
    44  	podchaostestcases "github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/podchaos"
    45  	sidecartestcases "github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/sidecar"
    46  	stresstestcases "github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/stresschaos"
    47  	timechaostestcases "github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/timechaos"
    48  	e2econfig "github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/config"
    49  	"github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/e2econst"
    50  	"github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/util"
    51  	"github.com/chaos-mesh/chaos-mesh/e2e-test/pkg/fixture"
    52  	"github.com/chaos-mesh/chaos-mesh/pkg/log"
    53  	"github.com/chaos-mesh/chaos-mesh/pkg/portforward" // testcases
    54  )
    55  
    56  var _ = ginkgo.Describe("[Basic]", func() {
    57  	f := framework.NewDefaultFramework("chaos-mesh")
    58  	f.NamespacePodSecurityEnforceLevel = api.LevelPrivileged
    59  	var ns string
    60  	var fwCancel context.CancelFunc
    61  	var fw portforward.PortForward
    62  	var kubeCli kubernetes.Interface
    63  	var config *restClient.Config
    64  	var cli client.Client
    65  	c := http.Client{
    66  		Timeout: 10 * time.Second,
    67  	}
    68  
    69  	ginkgo.BeforeEach(func() {
    70  		ns = f.Namespace.Name
    71  		ctx, cancel := context.WithCancel(context.Background())
    72  		clientRawConfig, err := e2econfig.LoadClientRawConfig()
    73  		framework.ExpectNoError(err, "failed to load raw config")
    74  		logger, err := log.NewDefaultZapLogger()
    75  		framework.ExpectNoError(err, "failed to create logger")
    76  		fw, err = portforward.NewPortForwarder(ctx, e2econfig.NewSimpleRESTClientGetter(clientRawConfig), true, logger)
    77  		framework.ExpectNoError(err, "failed to create port forwarder")
    78  		fwCancel = cancel
    79  		kubeCli = f.ClientSet
    80  		config, err = framework.LoadConfig()
    81  		framework.ExpectNoError(err, "config error")
    82  		scheme := runtime.NewScheme()
    83  		_ = clientgoscheme.AddToScheme(scheme)
    84  		_ = v1alpha1.AddToScheme(scheme)
    85  		cli, err = client.New(config, client.Options{Scheme: scheme})
    86  		framework.ExpectNoError(err, "create client error")
    87  	})
    88  
    89  	ginkgo.AfterEach(func() {
    90  		if fwCancel != nil {
    91  			fwCancel()
    92  		}
    93  	})
    94  
    95  	ginkgo.Context("[PodChaos]", func() {
    96  		ginkgo.Context("[PodFailure]", func() {
    97  			ginkgo.It("[Schedule]", func() {
    98  				podchaostestcases.TestcasePodFailureOnceThenDelete(ns, kubeCli, cli)
    99  			})
   100  			ginkgo.It("[Pause]", func() {
   101  				podchaostestcases.TestcasePodFailurePauseThenUnPause(ns, kubeCli, cli)
   102  			})
   103  		})
   104  		ginkgo.Context("[PodKill]", func() {
   105  			ginkgo.It("[Schedule]", func() {
   106  				podchaostestcases.TestcasePodKillOnceThenDelete(ns, kubeCli, cli)
   107  			})
   108  			ginkgo.It("[Pause]", func() {
   109  				podchaostestcases.TestcasePodKillPauseThenUnPause(ns, kubeCli, cli)
   110  			})
   111  		})
   112  		ginkgo.Context("[ContainerKill]", func() {
   113  			ginkgo.It("[Schedule]", func() {
   114  				podchaostestcases.TestcaseContainerKillOnceThenDelete(ns, kubeCli, cli)
   115  			})
   116  			ginkgo.It("[Pause]", func() {
   117  				podchaostestcases.TestcaseContainerKillPauseThenUnPause(ns, kubeCli, cli)
   118  			})
   119  		})
   120  	})
   121  
   122  	// time chaos case in [TimeChaos] context
   123  	ginkgo.Context("[TimeChaos]", func() {
   124  
   125  		var err error
   126  		var port uint16
   127  		var pfCancel context.CancelFunc
   128  
   129  		ginkgo.JustBeforeEach(func() {
   130  			svc := fixture.NewE2EService("timer", ns)
   131  			_, err = kubeCli.CoreV1().Services(ns).Create(context.TODO(), svc, metav1.CreateOptions{})
   132  			framework.ExpectNoError(err, "create service error")
   133  			nd := fixture.NewTimerDeployment("timer", ns)
   134  			_, err = kubeCli.AppsV1().Deployments(ns).Create(context.TODO(), nd, metav1.CreateOptions{})
   135  			framework.ExpectNoError(err, "create timer deployment error")
   136  			err = util.WaitDeploymentReady("timer", ns, kubeCli)
   137  			framework.ExpectNoError(err, "wait timer deployment ready error")
   138  			_, port, pfCancel, err = portforward.ForwardOnePort(fw, ns, "svc/timer", 8080)
   139  			framework.ExpectNoError(err, "create helper port-forward failed")
   140  		})
   141  
   142  		ginkgo.JustAfterEach(func() {
   143  			if pfCancel != nil {
   144  				pfCancel()
   145  			}
   146  		})
   147  
   148  		// time skew chaos case in [TimeSkew] context
   149  		ginkgo.Context("[TimeSkew]", func() {
   150  
   151  			ginkgo.It("[Schedule]", func() {
   152  				timechaostestcases.TestcaseTimeSkewOnceThenRecover(ns, cli, c, port)
   153  			})
   154  
   155  			ginkgo.It("[Pause]", func() {
   156  				timechaostestcases.TestcaseTimeSkewPauseThenUnpause(ns, cli, c, port)
   157  			})
   158  
   159  			ginkgo.It("[Child Process]", func() {
   160  				timechaostestcases.TestcaseTimeSkewPauseThenUnpause(ns, cli, c, port)
   161  			})
   162  		})
   163  	})
   164  
   165  	// io chaos case in [IOChaos] context
   166  	ginkgo.Context("[IOChaos]", func() {
   167  
   168  		var (
   169  			err      error
   170  			port     uint16
   171  			pfCancel context.CancelFunc
   172  		)
   173  
   174  		ginkgo.JustBeforeEach(func() {
   175  			svc := fixture.NewE2EService("io", ns)
   176  			_, err = kubeCli.CoreV1().Services(ns).Create(context.TODO(), svc, metav1.CreateOptions{})
   177  			framework.ExpectNoError(err, "create service error")
   178  			nd := fixture.NewIOTestDeployment("io-test", ns)
   179  			_, err = kubeCli.AppsV1().Deployments(ns).Create(context.TODO(), nd, metav1.CreateOptions{})
   180  			framework.ExpectNoError(err, "create io-test deployment error")
   181  			err = util.WaitDeploymentReady("io-test", ns, kubeCli)
   182  			framework.ExpectNoError(err, "wait io-test deployment ready error")
   183  			_, port, pfCancel, err = portforward.ForwardOnePort(fw, ns, "svc/io", 8080)
   184  			framework.ExpectNoError(err, "create helper io port port-forward failed")
   185  		})
   186  
   187  		ginkgo.JustAfterEach(func() {
   188  			if pfCancel != nil {
   189  				pfCancel()
   190  			}
   191  		})
   192  
   193  		// io chaos case in [IODelay] context
   194  		ginkgo.Context("[IODelay]", func() {
   195  
   196  			ginkgo.It("[Schedule]", func() {
   197  				iochaostestcases.TestcaseIODelayDurationForATimeThenRecover(ns, cli, c, port)
   198  			})
   199  
   200  			ginkgo.It("[Pause]", func() {
   201  				iochaostestcases.TestcaseIODelayDurationForATimePauseAndUnPause(ns, cli, c, port)
   202  			})
   203  			ginkgo.It("[SpecifyContainer]", func() {
   204  				iochaostestcases.TestcaseIODelayWithSpecifiedContainer(ns, cli, c, port)
   205  			})
   206  			ginkgo.It("[WrongSpec]", func() {
   207  				iochaostestcases.TestcaseIODelayWithWrongSpec(ns, cli, c, port)
   208  			})
   209  		})
   210  
   211  		// io chaos case in [IOError] context
   212  		ginkgo.Context("[IOErrno]", func() {
   213  
   214  			ginkgo.It("[Schedule]", func() {
   215  				iochaostestcases.TestcaseIOErrorDurationForATimeThenRecover(ns, cli, c, port)
   216  			})
   217  			ginkgo.It("[Pause]", func() {
   218  				iochaostestcases.TestcaseIOErrorDurationForATimePauseAndUnPause(ns, cli, c, port)
   219  			})
   220  			ginkgo.It("[SpecifyContainer]", func() {
   221  				iochaostestcases.TestcaseIOErrorWithSpecifiedContainer(ns, cli, c, port)
   222  			})
   223  		})
   224  
   225  		// io mistake case in [IOMistake] context
   226  		ginkgo.Context("[IOMistake]", func() {
   227  
   228  			ginkgo.It("[Schedule]", func() {
   229  				iochaostestcases.TestcaseIOMistakeDurationForATimeThenRecover(ns, cli, c, port)
   230  			})
   231  			ginkgo.It("[Pause]", func() {
   232  				iochaostestcases.TestcaseIOMistakeDurationForATimePauseAndUnPause(ns, cli, c, port)
   233  			})
   234  			ginkgo.It("[SpecifyContainer]", func() {
   235  				iochaostestcases.TestcaseIOMistakeWithSpecifiedContainer(ns, cli, c, port)
   236  			})
   237  		})
   238  	})
   239  
   240  	//http chaos case in [HTTPChaos] context
   241  	ginkgo.Context("[HTTPChaos]", func() {
   242  		var (
   243  			err      error
   244  			port     uint16
   245  			tlsPort  uint16
   246  			pfCancel context.CancelFunc
   247  			client   httpchaostestcases.HTTPE2EClient
   248  		)
   249  
   250  		ginkgo.JustBeforeEach(func() {
   251  			svc := fixture.NewE2EService("http", ns)
   252  			svc, err = kubeCli.CoreV1().Services(ns).Create(context.TODO(), svc, metav1.CreateOptions{})
   253  			framework.ExpectNoError(err, "create service error")
   254  			for _, servicePort := range svc.Spec.Ports {
   255  				if servicePort.Name == "http" {
   256  					port = uint16(servicePort.NodePort)
   257  					continue
   258  				}
   259  				if servicePort.Name == "https" {
   260  					tlsPort = uint16(servicePort.NodePort)
   261  					continue
   262  				}
   263  			}
   264  			nd := fixture.NewHTTPTestDeployment("http-test", ns)
   265  			_, err = kubeCli.AppsV1().Deployments(ns).Create(context.TODO(), nd, metav1.CreateOptions{})
   266  			framework.ExpectNoError(err, "create http-test deployment error")
   267  			err = util.WaitDeploymentReady("http-test", ns, kubeCli)
   268  			framework.ExpectNoError(err, "wait http-test deployment ready error")
   269  			podlist, err := kubeCli.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{})
   270  			framework.ExpectNoError(err, "find pod list error")
   271  			for _, item := range podlist.Items {
   272  				if strings.Contains(item.Name, "http-test") {
   273  					framework.Logf("get http-test-pod %v", item)
   274  					client.IP = item.Status.HostIP
   275  					break
   276  				}
   277  			}
   278  			client.C = &c
   279  		})
   280  
   281  		ginkgo.JustAfterEach(func() {
   282  			if pfCancel != nil {
   283  				pfCancel()
   284  			}
   285  		})
   286  
   287  		// http chaos case in [HTTPDelay] context
   288  		ginkgo.Context("[HTTPDelay]", func() {
   289  			ginkgo.It("[Schedule]", func() {
   290  				httpchaostestcases.TestcaseHttpDelayDurationForATimeThenRecover(ns, cli, client, port)
   291  			})
   292  			ginkgo.It("[Pause]", func() {
   293  				httpchaostestcases.TestcaseHttpDelayDurationForATimePauseAndUnPause(ns, cli, client, port)
   294  			})
   295  		})
   296  
   297  		// http chaos case in [HTTPAbort] context
   298  		ginkgo.Context("[HTTPAbort]", func() {
   299  			ginkgo.It("[Schedule]", func() {
   300  				httpchaostestcases.TestcaseHttpAbortThenRecover(ns, cli, client, port)
   301  			})
   302  			ginkgo.It("[Pause]", func() {
   303  				httpchaostestcases.TestcaseHttpAbortPauseAndUnPause(ns, cli, client, port)
   304  			})
   305  		})
   306  
   307  		// http chaos case in [HTTPReplace] context
   308  		ginkgo.Context("[HTTPReplace]", func() {
   309  			ginkgo.It("[Schedule]", func() {
   310  				httpchaostestcases.TestcaseHttpReplaceThenRecover(ns, cli, client, port)
   311  			})
   312  			ginkgo.It("[Pause]", func() {
   313  				httpchaostestcases.TestcaseHttpReplacePauseAndUnPause(ns, cli, client, port)
   314  			})
   315  		})
   316  
   317  		// http chaos case in [HTTPReplaceBody] context
   318  		ginkgo.Context("[HTTPReplaceBody]", func() {
   319  			ginkgo.It("[Schedule]", func() {
   320  				httpchaostestcases.TestcaseHttpReplaceBodyThenRecover(ns, cli, client, port)
   321  			})
   322  			ginkgo.It("[Pause]", func() {
   323  				httpchaostestcases.TestcaseHttpReplaceBodyPauseAndUnPause(ns, cli, client, port)
   324  			})
   325  		})
   326  
   327  		// http chaos case in [HTTPPatch] context
   328  		ginkgo.Context("[HTTPPatch]", func() {
   329  			ginkgo.It("[Schedule]", func() {
   330  				httpchaostestcases.TestcaseHttpPatchThenRecover(ns, cli, client, port)
   331  			})
   332  			ginkgo.It("[Pause]", func() {
   333  				httpchaostestcases.TestcaseHttpPatchPauseAndUnPause(ns, cli, client, port)
   334  			})
   335  		})
   336  
   337  		// http chaos case in [HTTPPatch] context
   338  		ginkgo.Context("[HTTP TLS]", func() {
   339  			ginkgo.It("[Schedule]", func() {
   340  				httpchaostestcases.TestcaseHttpTLSThenRecover(ns, kubeCli, cli, client, port, tlsPort)
   341  			})
   342  		})
   343  	})
   344  
   345  	ginkgo.Context("[Sidecar Config]", func() {
   346  		var (
   347  			cmName      string
   348  			cmNamespace string
   349  		)
   350  
   351  		// delete the created config map in each test case
   352  		ginkgo.JustAfterEach(func() {
   353  			kubeCli.CoreV1().ConfigMaps(cmNamespace).Delete(context.TODO(), cmName, metav1.DeleteOptions{})
   354  		})
   355  
   356  		ginkgo.Context("[Template Config]", func() {
   357  
   358  			ginkgo.It("[InValid ConfigMap key]", func() {
   359  				cmName = "incorrect-key-name"
   360  				cmNamespace = e2econst.ChaosMeshNamespace
   361  				sidecartestcases.TestcaseInvalidConfigMapKey(ns, cmNamespace, cmName, kubeCli, cli)
   362  			})
   363  
   364  			ginkgo.It("[InValid Configuration]", func() {
   365  				cmName = "incorrect-configuration"
   366  				cmNamespace = e2econst.ChaosMeshNamespace
   367  				sidecartestcases.TestcaseInvalidConfiguration(ns, cmNamespace, cmName, kubeCli, cli)
   368  			})
   369  		})
   370  
   371  		ginkgo.Context("[Injection Config]", func() {
   372  			ginkgo.It("[No Template]", func() {
   373  				cmName = "no-template-name"
   374  				cmNamespace = e2econst.ChaosMeshNamespace
   375  				sidecartestcases.TestcaseNoTemplate(ns, cmNamespace, cmName, kubeCli, cli)
   376  			})
   377  
   378  			ginkgo.It("[No Template Args]", func() {
   379  				cmName = "no-template-args"
   380  				cmNamespace = e2econst.ChaosMeshNamespace
   381  				sidecartestcases.TestcaseNoTemplateArgs(ns, cmNamespace, cmName, kubeCli, cli)
   382  			})
   383  		})
   384  	})
   385  
   386  	ginkgo.Context("[NetworkChaos]", func() {
   387  		var err error
   388  
   389  		var networkPeers []*v1.Pod
   390  		var ports []uint16
   391  		var pfCancels []context.CancelFunc
   392  
   393  		ginkgo.JustBeforeEach(func() {
   394  			ports = []uint16{}
   395  			networkPeers = []*v1.Pod{}
   396  			for index := 0; index < 4; index++ {
   397  				name := fmt.Sprintf("network-peer-%d", index)
   398  
   399  				svc := fixture.NewE2EService(name, ns)
   400  				_, err = kubeCli.CoreV1().Services(ns).Create(context.TODO(), svc, metav1.CreateOptions{})
   401  				framework.ExpectNoError(err, "create service error")
   402  				nd := fixture.NewNetworkTestDeployment(name, ns, map[string]string{"partition": strconv.Itoa(index % 2)})
   403  				_, err = kubeCli.AppsV1().Deployments(ns).Create(context.TODO(), nd, metav1.CreateOptions{})
   404  				framework.ExpectNoError(err, "create network-peer deployment error")
   405  				err = util.WaitDeploymentReady(name, ns, kubeCli)
   406  				framework.ExpectNoError(err, "wait network-peer deployment ready error")
   407  
   408  				pod, err := getPod(kubeCli, ns, name)
   409  				framework.ExpectNoError(err, "select network-peer pod error")
   410  				networkPeers = append(networkPeers, pod)
   411  
   412  				_, port, pfCancel, err := portforward.ForwardOnePort(fw, ns, "svc/"+svc.Name, 8080)
   413  				ports = append(ports, port)
   414  				pfCancels = append(pfCancels, pfCancel)
   415  				framework.ExpectNoError(err, "create helper io port port-forward failed")
   416  			}
   417  		})
   418  
   419  		ginkgo.Context("[ForbidHostNetwork]", func() {
   420  			ginkgo.It("[Schedule]", func() {
   421  				networkchaostestcases.TestcaseForbidHostNetwork(ns, kubeCli, cli)
   422  			})
   423  		})
   424  
   425  		ginkgo.Context("[NetworkPartition]", func() {
   426  			ginkgo.It("[Schedule]", func() {
   427  				networkchaostestcases.TestcaseNetworkPartition(ns, cli, networkPeers, ports, c)
   428  			})
   429  		})
   430  
   431  		ginkgo.Context("[Netem]", func() {
   432  			ginkgo.It("[Schedule]", func() {
   433  				networkchaostestcases.TestcaseNetworkDelay(ns, cli, networkPeers, ports, c)
   434  			})
   435  			ginkgo.It("[PeersCrossoverWithDirectionBoth]", func() {
   436  				networkchaostestcases.TestcasePeersCrossover(ns, cli, networkPeers, ports, c)
   437  			})
   438  		})
   439  
   440  		ginkgo.JustAfterEach(func() {
   441  			for _, cancel := range pfCancels {
   442  				cancel()
   443  			}
   444  		})
   445  	})
   446  	// DNS chaos case in [DNSChaos] context
   447  	ginkgo.Context("[DNSChaos]", func() {
   448  		var err error
   449  		var port uint16
   450  
   451  		ginkgo.JustBeforeEach(func() {
   452  			name := "network-peer"
   453  
   454  			svc := fixture.NewE2EService(name, ns)
   455  			_, err = kubeCli.CoreV1().Services(ns).Create(context.TODO(), svc, metav1.CreateOptions{})
   456  			framework.ExpectNoError(err, "create service error")
   457  			nd := fixture.NewNetworkTestDeployment(name, ns, map[string]string{"partition": "0"})
   458  			_, err = kubeCli.AppsV1().Deployments(ns).Create(context.TODO(), nd, metav1.CreateOptions{})
   459  			framework.ExpectNoError(err, "create network-peer deployment error")
   460  			err = util.WaitDeploymentReady(name, ns, kubeCli)
   461  			framework.ExpectNoError(err, "wait network-peer deployment ready error")
   462  
   463  			_, err = getPod(kubeCli, ns, name)
   464  			framework.ExpectNoError(err, "select network-peer pod error")
   465  
   466  			_, port, _, err = portforward.ForwardOnePort(fw, ns, "svc/"+svc.Name, 8080)
   467  			framework.ExpectNoError(err, "create helper io port port-forward failed")
   468  		})
   469  		ginkgo.It("[RANDOM]", func() {
   470  			dnschaostestcases.TestcaseDNSRandom(ns, cli, port, c)
   471  		})
   472  
   473  		ginkgo.It("[ERROR]", func() {
   474  			dnschaostestcases.TestcaseDNSError(ns, cli, port, c)
   475  		})
   476  	})
   477  	// DNS chaos case in [StressChaos] context
   478  	ginkgo.Context("[StressChaos]", func() {
   479  		var err error
   480  
   481  		var ports []uint16
   482  		var stressPeers []*v1.Pod
   483  		var pfCancels []context.CancelFunc
   484  
   485  		ginkgo.JustBeforeEach(func() {
   486  			ports = []uint16{}
   487  			stressPeers = []*v1.Pod{}
   488  			for index := 0; index < 2; index++ {
   489  				name := fmt.Sprintf("stress-peer-%d", index)
   490  
   491  				svc := fixture.NewE2EService(name, ns)
   492  				_, err = kubeCli.CoreV1().Services(ns).Create(context.TODO(), svc, metav1.CreateOptions{})
   493  				framework.ExpectNoError(err, "create service error")
   494  				nd := fixture.NewStressTestDeployment(name, ns, map[string]string{"partition": strconv.Itoa(index % 2)})
   495  				_, err = kubeCli.AppsV1().Deployments(ns).Create(context.TODO(), nd, metav1.CreateOptions{})
   496  				framework.ExpectNoError(err, "create network-peer deployment error")
   497  				err = util.WaitDeploymentReady(name, ns, kubeCli)
   498  				framework.ExpectNoError(err, "wait network-peer deployment ready error")
   499  
   500  				pod, err := getPod(kubeCli, ns, name)
   501  				framework.ExpectNoError(err, "select network-peer pod error")
   502  				stressPeers = append(stressPeers, pod)
   503  
   504  				_, port, pfCancel, err := portforward.ForwardOnePort(fw, ns, "svc/"+svc.Name, 8080)
   505  				ports = append(ports, port)
   506  				pfCancels = append(pfCancels, pfCancel)
   507  				framework.ExpectNoError(err, "create helper io port port-forward failed")
   508  			}
   509  		})
   510  
   511  		ginkgo.It("[CPU]", func() {
   512  			stresstestcases.TestcaseCPUStressInjectionOnceThenRecover(ns, cli, stressPeers, ports, c)
   513  		})
   514  
   515  		// TODO: unstable test
   516  		ginkgo.It("[Memory]", func() {
   517  			stresstestcases.TestcaseMemoryStressInjectionOnceThenRecover(ns, cli, stressPeers, ports, c)
   518  		})
   519  
   520  		ginkgo.JustAfterEach(func() {
   521  			for _, cancel := range pfCancels {
   522  				cancel()
   523  			}
   524  		})
   525  	})
   526  })
   527  
   528  func getPod(kubeCli kubernetes.Interface, ns string, appLabel string) (*v1.Pod, error) {
   529  	listOption := metav1.ListOptions{
   530  		LabelSelector: labels.SelectorFromSet(map[string]string{
   531  			"app": appLabel,
   532  		}).String(),
   533  	}
   534  
   535  	pods, err := kubeCli.CoreV1().Pods(ns).List(context.TODO(), listOption)
   536  	if err != nil {
   537  		return nil, err
   538  	}
   539  
   540  	if len(pods.Items) > 1 {
   541  		return nil, errors.New("select more than one pod")
   542  	}
   543  
   544  	if len(pods.Items) == 0 {
   545  		return nil, errors.New("cannot select any pod")
   546  	}
   547  
   548  	return &pods.Items[0], nil
   549  }
   550