http_replace.go

Documentation: github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/httpchaos

     1  // Copyright 2021 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  // http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  
    16  package httpchaos
    17  
    18  import (
    19  	"context"
    20  	"io"
    21  	"time"
    22  
    23  	. "github.com/onsi/ginkgo/v2"
    24  	corev1 "k8s.io/api/core/v1"
    25  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    26  	"k8s.io/apimachinery/pkg/types"
    27  	"k8s.io/apimachinery/pkg/util/wait"
    28  	"k8s.io/klog/v2"
    29  	"k8s.io/kubernetes/test/e2e/framework"
    30  	"sigs.k8s.io/controller-runtime/pkg/client"
    31  
    32  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    33  	"github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/util"
    34  )
    35  
    36  func TestcaseHttpReplaceThenRecover(
    37  	ns string,
    38  	cli client.Client,
    39  	c HTTPE2EClient,
    40  	port uint16,
    41  ) {
    42  	ctx, cancel := context.WithCancel(context.Background())
    43  	defer cancel()
    44  
    45  	By("waiting on e2e helper ready")
    46  	err := util.WaitHTTPE2EHelperReady(*c.C, c.IP, port)
    47  	framework.ExpectNoError(err, "wait e2e helper ready error")
    48  
    49  	secret := "Bar"
    50  
    51  	By("waiting for assertion normal behaviour")
    52  	err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
    53  		resp, err := getPodHttp(c, port, secret, "")
    54  		if err != nil {
    55  			return false, err
    56  		}
    57  		defer resp.Body.Close()
    58  
    59  		s := resp.Header.Get(SECRET)
    60  		b, err := io.ReadAll(resp.Body)
    61  		if err != nil {
    62  			return false, err
    63  		}
    64  
    65  		klog.Infof("Status(%d), Secret(%s), Body(%s)", resp.StatusCode, s, string(b))
    66  
    67  		if s == secret {
    68  			return true, nil
    69  		}
    70  		return false, nil
    71  	})
    72  	framework.ExpectNoError(err, "helper server doesn't work as expected")
    73  	By("deploy helper server successfully")
    74  
    75  	By("create http replace chaos CRD objects")
    76  	replaceSecret := "Foo!"
    77  
    78  	httpChaos := &v1alpha1.HTTPChaos{
    79  		ObjectMeta: metav1.ObjectMeta{
    80  			Name:      "http-chaos",
    81  			Namespace: ns,
    82  		},
    83  		Spec: v1alpha1.HTTPChaosSpec{
    84  			PodSelector: v1alpha1.PodSelector{
    85  				Selector: v1alpha1.PodSelectorSpec{
    86  					GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
    87  						Namespaces:     []string{ns},
    88  						LabelSelectors: map[string]string{"app": "http"},
    89  					},
    90  				},
    91  				Mode: v1alpha1.OneMode,
    92  			},
    93  			Port:   8080,
    94  			Target: "Request",
    95  			PodHttpChaosActions: v1alpha1.PodHttpChaosActions{
    96  				Replace: &v1alpha1.PodHttpChaosReplaceActions{
    97  					Headers: map[string]string{
    98  						SECRET: replaceSecret,
    99  					},
   100  				},
   101  			},
   102  		},
   103  	}
   104  	err = cli.Create(ctx, httpChaos)
   105  	framework.ExpectNoError(err, "create http chaos error")
   106  
   107  	By("waiting for assertion HTTP replace")
   108  	err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
   109  		resp, err := getPodHttp(c, port, secret, "")
   110  		if err != nil {
   111  			return false, err
   112  		}
   113  		defer resp.Body.Close()
   114  
   115  		s := resp.Header.Get(SECRET)
   116  		b, err := io.ReadAll(resp.Body)
   117  		if err != nil {
   118  			return false, err
   119  		}
   120  
   121  		klog.Infof("Status(%d), Secret(%s), Body(%s)", resp.StatusCode, s, string(b))
   122  
   123  		if s == replaceSecret {
   124  			return true, nil
   125  		}
   126  		return false, nil
   127  	})
   128  	framework.ExpectNoError(err, "http chaos doesn't work as expected")
   129  	By("apply http chaos successfully")
   130  
   131  	By("delete chaos CRD objects")
   132  	// delete chaos CRD
   133  	err = cli.Delete(ctx, httpChaos)
   134  	framework.ExpectNoError(err, "failed to delete http chaos")
   135  
   136  	By("waiting for assertion recovering")
   137  	err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
   138  		resp, err := getPodHttp(c, port, secret, "")
   139  		if err != nil {
   140  			return false, err
   141  		}
   142  		defer resp.Body.Close()
   143  
   144  		s := resp.Header.Get(SECRET)
   145  		b, err := io.ReadAll(resp.Body)
   146  		if err != nil {
   147  			return false, err
   148  		}
   149  
   150  		klog.Infof("Status(%d), Secret(%s), Body(%s)", resp.StatusCode, s, string(b))
   151  
   152  		if s == secret {
   153  			return true, nil
   154  		}
   155  		return false, nil
   156  	})
   157  	framework.ExpectNoError(err, "fail to recover http chaos")
   158  }
   159  
   160  func TestcaseHttpReplacePauseAndUnPause(
   161  	ns string,
   162  	cli client.Client,
   163  	c HTTPE2EClient,
   164  	port uint16,
   165  ) {
   166  	ctx, cancel := context.WithCancel(context.Background())
   167  	defer cancel()
   168  
   169  	By("waiting on e2e helper ready")
   170  	err := util.WaitHTTPE2EHelperReady(*c.C, c.IP, port)
   171  	framework.ExpectNoError(err, "wait e2e helper ready error")
   172  
   173  	secret := "Bar"
   174  
   175  	By("waiting for assertion normal behaviour")
   176  	err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
   177  		resp, err := getPodHttp(c, port, secret, "")
   178  		if err != nil {
   179  			return false, err
   180  		}
   181  		defer resp.Body.Close()
   182  
   183  		s := resp.Header.Get(SECRET)
   184  		b, err := io.ReadAll(resp.Body)
   185  		if err != nil {
   186  			return false, err
   187  		}
   188  
   189  		klog.Infof("Status(%d), Secret(%s), Body(%s)", resp.StatusCode, s, string(b))
   190  
   191  		if s == secret {
   192  			return true, nil
   193  		}
   194  		return false, nil
   195  	})
   196  	framework.ExpectNoError(err, "helper server doesn't work as expected")
   197  	By("deploy helper server successfully")
   198  
   199  	By("create http replace chaos CRD objects")
   200  	replaceSecret := "Foo!"
   201  
   202  	httpChaos := &v1alpha1.HTTPChaos{
   203  		ObjectMeta: metav1.ObjectMeta{
   204  			Name:      "http-chaos",
   205  			Namespace: ns,
   206  		},
   207  		Spec: v1alpha1.HTTPChaosSpec{
   208  			PodSelector: v1alpha1.PodSelector{
   209  				Selector: v1alpha1.PodSelectorSpec{
   210  					GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
   211  						Namespaces:     []string{ns},
   212  						LabelSelectors: map[string]string{"app": "http"},
   213  					},
   214  				},
   215  				Mode: v1alpha1.OneMode,
   216  			},
   217  			Port:   8080,
   218  			Target: "Request",
   219  			PodHttpChaosActions: v1alpha1.PodHttpChaosActions{
   220  				Replace: &v1alpha1.PodHttpChaosReplaceActions{
   221  					Headers: map[string]string{
   222  						SECRET: replaceSecret,
   223  					},
   224  				},
   225  			},
   226  		},
   227  	}
   228  	err = cli.Create(ctx, httpChaos)
   229  	framework.ExpectNoError(err, "create http chaos error")
   230  
   231  	chaosKey := types.NamespacedName{
   232  		Namespace: ns,
   233  		Name:      "http-chaos",
   234  	}
   235  
   236  	By("waiting for assertion http chaos")
   237  	err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
   238  		chaos := &v1alpha1.HTTPChaos{}
   239  		err = cli.Get(ctx, chaosKey, chaos)
   240  		framework.ExpectNoError(err, "get http chaos error")
   241  
   242  		for _, c := range chaos.GetStatus().Conditions {
   243  			if c.Type == v1alpha1.ConditionAllInjected {
   244  				if c.Status != corev1.ConditionTrue {
   245  					return false, nil
   246  				}
   247  			} else if c.Type == v1alpha1.ConditionSelected {
   248  				if c.Status != corev1.ConditionTrue {
   249  					return false, nil
   250  				}
   251  			}
   252  		}
   253  
   254  		resp, err := getPodHttp(c, port, secret, "")
   255  		if err != nil {
   256  			return false, err
   257  		}
   258  		defer resp.Body.Close()
   259  
   260  		s := resp.Header.Get(SECRET)
   261  		b, err := io.ReadAll(resp.Body)
   262  		if err != nil {
   263  			return false, err
   264  		}
   265  
   266  		klog.Infof("Status(%d), Secret(%s), Body(%s)", resp.StatusCode, s, string(b))
   267  
   268  		if s == replaceSecret {
   269  			return true, nil
   270  		}
   271  		return false, nil
   272  	})
   273  	framework.ExpectNoError(err, "http chaos doesn't work as expected")
   274  
   275  	By("pause http replace chaos experiment")
   276  	// pause experiment
   277  	err = util.PauseChaos(ctx, cli, httpChaos)
   278  	framework.ExpectNoError(err, "pause chaos error")
   279  
   280  	By("waiting for assertion about pause")
   281  	err = wait.Poll(1*time.Second, 1*time.Minute, func() (done bool, err error) {
   282  		chaos := &v1alpha1.HTTPChaos{}
   283  		err = cli.Get(ctx, chaosKey, chaos)
   284  		framework.ExpectNoError(err, "get http chaos error")
   285  
   286  		for _, c := range chaos.GetStatus().Conditions {
   287  			if c.Type == v1alpha1.ConditionAllRecovered {
   288  				if c.Status != corev1.ConditionTrue {
   289  					return false, nil
   290  				}
   291  			} else if c.Type == v1alpha1.ConditionSelected {
   292  				if c.Status != corev1.ConditionTrue {
   293  					return false, nil
   294  				}
   295  			}
   296  		}
   297  
   298  		return true, err
   299  	})
   300  	framework.ExpectNoError(err, "check paused chaos failed")
   301  
   302  	// wait 1 min to check whether io replace still exists
   303  	err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
   304  		resp, err := getPodHttp(c, port, secret, "")
   305  		if err != nil {
   306  			return false, err
   307  		}
   308  		defer resp.Body.Close()
   309  
   310  		s := resp.Header.Get(SECRET)
   311  		b, err := io.ReadAll(resp.Body)
   312  		if err != nil {
   313  			return false, err
   314  		}
   315  
   316  		klog.Infof("Status(%d), Secret(%s), Body(%s)", resp.StatusCode, s, string(b))
   317  
   318  		if s == secret {
   319  			return true, nil
   320  		}
   321  		return false, nil
   322  	})
   323  	framework.ExpectNoError(err, "fail to recover http chaos")
   324  	By("resume http replace chaos experiment")
   325  	// resume experiment
   326  	err = util.UnPauseChaos(ctx, cli, httpChaos)
   327  	framework.ExpectNoError(err, "resume chaos error")
   328  
   329  	By("assert that http replace is effective again")
   330  	err = wait.Poll(1*time.Second, 1*time.Minute, func() (done bool, err error) {
   331  		chaos := &v1alpha1.HTTPChaos{}
   332  		err = cli.Get(ctx, chaosKey, chaos)
   333  		framework.ExpectNoError(err, "get http chaos error")
   334  
   335  		for _, c := range chaos.GetStatus().Conditions {
   336  			if c.Type == v1alpha1.ConditionAllInjected {
   337  				if c.Status != corev1.ConditionTrue {
   338  					return false, nil
   339  				}
   340  			} else if c.Type == v1alpha1.ConditionSelected {
   341  				if c.Status != corev1.ConditionTrue {
   342  					return false, nil
   343  				}
   344  			}
   345  		}
   346  
   347  		return true, err
   348  	})
   349  	framework.ExpectNoError(err, "check resumed chaos failed")
   350  
   351  	err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
   352  		resp, err := getPodHttp(c, port, secret, "")
   353  		if err != nil {
   354  			return false, err
   355  		}
   356  		defer resp.Body.Close()
   357  
   358  		s := resp.Header.Get(SECRET)
   359  		b, err := io.ReadAll(resp.Body)
   360  		if err != nil {
   361  			return false, err
   362  		}
   363  
   364  		klog.Infof("Status(%d), Secret(%s), Body(%s)", resp.StatusCode, s, string(b))
   365  
   366  		if s == replaceSecret {
   367  			return true, nil
   368  		}
   369  		return false, nil
   370  	})
   371  	framework.ExpectNoError(err, "HTTP chaos doesn't work as expected")
   372  
   373  	By("cleanup")
   374  	// cleanup
   375  	cli.Delete(ctx, httpChaos)
   376  }
   377  
   378  func TestcaseHttpReplaceBodyThenRecover(
   379  	ns string,
   380  	cli client.Client,
   381  	c HTTPE2EClient,
   382  	port uint16,
   383  ) {
   384  	ctx, cancel := context.WithCancel(context.Background())
   385  	defer cancel()
   386  
   387  	By("waiting on e2e helper ready")
   388  	err := util.WaitHTTPE2EHelperReady(*c.C, c.IP, port)
   389  	framework.ExpectNoError(err, "wait e2e helper ready error")
   390  
   391  	body := "Hello World"
   392  	secret := "Bar"
   393  
   394  	By("waiting for assertion normal behaviour")
   395  	err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
   396  		resp, err := getPodHttp(c, port, secret, body)
   397  		if err != nil {
   398  			return false, err
   399  		}
   400  		defer resp.Body.Close()
   401  
   402  		s := resp.Header.Get(SECRET)
   403  		b, err := io.ReadAll(resp.Body)
   404  		if err != nil {
   405  			return false, err
   406  		}
   407  
   408  		klog.Infof("Status(%d), Secret(%s), Body(%s)", resp.StatusCode, s, string(b))
   409  
   410  		if s == secret && string(b) == body {
   411  			return true, nil
   412  		}
   413  		return false, nil
   414  	})
   415  	framework.ExpectNoError(err, "helper server doesn't work as expected")
   416  	By("deploy helper server successfully")
   417  
   418  	By("create http replace chaos CRD objects")
   419  	replacebody := "Hello Chaos Mesh"
   420  	replaceSecret := "Foo!"
   421  
   422  	httpChaos := &v1alpha1.HTTPChaos{
   423  		ObjectMeta: metav1.ObjectMeta{
   424  			Name:      "http-chaos",
   425  			Namespace: ns,
   426  		},
   427  		Spec: v1alpha1.HTTPChaosSpec{
   428  			PodSelector: v1alpha1.PodSelector{
   429  				Selector: v1alpha1.PodSelectorSpec{
   430  					GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
   431  						Namespaces:     []string{ns},
   432  						LabelSelectors: map[string]string{"app": "http"},
   433  					},
   434  				},
   435  				Mode: v1alpha1.OneMode,
   436  			},
   437  			Port:   8080,
   438  			Target: "Request",
   439  			PodHttpChaosActions: v1alpha1.PodHttpChaosActions{
   440  				Replace: &v1alpha1.PodHttpChaosReplaceActions{
   441  					Headers: map[string]string{
   442  						SECRET: replaceSecret,
   443  					},
   444  					Body: []byte(replacebody),
   445  				},
   446  			},
   447  		},
   448  	}
   449  	err = cli.Create(ctx, httpChaos)
   450  	framework.ExpectNoError(err, "create http chaos error")
   451  
   452  	By("waiting for assertion HTTP replace")
   453  	err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
   454  		resp, err := getPodHttp(c, port, secret, body)
   455  		if err != nil {
   456  			return false, err
   457  		}
   458  		defer resp.Body.Close()
   459  
   460  		s := resp.Header.Get(SECRET)
   461  		b, err := io.ReadAll(resp.Body)
   462  		if err != nil {
   463  			return false, err
   464  		}
   465  
   466  		klog.Infof("Status(%d), Secret(%s), Body(%s)", resp.StatusCode, s, string(b))
   467  
   468  		if s == replaceSecret && string(b) == replacebody {
   469  			return true, nil
   470  		}
   471  		return false, nil
   472  	})
   473  	framework.ExpectNoError(err, "http chaos doesn't work as expected")
   474  	By("apply http chaos successfully")
   475  
   476  	By("delete chaos CRD objects")
   477  	// delete chaos CRD
   478  	err = cli.Delete(ctx, httpChaos)
   479  	framework.ExpectNoError(err, "failed to delete http chaos")
   480  
   481  	By("waiting for assertion recovering")
   482  	err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
   483  		resp, err := getPodHttp(c, port, secret, body)
   484  		if err != nil {
   485  			return false, err
   486  		}
   487  		defer resp.Body.Close()
   488  
   489  		s := resp.Header.Get(SECRET)
   490  		b, err := io.ReadAll(resp.Body)
   491  		if err != nil {
   492  			return false, err
   493  		}
   494  
   495  		klog.Infof("Status(%d), Secret(%s), Body(%s)", resp.StatusCode, s, string(b))
   496  
   497  		if s == secret && string(b) == body {
   498  			return true, nil
   499  		}
   500  		return false, nil
   501  	})
   502  	framework.ExpectNoError(err, "fail to recover http chaos")
   503  }
   504  
   505  func TestcaseHttpReplaceBodyPauseAndUnPause(
   506  	ns string,
   507  	cli client.Client,
   508  	c HTTPE2EClient,
   509  	port uint16,
   510  ) {
   511  	ctx, cancel := context.WithCancel(context.Background())
   512  	defer cancel()
   513  
   514  	By("waiting on e2e helper ready")
   515  	err := util.WaitHTTPE2EHelperReady(*c.C, c.IP, port)
   516  	framework.ExpectNoError(err, "wait e2e helper ready error")
   517  
   518  	body := "Hello World"
   519  	secret := "Bar"
   520  
   521  	By("waiting for assertion normal behaviour")
   522  	err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
   523  		resp, err := getPodHttp(c, port, secret, body)
   524  		if err != nil {
   525  			return false, err
   526  		}
   527  		defer resp.Body.Close()
   528  
   529  		s := resp.Header.Get(SECRET)
   530  		b, err := io.ReadAll(resp.Body)
   531  		if err != nil {
   532  			return false, err
   533  		}
   534  
   535  		klog.Infof("Status(%d), Secret(%s), Body(%s)", resp.StatusCode, s, string(b))
   536  
   537  		if s == secret && string(b) == body {
   538  			return true, nil
   539  		}
   540  		return false, nil
   541  	})
   542  	framework.ExpectNoError(err, "helper server doesn't work as expected")
   543  	By("deploy helper server successfully")
   544  
   545  	By("create http replace chaos CRD objects")
   546  	replacebody := "Hello Chaos Mesh"
   547  	replaceSecret := "Foo!"
   548  
   549  	httpChaos := &v1alpha1.HTTPChaos{
   550  		ObjectMeta: metav1.ObjectMeta{
   551  			Name:      "http-chaos",
   552  			Namespace: ns,
   553  		},
   554  		Spec: v1alpha1.HTTPChaosSpec{
   555  			PodSelector: v1alpha1.PodSelector{
   556  				Selector: v1alpha1.PodSelectorSpec{
   557  					GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
   558  						Namespaces:     []string{ns},
   559  						LabelSelectors: map[string]string{"app": "http"},
   560  					},
   561  				},
   562  				Mode: v1alpha1.OneMode,
   563  			},
   564  			Port:   8080,
   565  			Target: "Request",
   566  			PodHttpChaosActions: v1alpha1.PodHttpChaosActions{
   567  				Replace: &v1alpha1.PodHttpChaosReplaceActions{
   568  					Headers: map[string]string{
   569  						SECRET: replaceSecret,
   570  					},
   571  					Body: []byte(replacebody),
   572  				},
   573  			},
   574  		},
   575  	}
   576  	err = cli.Create(ctx, httpChaos)
   577  	framework.ExpectNoError(err, "create http chaos error")
   578  
   579  	chaosKey := types.NamespacedName{
   580  		Namespace: ns,
   581  		Name:      "http-chaos",
   582  	}
   583  
   584  	By("waiting for assertion http chaos")
   585  	err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
   586  		chaos := &v1alpha1.HTTPChaos{}
   587  		err = cli.Get(ctx, chaosKey, chaos)
   588  		framework.ExpectNoError(err, "get http chaos error")
   589  
   590  		for _, c := range chaos.GetStatus().Conditions {
   591  			if c.Type == v1alpha1.ConditionAllInjected {
   592  				if c.Status != corev1.ConditionTrue {
   593  					return false, nil
   594  				}
   595  			} else if c.Type == v1alpha1.ConditionSelected {
   596  				if c.Status != corev1.ConditionTrue {
   597  					return false, nil
   598  				}
   599  			}
   600  		}
   601  
   602  		resp, err := getPodHttp(c, port, secret, body)
   603  		if err != nil {
   604  			return false, err
   605  		}
   606  		defer resp.Body.Close()
   607  
   608  		s := resp.Header.Get(SECRET)
   609  		b, err := io.ReadAll(resp.Body)
   610  		if err != nil {
   611  			return false, err
   612  		}
   613  
   614  		klog.Infof("Status(%d), Secret(%s), Body(%s)", resp.StatusCode, s, string(b))
   615  
   616  		if s == replaceSecret && string(b) == replacebody {
   617  			return true, nil
   618  		}
   619  		return false, nil
   620  	})
   621  	framework.ExpectNoError(err, "http chaos doesn't work as expected")
   622  
   623  	By("pause http replace chaos experiment")
   624  	// pause experiment
   625  	err = util.PauseChaos(ctx, cli, httpChaos)
   626  	framework.ExpectNoError(err, "pause chaos error")
   627  
   628  	By("waiting for assertion about pause")
   629  	err = wait.Poll(1*time.Second, 1*time.Minute, func() (done bool, err error) {
   630  		chaos := &v1alpha1.HTTPChaos{}
   631  		err = cli.Get(ctx, chaosKey, chaos)
   632  		framework.ExpectNoError(err, "get http chaos error")
   633  
   634  		for _, c := range chaos.GetStatus().Conditions {
   635  			if c.Type == v1alpha1.ConditionAllRecovered {
   636  				if c.Status != corev1.ConditionTrue {
   637  					return false, nil
   638  				}
   639  			} else if c.Type == v1alpha1.ConditionSelected {
   640  				if c.Status != corev1.ConditionTrue {
   641  					return false, nil
   642  				}
   643  			}
   644  		}
   645  
   646  		return true, err
   647  	})
   648  	framework.ExpectNoError(err, "check paused chaos failed")
   649  
   650  	// wait 1 min to check whether io replace still exists
   651  	err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
   652  		resp, err := getPodHttp(c, port, secret, body)
   653  		if err != nil {
   654  			return false, err
   655  		}
   656  		defer resp.Body.Close()
   657  
   658  		s := resp.Header.Get(SECRET)
   659  		b, err := io.ReadAll(resp.Body)
   660  		if err != nil {
   661  			return false, err
   662  		}
   663  
   664  		klog.Infof("Status(%d), Secret(%s), Body(%s)", resp.StatusCode, s, string(b))
   665  
   666  		if s == secret && string(b) == body {
   667  			return true, nil
   668  		}
   669  		return false, nil
   670  	})
   671  	framework.ExpectNoError(err, "fail to recover http chaos")
   672  	By("resume http replace chaos experiment")
   673  	// resume experiment
   674  	err = util.UnPauseChaos(ctx, cli, httpChaos)
   675  	framework.ExpectNoError(err, "resume chaos error")
   676  
   677  	By("assert that http replace is effective again")
   678  	err = wait.Poll(1*time.Second, 1*time.Minute, func() (done bool, err error) {
   679  		chaos := &v1alpha1.HTTPChaos{}
   680  		err = cli.Get(ctx, chaosKey, chaos)
   681  		framework.ExpectNoError(err, "get http chaos error")
   682  
   683  		for _, c := range chaos.GetStatus().Conditions {
   684  			if c.Type == v1alpha1.ConditionAllInjected {
   685  				if c.Status != corev1.ConditionTrue {
   686  					return false, nil
   687  				}
   688  			} else if c.Type == v1alpha1.ConditionSelected {
   689  				if c.Status != corev1.ConditionTrue {
   690  					return false, nil
   691  				}
   692  			}
   693  		}
   694  
   695  		return true, err
   696  	})
   697  	framework.ExpectNoError(err, "check resumed chaos failed")
   698  
   699  	err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
   700  		resp, err := getPodHttp(c, port, secret, body)
   701  		if err != nil {
   702  			return false, err
   703  		}
   704  		defer resp.Body.Close()
   705  
   706  		s := resp.Header.Get(SECRET)
   707  		b, err := io.ReadAll(resp.Body)
   708  		if err != nil {
   709  			return false, err
   710  		}
   711  
   712  		klog.Infof("Status(%d), Secret(%s), Body(%s)", resp.StatusCode, s, string(b))
   713  
   714  		if s == replaceSecret && string(b) == replacebody {
   715  			return true, nil
   716  		}
   717  		return false, nil
   718  	})
   719  	framework.ExpectNoError(err, "HTTP chaos doesn't work as expected")
   720  
   721  	By("cleanup")
   722  	// cleanup
   723  	cli.Delete(ctx, httpChaos)
   724  }
   725