time_skew.go

Documentation: github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/chaos/timechaos

     1  // Copyright 2021 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  // http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  
    16  package timechaos
    17  
    18  import (
    19  	"context"
    20  	"net/http"
    21  	"time"
    22  
    23  	. "github.com/onsi/ginkgo/v2"
    24  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    25  	"k8s.io/apimachinery/pkg/types"
    26  	"k8s.io/apimachinery/pkg/util/wait"
    27  	"k8s.io/kubernetes/test/e2e/framework"
    28  	"k8s.io/utils/pointer"
    29  	"sigs.k8s.io/controller-runtime/pkg/client"
    30  
    31  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    32  	"github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/util"
    33  )
    34  
    35  func TestcaseTimeSkewOnceThenRecover(
    36  	ns string,
    37  	cli client.Client,
    38  	c http.Client,
    39  	port uint16,
    40  ) {
    41  
    42  	ctx, cancel := context.WithCancel(context.Background())
    43  	defer cancel()
    44  
    45  	By("wait e2e helper ready")
    46  	err := util.WaitE2EHelperReady(c, port)
    47  	framework.ExpectNoError(err, "wait e2e helper ready error")
    48  
    49  	By("create chaos CRD objects")
    50  	initTime, err := getPodTimeNS(c, port)
    51  	framework.ExpectNoError(err, "failed to get pod time")
    52  
    53  	timeChaos := &v1alpha1.TimeChaos{
    54  		ObjectMeta: metav1.ObjectMeta{
    55  			Name:      "timer-time-chaos",
    56  			Namespace: ns,
    57  		},
    58  		Spec: v1alpha1.TimeChaosSpec{
    59  			Duration:   pointer.StringPtr("9m"),
    60  			TimeOffset: "-1h",
    61  			ContainerSelector: v1alpha1.ContainerSelector{
    62  				PodSelector: v1alpha1.PodSelector{
    63  					Selector: v1alpha1.PodSelectorSpec{
    64  						GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
    65  							Namespaces:     []string{ns},
    66  							LabelSelectors: map[string]string{"app": "timer"},
    67  						},
    68  					},
    69  					Mode: v1alpha1.OneMode,
    70  				},
    71  			},
    72  		},
    73  	}
    74  	err = cli.Create(ctx, timeChaos)
    75  	framework.ExpectNoError(err, "create time chaos error")
    76  
    77  	By("waiting for assertion")
    78  	err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (done bool, err error) {
    79  		podTime, err := getPodTimeNS(c, port)
    80  		framework.ExpectNoError(err, "failed to get pod time")
    81  		if podTime.Before(*initTime) {
    82  			return true, nil
    83  		}
    84  		return false, nil
    85  	})
    86  	framework.ExpectNoError(err, "time chaos doesn't work as expected")
    87  
    88  	By("delete chaos CRD objects")
    89  	err = cli.Delete(ctx, timeChaos)
    90  	framework.ExpectNoError(err, "failed to delete time chaos")
    91  
    92  	By("waiting for assertion recovering")
    93  	err = wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) {
    94  		podTime, err := getPodTimeNS(c, port)
    95  		framework.ExpectNoError(err, "failed to get pod time")
    96  		// since there is no timechaos now, current pod time should not be earlier
    97  		// than the init time
    98  		if podTime.Before(*initTime) {
    99  			return true, nil
   100  		}
   101  		return false, nil
   102  	})
   103  	framework.ExpectError(err, "wait no timechaos error")
   104  	framework.ExpectEqual(err.Error(), wait.ErrWaitTimeout.Error())
   105  	By("success to perform time chaos")
   106  }
   107  
   108  func TestcaseTimeSkewPauseThenUnpause(
   109  	ns string,
   110  	cli client.Client,
   111  	c http.Client,
   112  	port uint16,
   113  ) {
   114  	ctx, cancel := context.WithCancel(context.Background())
   115  	defer cancel()
   116  
   117  	By("wait e2e helper ready")
   118  	err := util.WaitE2EHelperReady(c, port)
   119  	framework.ExpectNoError(err, "wait e2e helper ready error")
   120  
   121  	initTime, err := getPodTimeNS(c, port)
   122  	framework.ExpectNoError(err, "failed to get pod time")
   123  
   124  	By("create chaos CRD objects")
   125  	timeChaos := &v1alpha1.TimeChaos{
   126  		ObjectMeta: metav1.ObjectMeta{
   127  			Name:      "timer-time-chaos",
   128  			Namespace: ns,
   129  		},
   130  		Spec: v1alpha1.TimeChaosSpec{
   131  			Duration:   pointer.StringPtr("9m"),
   132  			TimeOffset: "-1h",
   133  			ContainerSelector: v1alpha1.ContainerSelector{
   134  				PodSelector: v1alpha1.PodSelector{
   135  					Selector: v1alpha1.PodSelectorSpec{
   136  						GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
   137  							Namespaces:     []string{ns},
   138  							LabelSelectors: map[string]string{"app": "timer"},
   139  						},
   140  					},
   141  					Mode: v1alpha1.OneMode,
   142  				},
   143  			},
   144  		},
   145  	}
   146  	err = cli.Create(ctx, timeChaos)
   147  	framework.ExpectNoError(err, "create time chaos error")
   148  
   149  	By("waiting for assertion")
   150  	err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   151  		podTime, err := getPodTimeNS(c, port)
   152  		framework.ExpectNoError(err, "failed to get pod time")
   153  		if podTime.Before(*initTime) {
   154  			return true, nil
   155  		}
   156  		return false, nil
   157  	})
   158  	framework.ExpectNoError(err, "time chaos doesn't work as expected")
   159  
   160  	chaosKey := types.NamespacedName{
   161  		Namespace: ns,
   162  		Name:      "timer-time-chaos",
   163  	}
   164  
   165  	By("pause time skew chaos experiment")
   166  	// pause experiment
   167  	err = util.PauseChaos(ctx, cli, timeChaos)
   168  	framework.ExpectNoError(err, "pause chaos error")
   169  
   170  	By("assert pause is effective")
   171  	err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   172  		chaos := &v1alpha1.TimeChaos{}
   173  		err = cli.Get(ctx, chaosKey, chaos)
   174  		framework.ExpectNoError(err, "get time chaos error")
   175  		if chaos.Status.Experiment.DesiredPhase == v1alpha1.StoppedPhase {
   176  			return true, nil
   177  		}
   178  		return false, err
   179  	})
   180  	framework.ExpectNoError(err, "check paused chaos failed")
   181  
   182  	// wait for 1 minutes and check timer
   183  	framework.ExpectNoError(err, "get timer pod error")
   184  	err = wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) {
   185  		podTime, err := getPodTimeNS(c, port)
   186  		framework.ExpectNoError(err, "failed to get pod time")
   187  		if podTime.Before(*initTime) {
   188  			return true, nil
   189  		}
   190  		return false, nil
   191  	})
   192  	framework.ExpectError(err, "wait time chaos paused error")
   193  	framework.ExpectEqual(err.Error(), wait.ErrWaitTimeout.Error())
   194  
   195  	By("resume time skew chaos experiment")
   196  	err = util.UnPauseChaos(ctx, cli, timeChaos)
   197  	framework.ExpectNoError(err, "resume chaos error")
   198  
   199  	By("assert chaos experiment resumed")
   200  	err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   201  		chaos := &v1alpha1.TimeChaos{}
   202  		err = cli.Get(ctx, chaosKey, chaos)
   203  		framework.ExpectNoError(err, "get time chaos error")
   204  		if chaos.Status.Experiment.DesiredPhase == v1alpha1.RunningPhase {
   205  			return true, nil
   206  		}
   207  		return false, err
   208  	})
   209  	framework.ExpectNoError(err, "check resumed chaos failed")
   210  
   211  	// timechaos is running again, we want to check pod
   212  	// whether time is earlier than init time,
   213  	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (done bool, err error) {
   214  		podTime, err := getPodTimeNS(c, port)
   215  		framework.ExpectNoError(err, "failed to get pod time")
   216  		if podTime.Before(*initTime) {
   217  			return true, nil
   218  		}
   219  		return false, nil
   220  	})
   221  	framework.ExpectNoError(err, "time chaos failed")
   222  
   223  	By("delete chaos CRD objects")
   224  	cli.Delete(ctx, timeChaos)
   225  }
   226  
   227  func TestcaseTimeSkewShouldAlsoAffectChildProces(
   228  	ns string,
   229  	cli client.Client,
   230  	c http.Client,
   231  	port uint16,
   232  ) {
   233  
   234  	ctx, cancel := context.WithCancel(context.Background())
   235  	defer cancel()
   236  
   237  	By("wait e2e helper ready")
   238  	err := util.WaitE2EHelperReady(c, port)
   239  	framework.ExpectNoError(err, "wait e2e helper ready error")
   240  
   241  	By("create chaos CRD objects")
   242  	initTime, err := getPodChildProcessTimeNS(c, port)
   243  	framework.ExpectNoError(err, "failed to get pod time")
   244  
   245  	timeChaos := &v1alpha1.TimeChaos{
   246  		ObjectMeta: metav1.ObjectMeta{
   247  			Name:      "timer-time-chaos",
   248  			Namespace: ns,
   249  		},
   250  		Spec: v1alpha1.TimeChaosSpec{
   251  			Duration:   pointer.StringPtr("9m"),
   252  			TimeOffset: "-1h",
   253  			ContainerSelector: v1alpha1.ContainerSelector{
   254  				PodSelector: v1alpha1.PodSelector{
   255  					Selector: v1alpha1.PodSelectorSpec{
   256  						GenericSelectorSpec: v1alpha1.GenericSelectorSpec{
   257  							Namespaces:     []string{ns},
   258  							LabelSelectors: map[string]string{"app": "timer"},
   259  						},
   260  					},
   261  					Mode: v1alpha1.OneMode,
   262  				},
   263  			},
   264  		},
   265  	}
   266  	err = cli.Create(ctx, timeChaos)
   267  	framework.ExpectNoError(err, "create time chaos error")
   268  
   269  	By("waiting for assertion")
   270  	err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   271  		podTime, err := getPodChildProcessTimeNS(c, port)
   272  		framework.ExpectNoError(err, "failed to get pod time")
   273  		if podTime.Before(*initTime) {
   274  			return true, nil
   275  		}
   276  		return false, nil
   277  	})
   278  	framework.ExpectNoError(err, "time chaos doesn't work as expected")
   279  
   280  	By("delete chaos CRD objects")
   281  	err = cli.Delete(ctx, timeChaos)
   282  	framework.ExpectNoError(err, "failed to delete time chaos")
   283  
   284  	By("waiting for assertion recovering")
   285  	err = wait.Poll(5*time.Second, 1*time.Minute, func() (done bool, err error) {
   286  		podTime, err := getPodChildProcessTimeNS(c, port)
   287  		framework.ExpectNoError(err, "failed to get pod time")
   288  		// since there is no timechaos now, current pod time should not be earlier
   289  		// than the init time
   290  		if podTime.Before(*initTime) {
   291  			return true, nil
   292  		}
   293  		return false, nil
   294  	})
   295  	framework.ExpectError(err, "wait no timechaos error")
   296  	framework.ExpectEqual(err.Error(), wait.ErrWaitTimeout.Error())
   297  	By("success to perform time chaos")
   298  }
   299