e2e.go

Documentation: github.com/chaos-mesh/chaos-mesh/e2e-test/e2e

     1  // Copyright 2021 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  // http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  
    16  package e2e
    17  
    18  import (
    19  	"context"
    20  	// load pprof
    21  	_ "net/http/pprof"
    22  	"os/exec"
    23  	"time"
    24  
    25  	"github.com/onsi/ginkgo"
    26  	v1 "k8s.io/api/core/v1"
    27  	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
    28  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    29  	"k8s.io/client-go/kubernetes"
    30  	"k8s.io/klog/v2"
    31  	aggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
    32  	"k8s.io/kubernetes/test/e2e/framework"
    33  	e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
    34  	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
    35  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    36  	utilnet "k8s.io/utils/net"
    37  
    38  	test "github.com/chaos-mesh/chaos-mesh/e2e-test"
    39  	e2econfig "github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/config"
    40  
    41  	// ensure auth plugins are loaded
    42  	_ "k8s.io/client-go/plugin/pkg/client/auth"
    43  )
    44  
    45  const namespaceCleanupTimeout = 15 * time.Minute
    46  
    47  // This is modified from framework.SetupSuite().
    48  // setupSuite is the boilerplate that can be used to setup ginkgo test suites, on the SynchronizedBeforeSuite step.
    49  // There are certain operations we only want to run once per overall test invocation
    50  // (such as deleting old namespaces, or verifying that all system pods are running.
    51  // Because of the way Ginkgo runs tests in parallel, we must use SynchronizedBeforeSuite
    52  // to ensure that these operations only run on the first parallel Ginkgo node.
    53  func setupSuite() {
    54  	// Run only on Ginkgo node 1
    55  
    56  	c, err := framework.LoadClientset()
    57  	if err != nil {
    58  		klog.Fatal("Error loading client: ", err)
    59  	}
    60  
    61  	// Delete any namespaces except those created by the system. This ensures no
    62  	// lingering resources are left over from a previous test run.
    63  	if framework.TestContext.CleanStart {
    64  		deleted, err := framework.DeleteNamespaces(c, nil, /* deleteFilter */
    65  			[]string{
    66  				metav1.NamespaceSystem,
    67  				metav1.NamespaceDefault,
    68  				metav1.NamespacePublic,
    69  				v1.NamespaceNodeLease,
    70  				// kind local path provisioner namespace since 0.7.0
    71  				// https://github.com/kubernetes-sigs/kind/blob/v0.7.0/pkg/build/node/storage.go#L35
    72  				"local-path-storage",
    73  			})
    74  		if err != nil {
    75  			e2elog.Failf("Error deleting orphaned namespaces: %v", err)
    76  		}
    77  		klog.Infof("Waiting for deletion of the following namespaces: %v", deleted)
    78  		if err := framework.WaitForNamespacesDeleted(c, deleted, namespaceCleanupTimeout); err != nil {
    79  			e2elog.Failf("Failed to delete orphaned namespaces %v: %v", deleted, err)
    80  		}
    81  	}
    82  
    83  	// In large clusters we may get to this point but still have a bunch
    84  	// of nodes without Routes created. Since this would make a node
    85  	// unschedulable, we need to wait until all of them are schedulable.
    86  	framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout))
    87  
    88  	//// If NumNodes is not specified then auto-detect how many are scheduleable and not tainted
    89  	//if framework.TestContext.CloudConfig.NumNodes == framework.DefaultNumNodes {
    90  	//	framework.TestContext.CloudConfig.NumNodes = len(framework.GetReadySchedulableNodesOrDie(c).Items)
    91  	//}
    92  
    93  	// Ensure all pods are running and ready before starting tests (otherwise,
    94  	// cluster infrastructure pods that are being pulled or started can block
    95  	// test pods from running, and tests that ensure all pods are running and
    96  	// ready will fail).
    97  	podStartupTimeout := framework.TestContext.SystemPodsStartupTimeout
    98  	// TODO: In large clusters, we often observe a non-starting pods due to
    99  	// #41007. To avoid those pods preventing the whole test runs (and just
   100  	// wasting the whole run), we allow for some not-ready pods (with the
   101  	// number equal to the number of allowed not-ready nodes).
   102  	if err := e2epod.WaitForPodsRunningReady(c, metav1.NamespaceSystem, int32(framework.TestContext.MinStartupPods), int32(framework.TestContext.AllowedNotReadyNodes), podStartupTimeout, map[string]string{}); err != nil {
   103  		framework.DumpAllNamespaceInfo(c, metav1.NamespaceSystem)
   104  		e2ekubectl.LogFailedContainers(c, metav1.NamespaceSystem, e2elog.Logf)
   105  		e2elog.Failf("Error waiting for all pods to be running and ready: %v", err)
   106  	}
   107  
   108  	//if err := framework.WaitForDaemonSets(c, metav1.NamespaceSystem, int32(framework.TestContext.AllowedNotReadyNodes), framework.TestContext.SystemDaemonsetStartupTimeout); err != nil {
   109  	//	e2elog.Logf("WARNING: Waiting for all daemonsets to be ready failed: %v", err)
   110  	//}
   111  
   112  	dc := c.DiscoveryClient
   113  
   114  	serverVersion, serverErr := dc.ServerVersion()
   115  	if serverErr != nil {
   116  		e2elog.Logf("Unexpected server error retrieving version: %v", serverErr)
   117  	}
   118  	if serverVersion != nil {
   119  		e2elog.Logf("kube-apiserver version: %s", serverVersion.GitVersion)
   120  	}
   121  }
   122  
   123  var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
   124  	if e2econfig.TestConfig.InstallChaosMesh {
   125  		ginkgo.By("Clear all helm releases")
   126  		helmClearCmd := "helm ls --all --short | xargs -n 1 -r helm delete --purge"
   127  		if err := exec.Command("sh", "-c", helmClearCmd).Run(); err != nil {
   128  			framework.Failf("failed to clear helm releases (cmd: %q, error: %v", helmClearCmd, err)
   129  		}
   130  		ginkgo.By("Clear non-kubernetes apiservices")
   131  		clearNonK8SAPIServicesCmd := "kubectl delete apiservices -l kube-aggregator.kubernetes.io/automanaged!=onstart"
   132  		if err := exec.Command("sh", "-c", clearNonK8SAPIServicesCmd).Run(); err != nil {
   133  			framework.Failf("failed to clear non-kubernetes apiservices (cmd: %q, error: %v", clearNonK8SAPIServicesCmd, err)
   134  		}
   135  
   136  		setupSuite()
   137  
   138  		// Get clients
   139  		config, err := framework.LoadConfig()
   140  		framework.ExpectNoError(err, "failed to load config")
   141  		kubeCli, err := kubernetes.NewForConfig(config)
   142  		framework.ExpectNoError(err, "failed to create clientset")
   143  		aggrCli, err := aggregatorclientset.NewForConfig(config)
   144  		framework.ExpectNoError(err, "failed to create clientset")
   145  		apiExtCli, err := apiextensionsclientset.NewForConfig(config)
   146  		framework.ExpectNoError(err, "failed to create clientset")
   147  		oa := test.NewOperatorAction(kubeCli, aggrCli, apiExtCli, e2econfig.TestConfig)
   148  		ocfg := test.NewDefaultOperatorConfig()
   149  		ocfg.Manager.ImageRegistry = e2econfig.TestConfig.ManagerImageRegistry
   150  		ocfg.Manager.ImageRepository = e2econfig.TestConfig.ManagerImage
   151  		ocfg.Manager.ImageTag = e2econfig.TestConfig.ManagerTag
   152  		ocfg.Daemon.ImageRegistry = e2econfig.TestConfig.DaemonImageRegistry
   153  		ocfg.Daemon.ImageRepository = e2econfig.TestConfig.DaemonImage
   154  		ocfg.Daemon.ImageTag = e2econfig.TestConfig.DaemonTag
   155  		ocfg.DNSImage = e2econfig.TestConfig.ChaosDNSImage
   156  		ocfg.EnableDashboard = e2econfig.TestConfig.EnableDashboard
   157  
   158  		oa.CleanCRDOrDie()
   159  		err = oa.InstallCRD(ocfg)
   160  		framework.ExpectNoError(err, "failed to install crd")
   161  		err = oa.DeployOperator(ocfg)
   162  		framework.ExpectNoError(err, "failed to install chaos-mesh")
   163  	}
   164  	return nil
   165  }, func(data []byte) {
   166  	// Run on all Ginkgo nodes
   167  	setupSuitePerGinkgoNode()
   168  })
   169  
   170  func setupSuitePerGinkgoNode() {
   171  	c, err := framework.LoadClientset()
   172  	if err != nil {
   173  		klog.Fatal("Error loading client: ", err)
   174  	}
   175  	framework.TestContext.IPFamily = getDefaultClusterIPFamily(c)
   176  	e2elog.Logf("Cluster IP family: %s", framework.TestContext.IPFamily)
   177  }
   178  
   179  // getDefaultClusterIPFamily obtains the default IP family of the cluster
   180  // using the Cluster IP address of the kubernetes service created in the default namespace
   181  // This unequivocally identifies the default IP family because services are single family
   182  // TODO: dual-stack may support multiple families per service
   183  // but we can detect if a cluster is dual stack because pods have two addresses (one per family)
   184  func getDefaultClusterIPFamily(c kubernetes.Interface) string {
   185  	// Get the ClusterIP of the kubernetes service created in the default namespace
   186  	svc, err := c.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), "kubernetes", metav1.GetOptions{})
   187  	if err != nil {
   188  		e2elog.Failf("Failed to get kubernetes service ClusterIP: %v", err)
   189  	}
   190  
   191  	if utilnet.IsIPv6String(svc.Spec.ClusterIP) {
   192  		return "ipv6"
   193  	}
   194  	return "ipv4"
   195  }
   196