e2e.go

Documentation: github.com/chaos-mesh/chaos-mesh/e2e-test/e2e

     1  // Copyright 2020 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package e2e
    15  
    16  import (
    17  	// load pprof
    18  	_ "net/http/pprof"
    19  	"os/exec"
    20  
    21  	"github.com/onsi/ginkgo"
    22  	v1 "k8s.io/api/core/v1"
    23  	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
    24  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    25  	"k8s.io/client-go/kubernetes"
    26  	"k8s.io/klog"
    27  	aggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
    28  	"k8s.io/kubernetes/test/e2e/framework"
    29  	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
    30  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    31  	utilnet "k8s.io/utils/net"
    32  
    33  	test "github.com/chaos-mesh/chaos-mesh/e2e-test"
    34  	e2econfig "github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/config"
    35  
    36  	// ensure auth plugins are loaded
    37  	_ "k8s.io/client-go/plugin/pkg/client/auth"
    38  )
    39  
    40  // This is modified from framework.SetupSuite().
    41  // setupSuite is the boilerplate that can be used to setup ginkgo test suites, on the SynchronizedBeforeSuite step.
    42  // There are certain operations we only want to run once per overall test invocation
    43  // (such as deleting old namespaces, or verifying that all system pods are running.
    44  // Because of the way Ginkgo runs tests in parallel, we must use SynchronizedBeforeSuite
    45  // to ensure that these operations only run on the first parallel Ginkgo node.
    46  func setupSuite() {
    47  	// Run only on Ginkgo node 1
    48  
    49  	c, err := framework.LoadClientset()
    50  	if err != nil {
    51  		klog.Fatal("Error loading client: ", err)
    52  	}
    53  
    54  	// Delete any namespaces except those created by the system. This ensures no
    55  	// lingering resources are left over from a previous test run.
    56  	if framework.TestContext.CleanStart {
    57  		deleted, err := framework.DeleteNamespaces(c, nil, /* deleteFilter */
    58  			[]string{
    59  				metav1.NamespaceSystem,
    60  				metav1.NamespaceDefault,
    61  				metav1.NamespacePublic,
    62  				v1.NamespaceNodeLease,
    63  				// kind local path provisioner namespace since 0.7.0
    64  				// https://github.com/kubernetes-sigs/kind/blob/v0.7.0/pkg/build/node/storage.go#L35
    65  				"local-path-storage",
    66  			})
    67  		if err != nil {
    68  			e2elog.Failf("Error deleting orphaned namespaces: %v", err)
    69  		}
    70  		klog.Infof("Waiting for deletion of the following namespaces: %v", deleted)
    71  		if err := framework.WaitForNamespacesDeleted(c, deleted, framework.NamespaceCleanupTimeout); err != nil {
    72  			e2elog.Failf("Failed to delete orphaned namespaces %v: %v", deleted, err)
    73  		}
    74  	}
    75  
    76  	// In large clusters we may get to this point but still have a bunch
    77  	// of nodes without Routes created. Since this would make a node
    78  	// unschedulable, we need to wait until all of them are schedulable.
    79  	framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout))
    80  
    81  	//// If NumNodes is not specified then auto-detect how many are scheduleable and not tainted
    82  	//if framework.TestContext.CloudConfig.NumNodes == framework.DefaultNumNodes {
    83  	//	framework.TestContext.CloudConfig.NumNodes = len(framework.GetReadySchedulableNodesOrDie(c).Items)
    84  	//}
    85  
    86  	// Ensure all pods are running and ready before starting tests (otherwise,
    87  	// cluster infrastructure pods that are being pulled or started can block
    88  	// test pods from running, and tests that ensure all pods are running and
    89  	// ready will fail).
    90  	podStartupTimeout := framework.TestContext.SystemPodsStartupTimeout
    91  	// TODO: In large clusters, we often observe a non-starting pods due to
    92  	// #41007. To avoid those pods preventing the whole test runs (and just
    93  	// wasting the whole run), we allow for some not-ready pods (with the
    94  	// number equal to the number of allowed not-ready nodes).
    95  	if err := e2epod.WaitForPodsRunningReady(c, metav1.NamespaceSystem, int32(framework.TestContext.MinStartupPods), int32(framework.TestContext.AllowedNotReadyNodes), podStartupTimeout, map[string]string{}); err != nil {
    96  		framework.DumpAllNamespaceInfo(c, metav1.NamespaceSystem)
    97  		framework.LogFailedContainers(c, metav1.NamespaceSystem, e2elog.Logf)
    98  		e2elog.Failf("Error waiting for all pods to be running and ready: %v", err)
    99  	}
   100  
   101  	//if err := framework.WaitForDaemonSets(c, metav1.NamespaceSystem, int32(framework.TestContext.AllowedNotReadyNodes), framework.TestContext.SystemDaemonsetStartupTimeout); err != nil {
   102  	//	e2elog.Logf("WARNING: Waiting for all daemonsets to be ready failed: %v", err)
   103  	//}
   104  
   105  	dc := c.DiscoveryClient
   106  
   107  	serverVersion, serverErr := dc.ServerVersion()
   108  	if serverErr != nil {
   109  		e2elog.Logf("Unexpected server error retrieving version: %v", serverErr)
   110  	}
   111  	if serverVersion != nil {
   112  		e2elog.Logf("kube-apiserver version: %s", serverVersion.GitVersion)
   113  	}
   114  }
   115  
   116  var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
   117  	if e2econfig.TestConfig.InstallChaosMesh {
   118  		ginkgo.By("Clear all helm releases")
   119  		helmClearCmd := "helm ls --all --short | xargs -n 1 -r helm delete --purge"
   120  		if err := exec.Command("sh", "-c", helmClearCmd).Run(); err != nil {
   121  			framework.Failf("failed to clear helm releases (cmd: %q, error: %v", helmClearCmd, err)
   122  		}
   123  		ginkgo.By("Clear non-kubernetes apiservices")
   124  		clearNonK8SAPIServicesCmd := "kubectl delete apiservices -l kube-aggregator.kubernetes.io/automanaged!=onstart"
   125  		if err := exec.Command("sh", "-c", clearNonK8SAPIServicesCmd).Run(); err != nil {
   126  			framework.Failf("failed to clear non-kubernetes apiservices (cmd: %q, error: %v", clearNonK8SAPIServicesCmd, err)
   127  		}
   128  
   129  		setupSuite()
   130  
   131  		// Get clients
   132  		config, err := framework.LoadConfig()
   133  		framework.ExpectNoError(err, "failed to load config")
   134  		kubeCli, err := kubernetes.NewForConfig(config)
   135  		framework.ExpectNoError(err, "failed to create clientset")
   136  		aggrCli, err := aggregatorclientset.NewForConfig(config)
   137  		framework.ExpectNoError(err, "failed to create clientset")
   138  		apiExtCli, err := apiextensionsclientset.NewForConfig(config)
   139  		framework.ExpectNoError(err, "failed to create clientset")
   140  		oa := test.NewOperatorAction(kubeCli, aggrCli, apiExtCli, e2econfig.TestConfig)
   141  		ocfg := test.NewDefaultOperatorConfig()
   142  		ocfg.Manager.Image = e2econfig.TestConfig.ManagerImage
   143  		ocfg.Manager.Tag = e2econfig.TestConfig.ManagerTag
   144  		ocfg.Daemon.Image = e2econfig.TestConfig.DaemonImage
   145  		ocfg.Daemon.Tag = e2econfig.TestConfig.DaemonTag
   146  		ocfg.DNSImage = e2econfig.TestConfig.ChaosDNSImage
   147  		ocfg.EnableDashboard = e2econfig.TestConfig.EnableDashboard
   148  
   149  		oa.CleanCRDOrDie()
   150  		err = oa.InstallCRD(ocfg)
   151  		framework.ExpectNoError(err, "failed to install crd")
   152  		err = oa.DeployOperator(ocfg)
   153  		framework.ExpectNoError(err, "failed to install chaos-mesh")
   154  	}
   155  	return nil
   156  }, func(data []byte) {
   157  	// Run on all Ginkgo nodes
   158  	setupSuitePerGinkgoNode()
   159  })
   160  
   161  var _ = ginkgo.SynchronizedAfterSuite(func() {
   162  	framework.CleanupSuite()
   163  }, func() {
   164  	framework.AfterSuiteActions()
   165  })
   166  
   167  func setupSuitePerGinkgoNode() {
   168  	c, err := framework.LoadClientset()
   169  	if err != nil {
   170  		klog.Fatal("Error loading client: ", err)
   171  	}
   172  	framework.TestContext.IPFamily = getDefaultClusterIPFamily(c)
   173  	e2elog.Logf("Cluster IP family: %s", framework.TestContext.IPFamily)
   174  }
   175  
   176  // getDefaultClusterIPFamily obtains the default IP family of the cluster
   177  // using the Cluster IP address of the kubernetes service created in the default namespace
   178  // This unequivocally identifies the default IP family because services are single family
   179  // TODO: dual-stack may support multiple families per service
   180  // but we can detect if a cluster is dual stack because pods have two addresses (one per family)
   181  func getDefaultClusterIPFamily(c kubernetes.Interface) string {
   182  	// Get the ClusterIP of the kubernetes service created in the default namespace
   183  	svc, err := c.CoreV1().Services(metav1.NamespaceDefault).Get("kubernetes", metav1.GetOptions{})
   184  	if err != nil {
   185  		e2elog.Failf("Failed to get kubernetes service ClusterIP: %v", err)
   186  	}
   187  
   188  	if utilnet.IsIPv6String(svc.Spec.ClusterIP) {
   189  		return "ipv6"
   190  	}
   191  	return "ipv4"
   192  }
   193