stresschaos.go

Documentation: github.com/chaos-mesh/chaos-mesh/pkg/chaosctl/debug/stresschaos

     1  // Copyright 2019 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package stresschaos
    15  
    16  import (
    17  	"context"
    18  	"fmt"
    19  	"regexp"
    20  	"strconv"
    21  	"strings"
    22  
    23  	"github.com/pkg/errors"
    24  
    25  	"code.cloudfoundry.org/bytefmt"
    26  	v1 "k8s.io/api/core/v1"
    27  	"k8s.io/apimachinery/pkg/runtime"
    28  
    29  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    30  	cm "github.com/chaos-mesh/chaos-mesh/pkg/chaosctl/common"
    31  )
    32  
    33  // Debug get chaos debug information
    34  func Debug(ctx context.Context, chaos runtime.Object, c *cm.ClientSet, result *cm.ChaosResult) error {
    35  	stressChaos, ok := chaos.(*v1alpha1.StressChaos)
    36  	if !ok {
    37  		return fmt.Errorf("chaos is not stresschaos")
    38  	}
    39  	chaosStatus := stressChaos.Status.ChaosStatus
    40  	chaosSelector := stressChaos.Spec.GetSelector()
    41  
    42  	pods, daemons, err := cm.GetPods(ctx, stressChaos.GetName(), chaosStatus, chaosSelector, c.CtrlCli)
    43  	if err != nil {
    44  		return err
    45  	}
    46  
    47  	if err := cm.CheckFailedMessage(ctx, chaosStatus.FailedMessage, daemons, c); err != nil {
    48  		return err
    49  	}
    50  
    51  	for i := range pods {
    52  		podName := pods[i].Name
    53  		podResult := cm.PodResult{Name: podName}
    54  		_ = debugEachPod(ctx, pods[i], daemons[i], stressChaos, c, &podResult)
    55  		result.Pods = append(result.Pods, podResult)
    56  		// TODO: V(4) log when err != nil, wait for #1433
    57  	}
    58  	return nil
    59  }
    60  
    61  func debugEachPod(ctx context.Context, pod v1.Pod, daemon v1.Pod, chaos *v1alpha1.StressChaos, c *cm.ClientSet, result *cm.PodResult) error {
    62  	// get process path
    63  	cmd := "cat /proc/cgroups"
    64  	out, err := cm.ExecBypass(ctx, pod, daemon, cmd, c.KubeCli)
    65  	if err != nil {
    66  		return errors.Wrap(err, fmt.Sprintf("run command %s failed", cmd))
    67  	}
    68  
    69  	cmd = "ps"
    70  	out, err = cm.ExecBypass(ctx, pod, daemon, cmd, c.KubeCli)
    71  	if err != nil {
    72  		return errors.Wrap(err, fmt.Sprintf("run command %s failed", cmd))
    73  	}
    74  	result.Items = append(result.Items, cm.ItemResult{Name: "ps", Value: string(out)})
    75  	stressngLine := regexp.MustCompile("(.*)(stress-ng)").FindStringSubmatch(string(out))
    76  	if len(stressngLine) == 0 {
    77  		return fmt.Errorf("could not find stress-ng, StressChaos failed")
    78  	}
    79  
    80  	pids, commands, err := cm.GetPidFromPS(ctx, pod, daemon, c.KubeCli)
    81  	if err != nil {
    82  		return errors.Wrap(err, "get pid from ps failed")
    83  	}
    84  
    85  	for i := range pids {
    86  		cmd = fmt.Sprintf("cat /proc/%s/cgroup", pids[i])
    87  		out, err = cm.ExecBypass(ctx, pod, daemon, cmd, c.KubeCli)
    88  		if err != nil {
    89  			cm.L().WithName("stress-chaos").V(2).Info("failed to fetch cgroup ofr certain process",
    90  				"pod", fmt.Sprintf("%s/%s", pod.Namespace, pod.Name),
    91  				"pid", i,
    92  			)
    93  			result.Items = append(result.Items, cm.ItemResult{Name: fmt.Sprintf("/proc/%s/cgroup of %s", pids[i], commands[i]), Value: "No cgroup found"})
    94  		} else {
    95  			result.Items = append(result.Items, cm.ItemResult{Name: fmt.Sprintf("/proc/%s/cgroup of %s", pids[i], commands[i]), Value: string(out)})
    96  		}
    97  	}
    98  
    99  	// no more info for StressngStressors
   100  	if chaos.Spec.StressngStressors != "" {
   101  		return nil
   102  	}
   103  
   104  	isCPU := true
   105  	if cpuSpec := chaos.Spec.Stressors.CPUStressor; cpuSpec == nil {
   106  		isCPU = false
   107  	}
   108  
   109  	var expr, cpuMountType string
   110  	if isCPU {
   111  		if regexp.MustCompile("(cpu,cpuacct)").MatchString(string(out)) {
   112  			cpuMountType = "cpu,cpuacct"
   113  		} else {
   114  			cpuMountType = "cpu"
   115  		}
   116  		expr = "(?::" + cpuMountType + ":)(.*)"
   117  	} else {
   118  		expr = "(?::memory:)(.*)"
   119  	}
   120  	processPath := regexp.MustCompile(expr).FindStringSubmatch(string(out))[1]
   121  
   122  	// print out debug info
   123  	if isCPU {
   124  		cmd = fmt.Sprintf("cat /sys/fs/cgroup/%s/%s/cpu.cfs_quota_us", cpuMountType, processPath)
   125  		out, err = cm.Exec(ctx, daemon, cmd, c.KubeCli)
   126  		if err != nil {
   127  			return errors.Wrap(err, fmt.Sprintf("run command %s failed", cmd))
   128  		}
   129  		result.Items = append(result.Items, cm.ItemResult{Name: "cpu.cfs_quota_us", Value: string(out)})
   130  		quota, err := strconv.Atoi(strings.TrimSuffix(string(out), "\n"))
   131  		if err != nil {
   132  			return errors.Wrap(err, "could not get cpu.cfs_quota_us")
   133  		}
   134  
   135  		cmd = fmt.Sprintf("cat /sys/fs/cgroup/%s/%s/cpu.cfs_period_us", cpuMountType, processPath)
   136  		out, err = cm.Exec(ctx, daemon, cmd, c.KubeCli)
   137  		if err != nil {
   138  			return errors.Wrap(err, fmt.Sprintf("run command %s failed", cmd))
   139  		}
   140  		period, err := strconv.Atoi(strings.TrimSuffix(string(out), "\n"))
   141  		if err != nil {
   142  			return errors.Wrap(err, "could not get cpu.cfs_period_us")
   143  		}
   144  		itemResult := cm.ItemResult{Name: "cpu.cfs_period_us", Value: string(out)}
   145  
   146  		if quota == -1 {
   147  			itemResult.Status = cm.ItemFailure
   148  			itemResult.ErrInfo = "no cpu limit is set for now"
   149  		} else {
   150  			itemResult.Status = cm.ItemSuccess
   151  			itemResult.SucInfo = fmt.Sprintf("cpu limit is equals to %.2f", float64(quota)/float64(period))
   152  		}
   153  		result.Items = append(result.Items, itemResult)
   154  	} else {
   155  		cmd = fmt.Sprintf("cat /sys/fs/cgroup/memory/%s/memory.limit_in_bytes", processPath)
   156  		out, err = cm.Exec(ctx, daemon, cmd, c.KubeCli)
   157  		if err != nil {
   158  			return errors.Wrap(err, fmt.Sprintf("run command %s failed", cmd))
   159  		}
   160  		limit, err := strconv.ParseUint(strings.TrimSuffix(string(out), "\n"), 10, 64)
   161  		if err != nil {
   162  			return errors.Wrap(err, "could not get memory.limit_in_bytes")
   163  		}
   164  		result.Items = append(result.Items, cm.ItemResult{Name: "memory.limit_in_bytes", Value: bytefmt.ByteSize(limit) + "B"})
   165  	}
   166  	return nil
   167  }
   168