chaos-controller-manager.go

Documentation: github.com/chaos-mesh/chaos-mesh/pkg/metrics

     1  // Copyright 2021 Chaos Mesh Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  // http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  
    16  package metrics
    17  
    18  import (
    19  	"context"
    20  	"reflect"
    21  
    22  	"github.com/go-logr/logr"
    23  	"github.com/prometheus/client_golang/prometheus"
    24  	ctrl "sigs.k8s.io/controller-runtime"
    25  	"sigs.k8s.io/controller-runtime/pkg/cache"
    26  
    27  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    28  	"github.com/chaos-mesh/chaos-mesh/pkg/status"
    29  )
    30  
    31  const (
    32  	// chaosControllerManagerMetricsSubsystem is the subsystem name for chaos controller manager metrics
    33  	chaosControllerManagerMetricsSubsystem = "chaos_controller_manager"
    34  )
    35  
    36  // ChaosControllerManagerMetricsCollector implements prometheus.Collector interface
    37  type ChaosControllerManagerMetricsCollector struct {
    38  	logger              logr.Logger
    39  	store               cache.Cache
    40  	chaosExperiments    *prometheus.GaugeVec
    41  	SidecarTemplates    prometheus.Gauge
    42  	ConfigTemplates     *prometheus.GaugeVec
    43  	InjectionConfigs    *prometheus.GaugeVec
    44  	TemplateNotExist    *prometheus.CounterVec
    45  	TemplateLoadError   prometheus.Counter
    46  	ConfigNameDuplicate *prometheus.CounterVec
    47  	InjectRequired      *prometheus.CounterVec
    48  	Injections          *prometheus.CounterVec
    49  	chaosSchedules      *prometheus.GaugeVec
    50  	chaosWorkflows      *prometheus.GaugeVec
    51  	EmittedEvents       *prometheus.CounterVec
    52  }
    53  
    54  // NewChaosControllerManagerMetricsCollector initializes metrics and collector
    55  func NewChaosControllerManagerMetricsCollector(manager ctrl.Manager, registerer *prometheus.Registry, logger logr.Logger) *ChaosControllerManagerMetricsCollector {
    56  	var store cache.Cache
    57  	if manager != nil {
    58  		store = manager.GetCache()
    59  	}
    60  
    61  	c := &ChaosControllerManagerMetricsCollector{
    62  		logger: logger,
    63  		store:  store,
    64  		chaosExperiments: prometheus.NewGaugeVec(prometheus.GaugeOpts{
    65  			Subsystem: chaosControllerManagerMetricsSubsystem,
    66  			Name:      "chaos_experiments",
    67  			Help:      "Total number of chaos experiments and their phases",
    68  		}, []string{"namespace", "kind", "phase"}),
    69  		SidecarTemplates: prometheus.NewGauge(prometheus.GaugeOpts{
    70  			Subsystem: chaosControllerManagerMetricsSubsystem,
    71  			Name:      "chaos_mesh_templates",
    72  			Help:      "Total number of injection templates",
    73  		}),
    74  		ConfigTemplates: prometheus.NewGaugeVec(prometheus.GaugeOpts{
    75  			Subsystem: chaosControllerManagerMetricsSubsystem,
    76  			Name:      "chaos_mesh_config_templates",
    77  			Help:      "Total number of config templates",
    78  		}, []string{"namespace", "template"}),
    79  		InjectionConfigs: prometheus.NewGaugeVec(prometheus.GaugeOpts{
    80  			Subsystem: chaosControllerManagerMetricsSubsystem,
    81  			Name:      "chaos_mesh_injection_configs",
    82  			Help:      "Total number of injection configs",
    83  		}, []string{"namespace", "template"}),
    84  		TemplateNotExist: prometheus.NewCounterVec(prometheus.CounterOpts{
    85  			Subsystem: chaosControllerManagerMetricsSubsystem,
    86  			Name:      "chaos_mesh_template_not_exist_total",
    87  			Help:      "Total number of template not exist error",
    88  		}, []string{"namespace", "template"}),
    89  		ConfigNameDuplicate: prometheus.NewCounterVec(prometheus.CounterOpts{
    90  			Subsystem: chaosControllerManagerMetricsSubsystem,
    91  			Name:      "chaos_mesh_config_name_duplicate_total",
    92  			Help:      "Total number of config name duplication error",
    93  		}, []string{"namespace", "config"}),
    94  		TemplateLoadError: prometheus.NewCounter(prometheus.CounterOpts{
    95  			Subsystem: chaosControllerManagerMetricsSubsystem,
    96  			Name:      "chaos_mesh_template_load_failed_total",
    97  			Help:      "Total number of failures when rendering config args to template",
    98  		}),
    99  		InjectRequired: prometheus.NewCounterVec(prometheus.CounterOpts{
   100  			Subsystem: chaosControllerManagerMetricsSubsystem,
   101  			Name:      "chaos_mesh_inject_required_total",
   102  			Help:      "Total number of injections required",
   103  		}, []string{"namespace", "config"}),
   104  		Injections: prometheus.NewCounterVec(prometheus.CounterOpts{
   105  			Name: "chaos_mesh_injections_total",
   106  			Help: "Total number of sidecar injections performed on the webhook",
   107  		}, []string{"namespace", "config"}),
   108  		chaosSchedules: prometheus.NewGaugeVec(prometheus.GaugeOpts{
   109  			Subsystem: chaosControllerManagerMetricsSubsystem,
   110  			Name:      "chaos_schedules",
   111  			Help:      "Total number of chaos schedules",
   112  		}, []string{"namespace"}),
   113  		chaosWorkflows: prometheus.NewGaugeVec(prometheus.GaugeOpts{
   114  			Subsystem: chaosControllerManagerMetricsSubsystem,
   115  			Name:      "chaos_workflows",
   116  			Help:      "Total number of chaos workflows",
   117  		}, []string{"namespace"}),
   118  		EmittedEvents: prometheus.NewCounterVec(prometheus.CounterOpts{
   119  			Subsystem: chaosControllerManagerMetricsSubsystem,
   120  			Name:      "emitted_event_total",
   121  			Help:      "Total number of the emitted event by chaos-controller-manager",
   122  		}, []string{"type", "reason", "namespace"}),
   123  	}
   124  
   125  	if registerer != nil {
   126  		registerer.MustRegister(c)
   127  	}
   128  
   129  	return c
   130  }
   131  
   132  // Describe implements the prometheus.Collector interface.
   133  func (collector *ChaosControllerManagerMetricsCollector) Describe(ch chan<- *prometheus.Desc) {
   134  	collector.chaosExperiments.Describe(ch)
   135  	collector.SidecarTemplates.Describe(ch)
   136  	collector.ConfigTemplates.Describe(ch)
   137  	collector.InjectionConfigs.Describe(ch)
   138  	collector.TemplateNotExist.Describe(ch)
   139  	collector.ConfigNameDuplicate.Describe(ch)
   140  	collector.TemplateLoadError.Describe(ch)
   141  	collector.InjectRequired.Describe(ch)
   142  	collector.Injections.Describe(ch)
   143  	collector.EmittedEvents.Describe(ch)
   144  	collector.chaosSchedules.Describe(ch)
   145  	collector.chaosWorkflows.Describe(ch)
   146  }
   147  
   148  // Collect implements the prometheus.Collector interface.
   149  func (collector *ChaosControllerManagerMetricsCollector) Collect(ch chan<- prometheus.Metric) {
   150  	collector.collectChaosExperiments()
   151  	collector.collectChaosSchedules()
   152  	collector.collectChaosWorkflows()
   153  	collector.SidecarTemplates.Collect(ch)
   154  	collector.ConfigTemplates.Collect(ch)
   155  	collector.InjectionConfigs.Collect(ch)
   156  	collector.TemplateNotExist.Collect(ch)
   157  	collector.ConfigNameDuplicate.Collect(ch)
   158  	collector.TemplateLoadError.Collect(ch)
   159  	collector.InjectRequired.Collect(ch)
   160  	collector.Injections.Collect(ch)
   161  	collector.chaosExperiments.Collect(ch)
   162  	collector.chaosSchedules.Collect(ch)
   163  	collector.chaosWorkflows.Collect(ch)
   164  	collector.EmittedEvents.Collect(ch)
   165  }
   166  
   167  func (collector *ChaosControllerManagerMetricsCollector) collectChaosExperiments() {
   168  	// TODO(yeya24) if there is an error in List
   169  	// the experiment status will be lost
   170  	collector.chaosExperiments.Reset()
   171  
   172  	for kind, obj := range v1alpha1.AllKinds() {
   173  		expCache := map[string]map[string]int{}
   174  		chaosList := obj.SpawnList()
   175  		if err := collector.store.List(context.TODO(), chaosList); err != nil {
   176  			collector.logger.Error(err, "failed to list chaos", "kind", kind)
   177  			return
   178  		}
   179  
   180  		items := chaosList.GetItems()
   181  		for _, item := range items {
   182  			if _, ok := expCache[item.GetNamespace()]; !ok {
   183  				// There is only 4 supported phases
   184  				expCache[item.GetNamespace()] = make(map[string]int, 4)
   185  			}
   186  			innerObject := reflect.ValueOf(item).Interface().(v1alpha1.InnerObject)
   187  			expCache[item.GetNamespace()][string(status.GetChaosStatus(innerObject))]++
   188  		}
   189  
   190  		for ns, v := range expCache {
   191  			for phase, count := range v {
   192  				collector.chaosExperiments.WithLabelValues(ns, kind, phase).Set(float64(count))
   193  			}
   194  		}
   195  	}
   196  }
   197  
   198  func (collector *ChaosControllerManagerMetricsCollector) collectChaosSchedules() {
   199  	collector.chaosSchedules.Reset()
   200  
   201  	schedules := &v1alpha1.ScheduleList{}
   202  	if err := collector.store.List(context.TODO(), schedules); err != nil {
   203  		collector.logger.Error(err, "failed to list schedules")
   204  		return
   205  	}
   206  
   207  	countByNamespace := make(map[string]int)
   208  	items := schedules.GetItems()
   209  	for _, item := range items {
   210  		countByNamespace[item.GetNamespace()]++
   211  	}
   212  
   213  	for namespace, count := range countByNamespace {
   214  		collector.chaosSchedules.WithLabelValues(namespace).Set(float64(count))
   215  	}
   216  }
   217  
   218  func (collector *ChaosControllerManagerMetricsCollector) collectChaosWorkflows() {
   219  	collector.chaosWorkflows.Reset()
   220  
   221  	workflows := &v1alpha1.WorkflowList{}
   222  	if err := collector.store.List(context.TODO(), workflows); err != nil {
   223  		collector.logger.Error(err, "failed to list workflows")
   224  		return
   225  	}
   226  
   227  	countByNamespace := make(map[string]int)
   228  	items := workflows.GetItems()
   229  	for _, item := range items {
   230  		countByNamespace[item.GetNamespace()]++
   231  	}
   232  
   233  	for namespace, count := range countByNamespace {
   234  		collector.chaosWorkflows.WithLabelValues(namespace).Set(float64(count))
   235  	}
   236  }
   237