From 20e5bcd1e73c2b0ec477609a165d2f94d417b30f Mon Sep 17 00:00:00 2001 From: Rishab87 Date: Sat, 23 Aug 2025 00:08:04 +0530 Subject: [PATCH] added resource metric as config --- internal/store/builder.go | 23 ++- pkg/app/server.go | 61 ++++++++ .../custom_resource_metrics.go | 13 ++ pkg/options/options.go | 5 + pkg/resourcestate/factory.go | 144 ++++++++++++++++++ pkg/resourcestate/resource_metrics_config.go | 32 ++++ tests/manifests/resouce-metrics-config.yaml | 31 ++++ 7 files changed, 307 insertions(+), 2 deletions(-) create mode 100644 pkg/resourcestate/factory.go create mode 100644 pkg/resourcestate/resource_metrics_config.go create mode 100644 tests/manifests/resouce-metrics-config.yaml diff --git a/internal/store/builder.go b/internal/store/builder.go index 8cdf11dcf2..b7ab4a30a0 100644 --- a/internal/store/builder.go +++ b/internal/store/builder.go @@ -215,10 +215,29 @@ func (b *Builder) WithCustomResourceStoreFactories(fs ...customresource.Registry } else { gvrString = f.Name() } - if _, ok := availableStores[gvrString]; ok { + + key := gvrString + if _, ok := availableStores[f.Name()]; ok { + key = f.Name() + klog.InfoS("Overriding core resource store with custom factory", "resource", key) + } else if _, ok := availableStores[gvrString]; ok { klog.InfoS("Updating store", "GVR", gvrString) } - availableStores[gvrString] = func(b *Builder) []cache.Store { + + availableStores[key] = func(b *Builder) []cache.Store { + klog.InfoS("Building custom resource store", "gvrString", gvrString, "clientExists", b.customResourceClients[gvrString] != nil) + client := b.customResourceClients[gvrString] + klog.InfoS("About to call buildCustomResourceStoresFunc", + "factoryName", f.Name(), + "clientIsNil", client == nil, + "expectedType", f.ExpectedType(), + "metricsCount", len(f.MetricFamilyGenerators()), + ) + + // Test ListWatch before calling buildCustomResourceStoresFunc + lw := f.ListWatch(client, "", "") + klog.InfoS("ListWatch result", "isNil", lw == nil) + return b.buildCustomResourceStoresFunc( f.Name(), f.MetricFamilyGenerators(), diff --git a/pkg/app/server.go b/pkg/app/server.go index 6d04e873c8..932f15812f 100644 --- a/pkg/app/server.go +++ b/pkg/app/server.go @@ -61,6 +61,8 @@ import ( "k8s.io/kube-state-metrics/v2/pkg/options" "k8s.io/kube-state-metrics/v2/pkg/util" "k8s.io/kube-state-metrics/v2/pkg/util/proc" + + "k8s.io/kube-state-metrics/v2/pkg/resourcestate" ) const ( @@ -70,6 +72,19 @@ const ( readyzPath = "/readyz" ) +type dropDefaultsFilter struct { + overridden map[string]struct{} +} + +func (d dropDefaultsFilter) Test(fg generator.FamilyGenerator) bool { + for res := range d.overridden { + if strings.HasPrefix(fg.Name, "kube_"+res+"_") { + return false + } + } + return true +} + // RunKubeStateMetricsWrapper runs KSM with context cancellation. func RunKubeStateMetricsWrapper(ctx context.Context, opts *options.Options) error { err := RunKubeStateMetrics(ctx, opts) @@ -268,6 +283,52 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error { return fmt.Errorf("failed to create client: %v", err) } storeBuilder.WithKubeClient(kubeClient) + storeBuilder.WithGenerateCustomResourceStoresFunc(storeBuilder.DefaultGenerateCustomResourceStoresFunc()) + + if opts.ResourceMetricsConfigFile != "" { + cfg, err := resourcestate.LoadConfig(opts.ResourceMetricsConfigFile) + if err != nil { + return fmt.Errorf("failed to load ResourceMetricsConfig: %w", err) + } + + factories, err := resourcestate.BuildFactoriesFromConfig(cfg) + if err != nil { + return fmt.Errorf("failed to compile ResourceMetricsConfig: %w", err) + } + + crClients := make(map[string]interface{}) + for _, f := range factories { + klog.InfoS("Processing factory", "factoryName", f.Name()) + + // Use only util.GVRFromType since that's what the builder expects + utilGVR, err := util.GVRFromType(f.Name(), f.ExpectedType()) + if err != nil { + klog.ErrorS(err, "Failed to get GVR from type", "resourceName", f.Name()) + continue + } + + gvrString := utilGVR.String() + crClients[gvrString] = kubeClient + klog.InfoS("Registering CR client", "factoryName", f.Name(), "gvrKey", gvrString) + } + + klog.InfoS("Total registered clients", "count", len(crClients)) + + storeBuilder.WithCustomResourceClients(crClients) + storeBuilder.WithCustomResourceStoreFactories(factories...) + + if opts.DisableDefaultCoreMetrics { + overridden := map[string]struct{}{} + for _, f := range factories { + overridden[f.Name()] = struct{}{} + } + storeBuilder.WithFamilyGeneratorFilter(generator.NewCompositeFamilyGeneratorFilter( + allowDenyList, + optInMetricFamilyFilter, + dropDefaultsFilter{overridden: overridden}, + )) + } + } storeBuilder.WithSharding(opts.Shard, opts.TotalShards) if err := storeBuilder.WithAllowAnnotations(opts.AnnotationsAllowList); err != nil { diff --git a/pkg/customresourcestate/custom_resource_metrics.go b/pkg/customresourcestate/custom_resource_metrics.go index 9ca2700835..f6c1eb2604 100644 --- a/pkg/customresourcestate/custom_resource_metrics.go +++ b/pkg/customresourcestate/custom_resource_metrics.go @@ -33,6 +33,9 @@ import ( generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) +// CompiledFamily is an exported alias of the internal compiled family. +type CompiledFamily = compiledFamily + // customResourceMetrics is an implementation of the customresource.RegistryFactory // interface which provides metrics for custom resources defined in a configuration file. type customResourceMetrics struct { @@ -44,6 +47,16 @@ type customResourceMetrics struct { var _ customresource.RegistryFactory = &customResourceMetrics{} +// Compile exposes the internal compile(resource) to other packages. +func Compile(r Resource) ([]CompiledFamily, error) { + return compile(r) +} + +// FamGen exposes the internal famGen to other packages. +func FamGen(f CompiledFamily) generator.FamilyGenerator { + return famGen(f) +} + // NewCustomResourceMetrics creates a customresource.RegistryFactory from a configuration object. func NewCustomResourceMetrics(resource Resource) (customresource.RegistryFactory, error) { compiled, err := compile(resource) diff --git a/pkg/options/options.go b/pkg/options/options.go index 580cef6c76..ecd3f8584e 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -83,6 +83,9 @@ type Options struct { UseAPIServerCache bool `yaml:"use_api_server_cache"` ObjectLimit int64 `yaml:"object_limit"` AuthFilter bool `yaml:"auth_filter"` + + ResourceMetricsConfigFile string + DisableDefaultCoreMetrics bool } // GetConfigFile is the getter for --config value. @@ -159,6 +162,8 @@ func (o *Options) AddFlags(cmd *cobra.Command) { o.cmd.Flags().StringVar(&o.CustomResourceConfig, "custom-resource-state-config", "", "Inline Custom Resource State Metrics config YAML (experimental)") o.cmd.Flags().StringVar(&o.CustomResourceConfigFile, "custom-resource-state-config-file", "", "Path to a Custom Resource State Metrics config file (experimental)") o.cmd.Flags().BoolVar(&o.ContinueWithoutCustomResourceConfigFile, "continue-without-custom-resource-state-config-file", false, "If true, Kube-state-metrics continues to run even if the config file specified by --custom-resource-state-config-file is not present. This is useful for scenarios where config file is not provided at startup but is provided later, for e.g., via configmap. Kube-state-metrics will not exit with an error if the custom-resource-state-config file is not found, instead watches and reloads when it is created.") + o.cmd.Flags().StringVar(&o.ResourceMetricsConfigFile, "resource-metrics-config-file", "", "Path to ResourceMetricsConfig YAML for core resources") + o.cmd.Flags().BoolVar(&o.DisableDefaultCoreMetrics, "disable-default-core-metrics", false, "Disable built-in core metric families; only config-defined ones will be used") o.cmd.Flags().StringVar(&o.Host, "host", "::", `Host to expose metrics on.`) o.cmd.Flags().StringVar(&o.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig file") o.cmd.Flags().StringVar(&o.Namespace, "pod-namespace", "", "Name of the namespace of the pod specified by --pod. "+autoshardingNotice) diff --git a/pkg/resourcestate/factory.go b/pkg/resourcestate/factory.go new file mode 100644 index 0000000000..bff928442a --- /dev/null +++ b/pkg/resourcestate/factory.go @@ -0,0 +1,144 @@ +package resourcestate + +import ( + "context" + "fmt" + klog "k8s.io/klog/v2" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + + cr "k8s.io/kube-state-metrics/v2/pkg/customresource" + crs "k8s.io/kube-state-metrics/v2/pkg/customresourcestate" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/kube-state-metrics/v2/pkg/metric" +) + +type coreFactory struct { + name string + kind string + gvk metav1.GroupVersionKind // Add this field + fams []crs.CompiledFamily +} + +var _ cr.RegistryFactory = (*coreFactory)(nil) + +func (f *coreFactory) Name() string { return f.name } + +func (f *coreFactory) CreateClient(cfg *rest.Config) (interface{}, error) { + return kubernetes.NewForConfig(cfg) +} + +func (f *coreFactory) MetricFamilyGenerators() []generator.FamilyGenerator { + out := make([]generator.FamilyGenerator, 0, len(f.fams)) + for _, fam := range f.fams { + // Wrap the CRS generator to handle typed-to-unstructured conversion + crsGen := crs.FamGen(fam) + wrappedGen := generator.FamilyGenerator{ + Name: crsGen.Name, + Help: crsGen.Help, + Type: crsGen.Type, + DeprecatedVersion: crsGen.DeprecatedVersion, + StabilityLevel: crsGen.StabilityLevel, + OptIn: crsGen.OptIn, + GenerateFunc: func(obj interface{}) *metric.Family { + // Convert typed object to unstructured + unstructuredObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + klog.ErrorS(err, "Failed to convert to unstructured", "objType", fmt.Sprintf("%T", obj)) + return &metric.Family{} + } + + u := &unstructured.Unstructured{Object: unstructuredObj} + return crsGen.GenerateFunc(u) + }, + } + out = append(out, wrappedGen) + } + return out +} + +func (f *coreFactory) ExpectedType() interface{} { + switch f.kind { + case "Pod": + return &corev1.Pod{} + case "Deployment": + return &appsv1.Deployment{} + default: + return &corev1.Pod{} + } +} + +func (f *coreFactory) ListWatch(client interface{}, ns, fieldSelector string) cache.ListerWatcher { + cs := client.(kubernetes.Interface) + ctx := context.Background() + switch f.kind { + case "Pod": + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + opts.FieldSelector = fieldSelector + return cs.CoreV1().Pods(ns).List(ctx, opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + opts.FieldSelector = fieldSelector + return cs.CoreV1().Pods(ns).Watch(ctx, opts) + }, + } + case "Deployment": + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + opts.FieldSelector = fieldSelector + return cs.AppsV1().Deployments(ns).List(ctx, opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + opts.FieldSelector = fieldSelector + return cs.AppsV1().Deployments(ns).Watch(ctx, opts) + }, + } + default: + // Return a valid but empty ListWatch instead of nil + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + return &corev1.PodList{}, nil + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + return watch.NewEmptyWatch(), nil + }, + } + } +} + +func (f *coreFactory) GVRString() string { + return fmt.Sprintf("%s/%s, Resource=%s", f.gvk.Group, f.gvk.Version, strings.ToLower(f.name)) +} + +// BuildFactoriesFromConfig compiles the config into RegistryFactories. +func BuildFactoriesFromConfig(c *Config) ([]cr.RegistryFactory, error) { + var out []cr.RegistryFactory + for _, r := range c.Spec.Resources { + fams, err := crs.Compile(r) + if err != nil { + return nil, err + } + out = append(out, &coreFactory{ + name: r.GetResourceName(), + kind: r.GroupVersionKind.Kind, + gvk: metav1.GroupVersionKind{ + Group: r.GroupVersionKind.Group, + Version: r.GroupVersionKind.Version, + Kind: r.GroupVersionKind.Kind, + }, // Store the GVK + fams: fams, + }) + } + return out, nil +} diff --git a/pkg/resourcestate/resource_metrics_config.go b/pkg/resourcestate/resource_metrics_config.go new file mode 100644 index 0000000000..d00781c054 --- /dev/null +++ b/pkg/resourcestate/resource_metrics_config.go @@ -0,0 +1,32 @@ +package resourcestate + +import ( + "os" + + "gopkg.in/yaml.v3" + + crs "k8s.io/kube-state-metrics/v2/pkg/customresourcestate" +) + +// Config wraps a list of customresourcestate.Resource entries for core resources. +type Config struct { + Kind string `yaml:"kind"` // expect "ResourceMetricsConfig" + Spec struct { + Resources []crs.Resource `yaml:"resources"` + } `yaml:"spec"` +} + +func LoadConfig(path string) (*Config, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var c Config + if err := yaml.Unmarshal(b, &c); err != nil { + return nil, err + } + if c.Kind == "" { + c.Kind = "ResourceMetricsConfig" + } + return &c, nil +} diff --git a/tests/manifests/resouce-metrics-config.yaml b/tests/manifests/resouce-metrics-config.yaml new file mode 100644 index 0000000000..e372eb7259 --- /dev/null +++ b/tests/manifests/resouce-metrics-config.yaml @@ -0,0 +1,31 @@ +kind: ResourceMetricsConfig +spec: + resources: + - groupVersionKind: + group: "" + version: "v1" + kind: "Pod" + resourceName: "pods" + metricNamePrefix: "kube_pod" + metrics: + - name: "phase" + help: "Pod phase" + each: + type: stateset + stateSet: + labelName: "phase" + path: ["status","phase"] + list: ["Pending","Running","Succeeded","Failed","Unknown"] + - groupVersionKind: + group: "apps" + version: "v1" + kind: "Deployment" + resourceName: "deployments" + metricNamePrefix: "kube_deployment" + metrics: + - name: "spec_replicas" + help: "Desired replicas" + each: + type: gauge + gauge: + path: ["spec","replicas"] \ No newline at end of file