doc update

Author: vijay-stephen

README.md

@@ -35,9 +35,7 @@ The Observability Terraform Module is a comprehensive solution designed to simpl
 
 5. Signoz Community Edition Support
 - Adds native support for Signoz CE, an all-in-one observability platform.
-
 - Enables logs, metrics, and traces to be collected and correlated in one unified interface.
-
 - Simplifies tracing setup with OpenTelemetry Collector and works out of the box with distributed applications.
 
 For more information about this repository and its usage, please see [Terraform AWS ARC Observability Module Usage Guide](docs/module-usage-guide/README.md).
@@ -221,6 +219,7 @@ No providers.
 | <a name="module_jaeger"></a> [jaeger](#module\_jaeger) | ./modules/jaeger | n/a |
 | <a name="module_prometheus"></a> [prometheus](#module\_prometheus) | ./modules/prometheus | n/a |
 | <a name="module_signoz"></a> [signoz](#module\_signoz) | ./modules/signoz | n/a |
+| <a name="module_signoz_metrics_logs"></a> [signoz\_metrics\_logs](#module\_signoz\_metrics\_logs) | ./modules/signoz-infra | n/a |
 
 ## Resources
 
@@ -239,7 +238,8 @@ No resources.
 | <a name="input_namespace"></a> [namespace](#input\_namespace) | Namespace for the resources. | `string` | n/a | yes |
 | <a name="input_prometheus_config"></a> [prometheus\_config](#input\_prometheus\_config) | Configuration settings for deploying Prometheus | <pre>object({<br>    name = optional(string, "prometheus")<br>    k8s_namespace = object({<br>      name   = optional(string, "metrics")<br>      create = optional(bool, true)<br>    })<br>    log_level                 = optional(string, "info")<br>    replica_count             = optional(number, 1)<br>    storage                   = optional(string, "8Gi")<br>    storage_class             = optional(string, "gp2")<br>    enable_kube_state_metrics = optional(bool, true)<br>    enable_node_exporter      = optional(bool, true)<br>    cpu_limit                 = optional(string, "100m")<br>    memory_limit              = optional(string, "512Mi")<br>    cpu_request               = optional(string, "100m")<br>    memory_request            = optional(string, "128Mi")<br>    retention_period          = optional(string, "15d")<br><br>    grafana_config = object({<br>      name                = optional(string, "grafana")<br>      replica_count       = optional(number, 1)<br>      ingress_enabled     = optional(bool, false)<br>      lb_visibility       = optional(string, "internet-facing") # Options: "internal" or "internet-facing"<br>      aws_certificate_arn = optional(string, "")<br>      ingress_host        = optional(string, "")<br>      admin_user          = optional(string, "admin")<br>      cpu_limit           = optional(string, "100m")<br>      memory_limit        = optional(string, "128Mi")<br>      cpu_request         = optional(string, "100m")<br>      memory_request      = optional(string, "128Mi")<br>      dashboard_list = optional(list(object({<br>        name = string<br>        json = string<br>      })), [])<br>    })<br><br>    blackbox_exporter_config = object({<br>      name           = optional(string, "blackbox-exporter")<br>      replica_count  = optional(number, 1)<br>      cpu_limit      = optional(string, "100m")<br>      memory_limit   = optional(string, "500Mi")<br>      cpu_request    = optional(string, "100m")<br>      memory_request = optional(string, "50Mi")<br>      monitoring_targets = list(object({<br>        name                     = string                         # Target name (e.g., google)<br>        url                      = string                         # URL to monitor (e.g., https://google.com)<br>        scrape_interval          = optional(string, "60s")        # Scrape interval (e.g., 60s)<br>        scrape_timeout           = optional(string, "60s")        # Scrape timeout (e.g., 60s)<br>        status_code_pattern_list = optional(string, "[http_2xx]") # Blackbox module to use (e.g., http_2xx)<br>      }))<br>    })<br><br>    alertmanager_config = object({<br>      name                        = optional(string, "alertmanager")<br>      replica_count               = optional(number, 1)<br>      cpu_limit                   = optional(string, "100m")<br>      memory_limit                = optional(string, "128Mi")<br>      cpu_request                 = optional(string, "10m")<br>      memory_request              = optional(string, "32Mi")<br>      custom_alerts               = optional(string, "")<br>      alert_notification_settings = optional(string, "")<br>    })<br>  })</pre> | <pre>{<br>  "alertmanager_config": {<br>    "name": "alertmanager"<br>  },<br>  "blackbox_exporter_config": {<br>    "monitoring_targets": [],<br>    "name": "blackbox-exporter"<br>  },<br>  "enable_kube_state_metrics": true,<br>  "enable_node_exporter": true,<br>  "grafana_config": {<br>    "admin_user": "admin",<br>    "ingress_enabled": false,<br>    "lb_visibility": "internet-facing",<br>    "prometheus_endpoint": "prometheus"<br>  },<br>  "k8s_namespace": {<br>    "create": true,<br>    "name": "metrics"<br>  },<br>  "log_level": "info",<br>  "replica_count": 1,<br>  "resources": {<br>    "cpu_limit": "100m",<br>    "cpu_request": "100m",<br>    "memory_limit": "512Mi",<br>    "memory_request": "128Mi"<br>  },<br>  "retention_period": "15d",<br>  "storage": "8Gi"<br>}</pre> | no |
 | <a name="input_search_engine"></a> [search\_engine](#input\_search\_engine) | (optional) Search engine for logs | `string` | `null` | no |
-| <a name="input_signoz_config"></a> [signoz\_config](#input\_signoz\_config) | Configuration for observability components in the monitoring stack. This variable encapsulates<br>settings for the following components:<br><br>- ClickHouse:<br>  Used as the backend storage engine for observability data (like traces and metrics).<br>  Includes credentials and resource limits/requests for tuning performance.<br><br>- SigNoz:<br>  Provides the UI and analytics for monitoring and tracing applications.<br>  Includes ingress setup and compute resource configuration.<br><br>- Alertmanager:<br>  Handles alerting rules and notifications for monitoring data.<br>  Includes configuration for storage, scaling, and ingress settings.<br><br>- OTEL Collector:<br>  Collects telemetry data (logs, metrics, traces) from the applications and<br>  routes it to appropriate backends.<br>  Includes resource definitions and optional ingress configuration.<br><br>This structure enables centralized management of observability stack deployment in Kubernetes<br>via Terraform. | <pre>object({<br>    k8s_namespace = object({<br>      name   = optional(string, "signoz")<br>      create = optional(bool, false)<br>    })<br>    name          = string<br>    storage_class = optional(string, "gp3")<br>    cluster_name  = string<br>    clickhouse = optional(object({<br>      user           = optional(string, "admin")<br>      cpu_limit      = optional(string, "2000m")<br>      memory_limit   = optional(string, "4Gi")<br>      cpu_request    = optional(string, "100m")<br>      memory_request = optional(string, "200Mi")<br>      storage        = optional(string, "20Gi")<br>    }))<br><br>    signoz_bin = optional(object({<br>      replica_count              = optional(number, 1)<br>      cpu_limit                  = optional(string, "750m")<br>      memory_limit               = optional(string, "1000Mi")<br>      cpu_request                = optional(string, "100m")<br>      memory_request             = optional(string, "200Mi")<br>      ingress_enabled            = optional(bool, false)<br>      aws_certificate_arn        = optional(string, null)<br>      domain                     = string<br>      root_domain                = optional(string, null)              // if root domain is provided, it creates DNS record<br>      lb_visibility              = optional(string, "internet-facing") # Options: "internal" or "internet-facing"<br>      metric_collection_interval = optional(string, "30s")<br>    }))<br><br>    alertmanager = optional(object({<br>      enable              = optional(bool, false)<br>      replica_count       = optional(number, 1)<br>      cpu_limit           = optional(string, "750m")<br>      memory_limit        = optional(string, "1000Mi")<br>      cpu_request         = optional(string, "100m")<br>      memory_request      = optional(string, "200Mi")<br>      storage             = optional(string, "100Mi")<br>      enable_ingress      = optional(bool, false)<br>      aws_certificate_arn = optional(string, null)<br>      domain              = optional(string, "signoz.example.com")<br>    }))<br><br>    otel_collector = optional(object({<br>      cpu_limit           = optional(string, "1")<br>      memory_limit        = optional(string, "2Gi")<br>      cpu_request         = optional(string, "100m")<br>      memory_request      = optional(string, "200Mi")<br>      storage             = optional(string, "100Mi")<br>      enable_ingress      = optional(bool, false)<br>      aws_certificate_arn = optional(string, null)<br>      domain              = optional(string, "signoz.example.com")<br>    }))<br>  })</pre> | <pre>{<br>  "cluster_name": null,<br>  "k8s_namespace": {<br>    "create": true,<br>    "name": "signoz"<br>  },<br>  "name": null<br>}</pre> | no |
+| <a name="input_signoz_config"></a> [signoz\_config](#input\_signoz\_config) | Configuration for observability components in the monitoring stack. This variable encapsulates<br>settings for the following components:<br><br>- ClickHouse:<br>  Used as the backend storage engine for observability data (like traces and metrics).<br>  Includes credentials and resource limits/requests for tuning performance.<br><br>- SigNoz:<br>  Provides the UI and analytics for monitoring and tracing applications.<br>  Includes ingress setup and compute resource configuration.<br><br>- Alertmanager:<br>  Handles alerting rules and notifications for monitoring data.<br>  Includes configuration for storage, scaling, and ingress settings.<br><br>- OTEL Collector:<br>  Collects telemetry data (logs, metrics, traces) from the applications and<br>  routes it to appropriate backends.<br>  Includes resource definitions and optional ingress configuration.<br><br>This structure enables centralized management of observability stack deployment in Kubernetes<br>via Terraform. | <pre>object({<br>    k8s_namespace = object({<br>      name   = optional(string, "signoz")<br>      create = optional(bool, false)<br>    })<br>    name          = optional(string, "signoz")<br>    storage_class = optional(string, "gp3")<br>    cluster_name  = string<br>    clickhouse = optional(object({<br>      user           = optional(string, "admin")<br>      cpu_limit      = optional(string, "2000m")<br>      memory_limit   = optional(string, "4Gi")<br>      cpu_request    = optional(string, "100m")<br>      memory_request = optional(string, "200Mi")<br>      storage        = optional(string, "20Gi")<br>    }))<br><br>    signoz_bin = optional(object({<br>      replica_count       = optional(number, 1)<br>      cpu_limit           = optional(string, "750m")<br>      memory_limit        = optional(string, "1000Mi")<br>      cpu_request         = optional(string, "100m")<br>      memory_request      = optional(string, "200Mi")<br>      ingress_enabled     = optional(bool, false)<br>      aws_certificate_arn = optional(string, null)<br>      domain              = string<br>      root_domain         = optional(string, null)              // if root domain is provided, it creates DNS record<br>      lb_visibility       = optional(string, "internet-facing") # Options: "internal" or "internet-facing"<br>    }))<br><br>    alertmanager = optional(object({<br>      enable              = optional(bool, false)<br>      replica_count       = optional(number, 1)<br>      cpu_limit           = optional(string, "750m")<br>      memory_limit        = optional(string, "1000Mi")<br>      cpu_request         = optional(string, "100m")<br>      memory_request      = optional(string, "200Mi")<br>      storage             = optional(string, "100Mi")<br>      enable_ingress      = optional(bool, false)<br>      aws_certificate_arn = optional(string, null)<br>      domain              = optional(string, "signoz.example.com")<br>    }))<br><br>    otel_collector = optional(object({<br>      cpu_limit           = optional(string, "1")<br>      memory_limit        = optional(string, "2Gi")<br>      cpu_request         = optional(string, "100m")<br>      memory_request      = optional(string, "200Mi")<br>      storage             = optional(string, "100Mi")<br>      enable_ingress      = optional(bool, false)<br>      aws_certificate_arn = optional(string, null)<br>      domain              = optional(string, "signoz.example.com")<br>    }))<br>  })</pre> | <pre>{<br>  "cluster_name": null,<br>  "k8s_namespace": {<br>    "create": true,<br>    "name": "signoz"<br>  },<br>  "name": null<br>}</pre> | no |
+| <a name="input_signoz_infra_monitor_config"></a> [signoz\_infra\_monitor\_config](#input\_signoz\_infra\_monitor\_config) | Configuration object for deploying SigNoz infrastructure monitoring components.<br><br>Attributes:<br>- name: A name identifier for the monitoring deployment (used in naming resources).<br>- storage\_class: (Optional) The Kubernetes storage class to be used for persistent volumes. Defaults to "gp3".<br>- cluster\_name: The name of the Kubernetes cluster where SigNoz is being deployed.<br>- otel\_collector\_endpoint: The endpoint URL for the OpenTelemetry Collector to which metrics, logs, and traces will be exported.<br>- metric\_collection\_interval: (Optional) The interval at which metrics are collected. Defaults to "30s".<br>- if any one ofr the values enable\_log\_collection,enable\_metrics\_collection is true, then helm chart gets installed<br><br>This variable is used to centralize configuration related to monitoring infrastructure via SigNoz. | <pre>object({<br>    k8s_namespace = optional(object({<br>      name   = optional(string, "signoz")<br>      create = optional(bool, false)<br>    }))<br>    name                       = string<br>    storage_class              = optional(string, "gp3")<br>    cluster_name               = string<br>    enable_log_collection      = optional(bool, false)<br>    enable_metrics_collection  = optional(bool, false)<br>    otel_collector_endpoint    = optional(string, null)<br>    metric_collection_interval = optional(string, "30s")<br>  })</pre> | <pre>{<br>  "cluster_name": null,<br>  "name": null<br>}</pre> | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | (optional) Tags for AWS resources | `map(string)` | `{}` | no |
 | <a name="input_tracing_stack"></a> [tracing\_stack](#input\_tracing\_stack) | (optional) Distributed tracing stack | `string` | `null` | no |
 
@@ -249,6 +249,7 @@ No resources.
 |------|-------------|
 | <a name="output_grafana_lb_dns"></a> [grafana\_lb\_dns](#output\_grafana\_lb\_dns) | Grafana ingress loadbalancer DNS |
 | <a name="output_kibana_lb_dns"></a> [kibana\_lb\_dns](#output\_kibana\_lb\_dns) | Kibana ingress loadbalancer DNS |
+| <a name="output_otel_collector_endpoint"></a> [otel\_collector\_endpoint](#output\_otel\_collector\_endpoint) | OTEL collector endpoint |
 | <a name="output_signoz_lb_dns"></a> [signoz\_lb\_dns](#output\_signoz\_lb\_dns) | Signoz ingress loadbalancer DNS |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 

docs/module-usage-guide/README.md

@@ -86,6 +86,7 @@ For basic usage, see the [example](https://github.com/sourcefuse/terraform-aws-a
 
 - Users can choose their preferred search engine between AWS OpenSearch and Elasticsearch.
 - For log aggregation, users have the flexibility to select either Fluentd or FluentBit based on their requirements.
+- While destroying Signoz , ClickHouse is not automatically removed and must be deleted manually.
 
 ## Troubleshooting
 

examples/signoz/README.md

@@ -23,6 +23,7 @@
 
 | Name | Source | Version |
 |------|--------|---------|
+| <a name="module_logs_metrics"></a> [logs\_metrics](#module\_logs\_metrics) | ../../ | n/a |
 | <a name="module_signoz"></a> [signoz](#module\_signoz) | ../../ | n/a |
 
 ## Resources
@@ -45,5 +46,6 @@
 
 | Name | Description |
 |------|-------------|
+| <a name="output_otel_collector_endpoint"></a> [otel\_collector\_endpoint](#output\_otel\_collector\_endpoint) | OTEL collector endpoint |
 | <a name="output_signoz_lb_arn"></a> [signoz\_lb\_arn](#output\_signoz\_lb\_arn) | Signoz ingress loadbalancer DNS |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/signoz/locals.tf

@@ -2,7 +2,7 @@ data "aws_caller_identity" "current" {}
 
 locals {
   signoz_config = {
-    name          = "signoz-monitoring"
+    name          = "signoz"
     storage_class = "gp3"
     cluster_name  = data.aws_eks_cluster.this.name
 
@@ -51,4 +51,21 @@ locals {
       storage        = "200Mi"
     }
   }
+
+
+  metrics_logs_config = {
+    name          = "signoz"
+    storage_class = "gp3"
+    cluster_name  = data.aws_eks_cluster.this.name
+
+    k8s_namespace = {
+      name   = "signoz"
+      create = false
+    }
+    enable_log_collection      = true
+    enable_metrics_collection  = true
+    otel_collector_endpoint    = module.signoz.otel_collector_endpoint
+    metric_collection_interval = "30s"
+
+  }
 }

examples/signoz/main.tf

@@ -58,17 +58,30 @@ provider "aws" {
   region = var.region
 }
 
+// Deploy Signoz stack with Clickhouse , OTEL collector etc
 module "signoz" {
   source = "../../"
 
   environment = var.environment
   namespace   = var.namespace
 
 
-  search_engine             = "signoz"
+  search_engine = "signoz-clickhouse"
+  signoz_config = local.signoz_config
+}
+
+// Deploy metrics and log collection agents
+module "logs_metrics" {
+  source = "../../"
+
+  environment = var.environment
+  namespace   = var.namespace
+
   log_aggregator            = "signoz"
   tracing_stack             = "signoz"
   metrics_monitoring_system = "signoz"
 
-  signoz_config = local.signoz_config
+  signoz_infra_monitor_config = local.metrics_logs_config
+
+  depends_on = [module.signoz]
 }

examples/signoz/outputs.tf

@@ -2,3 +2,8 @@ output "signoz_lb_arn" {
   description = "Signoz ingress loadbalancer DNS"
   value       = module.signoz.signoz_lb_dns
 }
+
+output "otel_collector_endpoint" {
+  description = "OTEL collector endpoint"
+  value       = module.signoz.otel_collector_endpoint
+}

main.tf

@@ -131,21 +131,35 @@ module "jaeger" {
 
 }
 
+// Deploy Signoz stack with Clickhouse , OTEL collector etc
 module "signoz" {
   source = "./modules/signoz"
 
   count = (
-    var.search_engine == "signoz" ||
-    var.log_aggregator == "signoz" ||
-    var.metrics_monitoring_system == "signoz" ||
-    var.tracing_stack == "signoz"
+    var.search_engine == "signoz-clickhouse"
   ) ? 1 : 0
 
 
   k8s_namespace        = var.signoz_config.k8s_namespace.name
   create_k8s_namespace = var.signoz_config.k8s_namespace.create
-  environment          = var.environment
 
   signoz_config = var.signoz_config
+}
+
+// Deploy metrics and log collection agents
+module "signoz_metrics_logs" {
+  source = "./modules/signoz-infra"
+
+  count = (
+    var.log_aggregator == "signoz" ||
+    var.metrics_monitoring_system == "signoz"
+  ) ? 1 : 0
+
+
+  k8s_namespace           = var.signoz_infra_monitor_config.k8s_namespace.name
+  create_k8s_namespace    = var.signoz_infra_monitor_config.k8s_namespace.create
+  environment             = var.environment
+  otel_collector_endpoint = var.signoz_config.cluster_name != null && var.signoz_infra_monitor_config.otel_collector_endpoint == null ? "http://${var.signoz_config.name}-otel-collector:4317" : var.signoz_infra_monitor_config.otel_collector_endpoint
 
+  signoz_infra_monitor_config = var.signoz_infra_monitor_config
 }

modules/elasticsearch/README copy.md

@@ -0,0 +1,2 @@
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->