OpenTelemetry Setup: Observability That Grows With You

# docker/observability/otel-collector/config.signoz.yaml
# Extends base config for SigNoz backend

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:
    timeout: 5s
    send_batch_size: 1000

  memory_limiter:
    check_interval: 1s
    limit_mib: 512
    spike_limit_mib: 128

  resource:
    attributes:
      - key: environment
        value: ${ENVIRONMENT:-dev}
        action: upsert

exporters:
  otlp/signoz:
    endpoint: signoz-otel-collector:4317
    tls:
      insecure: true

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, batch, resource]
      exporters: [otlp/signoz]
    metrics:
      receivers: [otlp]
      processors: [memory_limiter, batch, resource]
      exporters: [otlp/signoz]
    logs:
      receivers: [otlp]
      processors: [memory_limiter, batch, resource]
      exporters: [otlp/signoz]

# docker/observability/docker-compose.signoz.yml
# SigNoz all-in-one for local development

services:
  signoz:
    image: signoz/signoz:latest
    container_name: signoz
    ports:
      - "3301:3301"      # SigNoz UI
      - "4317:4317"      # OTLP gRPC (internal collector)
      - "4318:4318"      # OTLP HTTP (internal collector)
    volumes:
      - signoz-data:/var/lib/signoz
    environment:
      - SIGNOZ_LOCAL_DB_PATH=/var/lib/signoz/signoz.db
    healthcheck:
      test: ["CMD", "wget", "-q", "--spider", "http://localhost:3301/api/v1/health"]
      interval: 30s
      timeout: 10s
      retries: 3
    restart: unless-stopped

  otel-collector:
    image: otel/opentelemetry-collector-contrib:latest
    container_name: otel-collector
    command: ["--config=/etc/otel/config.yaml"]
    volumes:
      - ./otel-collector/config.signoz.yaml:/etc/otel/config.yaml:ro
    ports:
      - "4317:4317"      # OTLP gRPC (apps send here)
      - "4318:4318"      # OTLP HTTP
      - "8888:8888"      # Collector metrics
    environment:
      - ENVIRONMENT=dev
    depends_on:
      signoz:
        condition: service_healthy
    restart: unless-stopped

volumes:
  signoz-data:

# docker/observability/otel-collector/config.grafana.yaml
# Extends base config for Grafana stack

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:
    timeout: 5s
    send_batch_size: 1000

  memory_limiter:
    check_interval: 1s
    limit_mib: 512
    spike_limit_mib: 128

  resource:
    attributes:
      - key: environment
        value: ${ENVIRONMENT:-dev}
        action: upsert

exporters:
  # Traces to Tempo
  otlp/tempo:
    endpoint: tempo:4317
    tls:
      insecure: true

  # Metrics to Prometheus (via remote write)
  prometheusremotewrite:
    endpoint: http://prometheus:9090/api/v1/write

  # Logs to Loki
  loki:
    endpoint: http://loki:3100/loki/api/v1/push

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, batch, resource]
      exporters: [otlp/tempo]
    metrics:
      receivers: [otlp]
      processors: [memory_limiter, batch, resource]
      exporters: [prometheusremotewrite]
    logs:
      receivers: [otlp]
      processors: [memory_limiter, batch, resource]
      exporters: [loki]

# docker/observability/docker-compose.grafana.yml
# Grafana + Tempo + Loki + Prometheus for local development

services:
  # Visualization
  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    ports:
      - "3000:3000"
    volumes:
      - grafana-data:/var/lib/grafana
      - ./grafana/provisioning:/etc/grafana/provisioning:ro
    environment:
      - GF_SECURITY_ADMIN_USER=admin
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer
    healthcheck:
      test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000/api/health"]
      interval: 30s
      timeout: 10s
      retries: 3
    restart: unless-stopped

  # Traces
  tempo:
    image: grafana/tempo:latest
    container_name: tempo
    command: ["-config.file=/etc/tempo/tempo.yaml"]
    volumes:
      - ./tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
      - tempo-data:/var/tempo
    ports:
      - "3200:3200"      # Tempo API
      - "4317"           # OTLP gRPC (internal)
    restart: unless-stopped

  # Logs
  loki:
    image: grafana/loki:latest
    container_name: loki
    command: ["-config.file=/etc/loki/loki.yaml"]
    volumes:
      - ./loki/loki.yaml:/etc/loki/loki.yaml:ro
      - loki-data:/var/loki
    ports:
      - "3100:3100"
    restart: unless-stopped

  # Metrics
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    command:
      - "--config.file=/etc/prometheus/prometheus.yml"
      - "--web.enable-remote-write-receiver"
      - "--storage.tsdb.path=/prometheus"
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - prometheus-data:/prometheus
    ports:
      - "9090:9090"
    restart: unless-stopped

  # OTel Collector
  otel-collector:
    image: otel/opentelemetry-collector-contrib:latest
    container_name: otel-collector
    command: ["--config=/etc/otel/config.yaml"]
    volumes:
      - ./otel-collector/config.grafana.yaml:/etc/otel/config.yaml:ro
    ports:
      - "4317:4317"      # OTLP gRPC
      - "4318:4318"      # OTLP HTTP
      - "8888:8888"      # Collector metrics
    environment:
      - ENVIRONMENT=dev
    depends_on:
      - tempo
      - loki
      - prometheus
    restart: unless-stopped

volumes:
  grafana-data:
  tempo-data:
  loki-data:
  prometheus-data:

# docker/observability/tempo/tempo.yaml

server:
  http_listen_port: 3200

distributor:
  receivers:
    otlp:
      protocols:
        grpc:
          endpoint: 0.0.0.0:4317
        http:
          endpoint: 0.0.0.0:4318

storage:
  trace:
    backend: local
    local:
      path: /var/tempo/traces
    wal:
      path: /var/tempo/wal

compactor:
  compaction:
    block_retention: 48h

# docker/observability/loki/loki.yaml

auth_enabled: false

server:
  http_listen_port: 3100

common:
  path_prefix: /var/loki
  storage:
    filesystem:
      chunks_directory: /var/loki/chunks
      rules_directory: /var/loki/rules
  replication_factor: 1
  ring:
    kvstore:
      store: inmemory

schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

# docker/observability/prometheus/prometheus.yml

global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'otel-collector'
    static_configs:
      - targets: ['otel-collector:8888']

# Remote write receiver enabled via command line flag

# docker/observability/grafana/provisioning/datasources/datasources.yaml

apiVersion: 1

datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: false

  - name: Tempo
    type: tempo
    access: proxy
    url: http://tempo:3200
    editable: false
    jsonData:
      tracesToLogs:
        datasourceUid: loki
        tags: ['service.name']
        mappedTags: [{ key: 'service.name', value: 'service_name' }]
        mapTagNamesEnabled: true
        filterByTraceID: true
        filterBySpanID: true

  - name: Loki
    type: loki
    access: proxy
    url: http://loki:3100
    editable: false
    jsonData:
      derivedFields:
        - datasourceUid: tempo
          matcherRegex: '"trace_id":"(\w+)"'
          name: TraceID
          url: '$${__value.raw}'

# terraform/modules/signoz-ec2/main.tf

variable "name_prefix" {
  description = "Prefix for resource names"
  type        = string
}

variable "vpc_id" {
  description = "VPC ID"
  type        = string
}

variable "subnet_id" {
  description = "Subnet ID for EC2 instance"
  type        = string
}

variable "instance_type" {
  description = "EC2 instance type"
  type        = string
  default     = "t3.medium"
}

variable "allowed_cidr_blocks" {
  description = "CIDR blocks allowed to access SigNoz UI"
  type        = list(string)
  default     = []
}

variable "key_name" {
  description = "SSH key name (optional, SSM preferred)"
  type        = string
  default     = null
}

variable "tags" {
  description = "Additional tags"
  type        = map(string)
  default     = {}
}

# Security Group
resource "aws_security_group" "signoz" {
  name        = "${var.name_prefix}-signoz"
  description = "SigNoz security group"
  vpc_id      = var.vpc_id

  # SigNoz UI
  ingress {
    from_port   = 3301
    to_port     = 3301
    protocol    = "tcp"
    cidr_blocks = var.allowed_cidr_blocks
    description = "SigNoz UI"
  }

  # OTLP gRPC
  ingress {
    from_port   = 4317
    to_port     = 4317
    protocol    = "tcp"
    cidr_blocks = var.allowed_cidr_blocks
    description = "OTLP gRPC"
  }

  # OTLP HTTP
  ingress {
    from_port   = 4318
    to_port     = 4318
    protocol    = "tcp"
    cidr_blocks = var.allowed_cidr_blocks
    description = "OTLP HTTP"
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }

  tags = merge(var.tags, {
    Name = "${var.name_prefix}-signoz"
  })
}

# IAM Role for SSM
resource "aws_iam_role" "signoz" {
  name = "${var.name_prefix}-signoz-role"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = "sts:AssumeRole"
        Effect = "Allow"
        Principal = {
          Service = "ec2.amazonaws.com"
        }
      }
    ]
  })

  tags = var.tags
}

resource "aws_iam_role_policy_attachment" "ssm" {
  role       = aws_iam_role.signoz.name
  policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
}

resource "aws_iam_instance_profile" "signoz" {
  name = "${var.name_prefix}-signoz-profile"
  role = aws_iam_role.signoz.name
}

# Get latest Amazon Linux 2023 AMI
data "aws_ami" "al2023" {
  most_recent = true
  owners      = ["amazon"]

  filter {
    name   = "name"
    values = ["al2023-ami-*-x86_64"]
  }

  filter {
    name   = "virtualization-type"
    values = ["hvm"]
  }
}

# EC2 Instance
resource "aws_instance" "signoz" {
  ami                    = data.aws_ami.al2023.id
  instance_type          = var.instance_type
  subnet_id              = var.subnet_id
  vpc_security_group_ids = [aws_security_group.signoz.id]
  iam_instance_profile   = aws_iam_instance_profile.signoz.name
  key_name               = var.key_name

  root_block_device {
    volume_size = 50
    volume_type = "gp3"
    encrypted   = true
  }

  user_data = base64encode(<<-EOF
    #!/bin/bash
    set -e

    # Install Docker
    dnf update -y
    dnf install -y docker
    systemctl enable docker
    systemctl start docker

    # Install Docker Compose
    curl -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
    chmod +x /usr/local/bin/docker-compose

    # Create SigNoz directory
    mkdir -p /opt/signoz
    cd /opt/signoz

    # Download SigNoz
    git clone -b main https://github.com/SigNoz/signoz.git
    cd signoz/deploy

    # Start SigNoz
    docker-compose -f docker/clickhouse-setup/docker-compose.yaml up -d
  EOF
  )

  tags = merge(var.tags, {
    Name = "${var.name_prefix}-signoz"
  })

  lifecycle {
    ignore_changes = [ami]
  }
}

# Outputs
output "instance_id" {
  description = "EC2 instance ID"
  value       = aws_instance.signoz.id
}

output "private_ip" {
  description = "Private IP address"
  value       = aws_instance.signoz.private_ip
}

output "public_ip" {
  description = "Public IP address (if applicable)"
  value       = aws_instance.signoz.public_ip
}

output "signoz_url" {
  description = "SigNoz UI URL"
  value       = "http://${aws_instance.signoz.public_ip}:3301"
}

output "otlp_endpoint" {
  description = "OTLP endpoint for applications"
  value       = "${aws_instance.signoz.private_ip}:4317"
}

output "security_group_id" {
  description = "Security group ID"
  value       = aws_security_group.signoz.id
}

# terraform/environments/dev/observability.tf

module "signoz" {
  source = "../../modules/signoz-ec2"

  name_prefix         = "${var.project_name}-${var.environment}"
  vpc_id              = module.vpc.vpc_id
  subnet_id           = module.vpc.public_subnets[0]
  instance_type       = "t3.medium"
  allowed_cidr_blocks = [var.my_ip_cidr]  # Your IP for UI access

  tags = {
    Component = "observability"
  }
}

output "signoz_url" {
  description = "SigNoz dashboard URL"
  value       = module.signoz.signoz_url
}

output "otlp_endpoint" {
  description = "OTLP endpoint for applications"
  value       = module.signoz.otlp_endpoint
}

# terraform/modules/grafana-stack-ec2/main.tf

variable "name_prefix" {
  description = "Prefix for resource names"
  type        = string
}

variable "vpc_id" {
  description = "VPC ID"
  type        = string
}

variable "subnet_id" {
  description = "Subnet ID for EC2 instance"
  type        = string
}

variable "instance_type" {
  description = "EC2 instance type"
  type        = string
  default     = "t3.medium"
}

variable "allowed_cidr_blocks" {
  description = "CIDR blocks allowed to access Grafana"
  type        = list(string)
  default     = []
}

variable "grafana_admin_password" {
  description = "Grafana admin password"
  type        = string
  sensitive   = true
}

variable "tags" {
  description = "Additional tags"
  type        = map(string)
  default     = {}
}

# Security Group
resource "aws_security_group" "grafana" {
  name        = "${var.name_prefix}-grafana"
  description = "Grafana stack security group"
  vpc_id      = var.vpc_id

  # Grafana UI
  ingress {
    from_port   = 3000
    to_port     = 3000
    protocol    = "tcp"
    cidr_blocks = var.allowed_cidr_blocks
    description = "Grafana UI"
  }

  # OTLP gRPC
  ingress {
    from_port   = 4317
    to_port     = 4317
    protocol    = "tcp"
    cidr_blocks = var.allowed_cidr_blocks
    description = "OTLP gRPC"
  }

  # OTLP HTTP
  ingress {
    from_port   = 4318
    to_port     = 4318
    protocol    = "tcp"
    cidr_blocks = var.allowed_cidr_blocks
    description = "OTLP HTTP"
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }

  tags = merge(var.tags, {
    Name = "${var.name_prefix}-grafana"
  })
}

# IAM Role
resource "aws_iam_role" "grafana" {
  name = "${var.name_prefix}-grafana-role"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = "sts:AssumeRole"
        Effect = "Allow"
        Principal = {
          Service = "ec2.amazonaws.com"
        }
      }
    ]
  })

  tags = var.tags
}

resource "aws_iam_role_policy_attachment" "ssm" {
  role       = aws_iam_role.grafana.name
  policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
}

# CloudWatch read access for AWS metrics
resource "aws_iam_role_policy" "cloudwatch_read" {
  name = "cloudwatch-read"
  role = aws_iam_role.grafana.id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Action = [
          "cloudwatch:DescribeAlarmsForMetric",
          "cloudwatch:DescribeAlarmHistory",
          "cloudwatch:DescribeAlarms",
          "cloudwatch:ListMetrics",
          "cloudwatch:GetMetricData",
          "cloudwatch:GetInsightRuleReport"
        ]
        Resource = "*"
      },
      {
        Effect = "Allow"
        Action = [
          "logs:DescribeLogGroups",
          "logs:GetLogGroupFields",
          "logs:StartQuery",
          "logs:StopQuery",
          "logs:GetQueryResults",
          "logs:GetLogEvents"
        ]
        Resource = "*"
      },
      {
        Effect = "Allow"
        Action = [
          "ec2:DescribeTags",
          "ec2:DescribeInstances",
          "ec2:DescribeRegions"
        ]
        Resource = "*"
      },
      {
        Effect = "Allow"
        Action = [
          "tag:GetResources"
        ]
        Resource = "*"
      }
    ]
  })
}

resource "aws_iam_instance_profile" "grafana" {
  name = "${var.name_prefix}-grafana-profile"
  role = aws_iam_role.grafana.name
}

# Get latest Amazon Linux 2023 AMI
data "aws_ami" "al2023" {
  most_recent = true
  owners      = ["amazon"]

  filter {
    name   = "name"
    values = ["al2023-ami-*-x86_64"]
  }

  filter {
    name   = "virtualization-type"
    values = ["hvm"]
  }
}

# EC2 Instance
resource "aws_instance" "grafana" {
  ami                    = data.aws_ami.al2023.id
  instance_type          = var.instance_type
  subnet_id              = var.subnet_id
  vpc_security_group_ids = [aws_security_group.grafana.id]
  iam_instance_profile   = aws_iam_instance_profile.grafana.name

  root_block_device {
    volume_size = 50
    volume_type = "gp3"
    encrypted   = true
  }

  user_data = base64encode(<<-EOF
    #!/bin/bash
    set -e

    # Install Docker
    dnf update -y
    dnf install -y docker git
    systemctl enable docker
    systemctl start docker

    # Install Docker Compose
    curl -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
    chmod +x /usr/local/bin/docker-compose

    # Create directories
    mkdir -p /opt/observability/{grafana/provisioning/datasources,tempo,loki,prometheus,otel-collector}
    cd /opt/observability

    # Create docker-compose.yaml
    cat > docker-compose.yaml << 'COMPOSE'
    ${file("${path.module}/templates/docker-compose.yaml")}
    COMPOSE

    # Create config files
    cat > tempo/tempo.yaml << 'TEMPO'
    ${file("${path.module}/templates/tempo.yaml")}
    TEMPO

    cat > loki/loki.yaml << 'LOKI'
    ${file("${path.module}/templates/loki.yaml")}
    LOKI

    cat > prometheus/prometheus.yml << 'PROM'
    ${file("${path.module}/templates/prometheus.yml")}
    PROM

    cat > otel-collector/config.yaml << 'OTEL'
    ${file("${path.module}/templates/otel-config.yaml")}
    OTEL

    cat > grafana/provisioning/datasources/datasources.yaml << 'DS'
    ${file("${path.module}/templates/datasources.yaml")}
    DS

    # Set Grafana password
    export GF_SECURITY_ADMIN_PASSWORD='${var.grafana_admin_password}'

    # Start stack
    docker-compose up -d
  EOF
  )

  tags = merge(var.tags, {
    Name = "${var.name_prefix}-grafana"
  })

  lifecycle {
    ignore_changes = [ami, user_data]
  }
}

# Outputs
output "instance_id" {
  value = aws_instance.grafana.id
}

output "private_ip" {
  value = aws_instance.grafana.private_ip
}

output "public_ip" {
  value = aws_instance.grafana.public_ip
}

output "grafana_url" {
  value = "http://${aws_instance.grafana.public_ip}:3000"
}

output "otlp_endpoint" {
  value = "${aws_instance.grafana.private_ip}:4317"
}

output "security_group_id" {
  value = aws_security_group.grafana.id
}

# terraform/environments/dev/observability.tf

module "grafana_stack" {
  source = "../../modules/grafana-stack-ec2"

  name_prefix            = "${var.project_name}-${var.environment}"
  vpc_id                 = module.vpc.vpc_id
  subnet_id              = module.vpc.public_subnets[0]
  instance_type          = "t3.medium"
  allowed_cidr_blocks    = [var.my_ip_cidr]
  grafana_admin_password = var.grafana_admin_password

  tags = {
    Component = "observability"
  }
}

output "grafana_url" {
  description = "Grafana dashboard URL"
  value       = module.grafana_stack.grafana_url
}

output "otlp_endpoint" {
  description = "OTLP endpoint for applications"
  value       = module.grafana_stack.otlp_endpoint
}