add prometheus collector for self and gitea

- Use ssl/ certificates for gRPC and HTTP TLS
- Add Prometheus scrape configs for local and gitea targets
  Keep matrix and caddy scrape targets commented for future use
- Switch exporters to OTLP gRPC and enable TLS certs
- Remove Tempo, Mimir, Loki exporters
- Use batch + resource_detection/docker in all pipelines
- Enable telemetry metrics
- Remove top-level volumes from docker-compose
- Mount docker socket and add docker group
- Drop unused depends_on entries
- Fix cert script to chown ssl dir with sudo -R

config/grafana-datasources.yaml

@@ -1,49 +0,0 @@
-# config/grafana-datasources.yaml
-# Auto-provisioned data sources for Grafana
-
-apiVersion: 1
-
-datasources:
-  # Tempo for traces
-  - name: Tempo
-    uid: tempo
-    type: tempo
-    access: proxy
-    url: http://tempo:3200
-    isDefault: false
-    jsonData:
-      tracesToLogsV2:
-        datasourceUid: loki
-        filterByTraceID: true
-      tracesToMetrics:
-        datasourceUid: mimir
-      nodeGraph:
-        enabled: true
-      serviceMap:
-        datasourceUid: mimir
-
-  # Mimir for metrics
-  - name: Mimir
-    uid: mimir
-    type: prometheus
-    access: proxy
-    url: http://mimir:9009/prometheus
-    isDefault: true
-    jsonData:
-      exemplarTraceIdDestinations:
-        - name: traceID
-          datasourceUid: tempo
-
-  # Loki for logs
-  - name: Loki
-    uid: loki
-    type: loki
-    access: proxy
-    url: http://loki:3100
-    isDefault: false
-    jsonData:
-      derivedFields:
-        - name: TraceID
-          datasourceUid: tempo
-          matcherRegex: "traceID=(\\w+)"
-          url: "$${__value.raw}"

config/loki.yaml

@@ -1,35 +0,0 @@
-# config/loki.yaml
-# Grafana Loki configuration for local development
-
-auth_enabled: false
-
-server:
-  http_listen_port: 3100
-
-# Common configuration shared across components
-common:
-  path_prefix: /loki
-  storage:
-    filesystem:
-      chunks_directory: /loki/chunks
-      rules_directory: /loki/rules
-  replication_factor: 1
-  ring:
-    kvstore:
-      store: inmemory
-
-# Schema configuration defines how data is stored
-schema_config:
-  configs:
-    - from: 2020-10-24
-      store: tsdb
-      object_store: filesystem
-      schema: v13
-      index:
-        prefix: index_
-        period: 24h
-
-# Limits for local development
-limits_config:
-  allow_structured_metadata: true
-  volume_enabled: true

config/mimir.yaml

@@ -1,54 +0,0 @@
-# config/mimir.yaml
-# Grafana Mimir configuration for local development
-
-# Run all components in a single process
-target: all
-
-# Server settings
-server:
-  http_listen_port: 9009
-  grpc_listen_port: 9095
-
-# Single-tenant mode for simplicity
-multitenancy_enabled: false
-
-# Block storage configuration
-blocks_storage:
-  backend: filesystem
-  filesystem:
-    dir: /data/blocks
-  tsdb:
-    dir: /data/tsdb
-  bucket_store:
-    sync_dir: /data/sync
-
-# Compactor settings
-compactor:
-  data_dir: /data/compactor
-  sharding_ring:
-    kvstore:
-      store: memberlist
-
-# Distributor accepts remote write from the Collector
-distributor:
-  ring:
-    kvstore:
-      store: memberlist
-
-# Ingester configuration
-ingester:
-  ring:
-    kvstore:
-      store: memberlist
-    replication_factor: 1
-
-# Store gateway for querying blocks
-store_gateway:
-  sharding_ring:
-    replication_factor: 1
-
-# Ruler for alerting rules
-ruler_storage:
-  backend: filesystem
-  filesystem:
-    dir: /data/rules

config/otel-collector.yaml

@@ -7,15 +7,36 @@ receivers:
       grpc:
         endpoint: 0.0.0.0:4317
         tls:
-          ca_file: /etc/otelcol-contrib/ca.crt
-          cert_file: /etc/otelcol-contrib/otel.crt
-          key_file: /etc/otelcol-contrib/otel.key
+          ca_file: /etc/otelcol-contrib/ssl/ca.crt
+          cert_file: /etc/otelcol-contrib/ssl/server.crt
+          key_file: /etc/otelcol-contrib/ssl/server.key
       http:
         endpoint: 0.0.0.0:4318
         tls:
-          ca_file: /etc/otelcol-contrib/ca.crt
-          cert_file: /etc/otelcol-contrib/otel.crt
-          key_file: /etc/otelcol-contrib/otel.key
+          ca_file: /etc/otelcol-contrib/ssl/ca.crt
+          cert_file: /etc/otelcol-contrib/ssl/server.crt
+          key_file: /etc/otelcol-contrib/ssl/server.key
+
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: 'otel-collector'
+          scrape_interval: 5s
+          static_configs:
+            - targets: ['127.0.0.1:8888']
+        - job_name: 'gitea'
+          scrape_interval: 5s
+          static_configs:
+            - targets: ['gitea_server:3000']
+        # - job_name: 'matrix'
+        #   scrape_interval: 5s
+        #   static_configs:
+        #     - targets: ['matrix-synapse:8888']
+        # - job_name: 'caddy'
+        #   scrape_interval: 5s
+        #   static_configs:
+        #     - targets: ['caddy-proxy:8888']
+
 
 processors:
   # Batch processor for efficient data transfer
@@ -23,24 +44,18 @@ processors:
     timeout: 5s
     send_batch_size: 512
 
+  resource_detection/docker:
+      detectors: [env, docker]
+      timeout: 2s
+      override: false
+
 exporters:
-  # Send traces to Tempo via OTLP
-  otlp/tempo:
-    endpoint: tempo:4317
+  otlp_grpc:
+    endpoint: shion.stuyckv.local:4317
     tls:
-      insecure: true
-
-  # Send metrics to Mimir via Prometheus remote write
-  prometheusremotewrite/mimir:
-    endpoint: http://mimir:9009/api/v1/push
-    tls:
-      insecure: true
-
-  # Send logs to Loki via its native OTLP HTTP endpoint
-  otlphttp/loki:
-    endpoint: http://loki:3100/otlp
-
-  # Keep debug output for development visibility
+      ca_file: /etc/otelcol-contrib/ssl/ca.crt
+      cert_file: /etc/otelcol-contrib/ssl/client.crt
+      key_file: /etc/otelcol-contrib/ssl/client.key
   debug:
     verbosity: basic
 
@@ -48,13 +63,16 @@ service:
   pipelines:
     traces:
       receivers: [otlp]
-      processors: [batch]
-      exporters: [otlp/tempo, debug]
+      processors: [batch, resource_detection/docker]
+      exporters: [otlp_grpc]
     metrics:
-      receivers: [otlp]
-      processors: [batch]
-      exporters: [prometheusremotewrite/mimir, debug]
+      receivers: [otlp, prometheus]
+      processors: [batch, resource_detection/docker]
+      exporters: [otlp_grpc, debug]
     logs:
       receivers: [otlp]
-      processors: [batch]
-      exporters: [otlphttp/loki, debug]
+      processors: [batch, resource_detection/docker]
+      exporters: [otlp_grpc]
+
+  telemetry:
+    metrics:

config/tempo.yaml

@@ -1,39 +0,0 @@
-# config/tempo.yaml
-# Grafana Tempo configuration for local development
-
-# Server configuration
-server:
-  http_listen_port: 3200
-
-# Distributor receives traces from the Collector
-distributor:
-  receivers:
-    otlp:
-      protocols:
-        grpc:
-          endpoint: 0.0.0.0:4317
-
-# Storage configuration using local filesystem
-storage:
-  trace:
-    backend: local
-    local:
-      path: /var/tempo/blocks
-    wal:
-      path: /var/tempo/wal
-
-# Metrics generator creates span metrics for RED dashboards
-metrics_generator:
-  registry:
-    external_labels:
-      source: tempo
-  storage:
-    path: /var/tempo/generator/wal
-    remote_write:
-      - url: http://mimir:9009/api/v1/push
-
-# Enable metrics-generator processors for the default tenant
-overrides:
-  defaults:
-    metrics_generator:
-      processors: [span-metrics, service-graphs]

docker-compose.yaml

@@ -1,9 +1,8 @@
 
-volumes:
-  tempo-data:
-  mimir-data:
-  loki-data:
-  grafana-data:
+networks:
+  gitea_database:
+    external: true
+
 
 services:
 
@@ -12,70 +11,14 @@ services:
     image: otel/opentelemetry-collector-contrib
     container_name: otel-collector
     restart: unless-stopped
+    group_add:
+      - 987 # docker group
     ports:
       - 4317:4317 # gprc
       - 4318:4318 # http
     volumes:
       - ./config/otel-collector.yaml:/etc/otelcol-contrib/config.yaml
-      - ./ssl/ca.crt:/etc/otelcol-contrib/ca.crt
-      - ./ssl/server.crt:/etc/otelcol-contrib/otel.crt
-      - ./ssl/server.key:/etc/otelcol-contrib/otel.key
-    depends_on:
-      - tempo
-      - mimir
-      - loki
-
-  # Grafana Tempo - distributed tracing backend
-  tempo:
-    image: grafana/tempo:latest
-    container_name: tempo
-    restart: unless-stopped
-    volumes:
-      - ./config/tempo.yaml:/etc/tempo/config.yaml
-      - tempo-data:/var/tempo
-    command: ["-config.file=/etc/tempo/config.yaml"]
-    ports:
-      - "3200:3200"   # Tempo API
-
-  # Grafana Mimir - long-term metrics storage
-  mimir:
-    image: grafana/mimir:latest
-    container_name: mimir
-    restart: unless-stopped
-    volumes:
-      - ./config/mimir.yaml:/etc/mimir/config.yaml
-      - mimir-data:/data
-    command: ["-config.file=/etc/mimir/config.yaml"]
-    ports:
-      - "9009:9009"   # Mimir API
-
-  # Grafana Loki - log aggregation
-  loki:
-    image: grafana/loki:latest
-    container_name: loki
-    restart: unless-stopped
-    volumes:
-      - ./config/loki.yaml:/etc/loki/config.yaml
-      - loki-data:/loki
-    command: ["-config.file=/etc/loki/config.yaml"]
-    ports:
-      - "3100:3100"   # Loki API
-
-  # Grafana - visualization and dashboards
-  grafana:
-    image: grafana/grafana:latest
-    container_name: grafana
-    restart: unless-stopped
-    environment:
-      - GF_AUTH_ANONYMOUS_ENABLED=true
-      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
-      - GF_AUTH_DISABLE_LOGIN_FORM=true
-    volumes:
-      - ./config/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
-      - grafana-data:/var/lib/grafana
-    ports:
-      - "3000:3000"   # Grafana UI
-    depends_on:
-      - tempo
-      - mimir
-      - loki
+      - ./ssl:/etc/otelcol-contrib/ssl
+      - /var/run/docker.sock:/var/run/docker.sock
+    networks:
+      - gitea_database

generate-certificates.sh

@@ -1,25 +1,43 @@
+#!/usr/bin/env bash
 
 sudo rm -rf ./ssl ; mkdir ssl
 cd ssl
 
-hostname=$(hostname).stuyckv.local
+lgtm_ssl=/home/vst/lgtm-stack/ssl
+scart=$lgtm_ssl/ca.crt
+hostname=$(hostname)
+subj="/CN=$hostname.stuyckv.local"
+san="subjectAltName=DNS:$hostname.stuyckv.local,DNS:localhost,IP:127.0.0.1"
+function request-certificate {
+    type=$1
+    scsr=$lgtm_ssl/$hostname.$type.csr
+    scrt=$lgtm_ssl/$hostname.$type.crt
 
-echo "subjectAltName=DNS:shion.stuyckv.local,DNS:localhost,IP:127.0.0.1" > shion.san
-# create root ca
-openssl req -x509 -newkey rsa:8192 \
-    -keyout ca.key -out ca.crt \
-    -days 36500 -nodes -subj "/CN=LGTM CA"
+    openssl req -new -newkey rsa:2048 \
+        -keyout $type.key -out $type.csr \
+        -nodes  -subj $subj
+    scp $type.csr vst@shion.stuyckv.local:$scsr
 
-openssl req -new -newkey rsa:2048 \
-    -keyout server.key -out server.csr \
-    -nodes -subj "/CN=$hostname"
-openssl x509 -req -CAcreateserial \
-    -in server.csr -out server.crt \
-    -CA ca.crt -CAkey ca.key \
-    -days 365 -extfile shion.san
-cat ca.crt >> server.crt
+    ssh vst@shion.stuyckv.local \
+        "echo '$san' > /tmp/$hostname.san &&
+         openssl x509 -req \
+            -in $scsr -out $scrt -days 365 \
+            -CA $scart -CAkey $lgtm_ssl/ca.key \
+            -CAcreateserial \
+            -extfile /tmp/$hostname.san && \
+         rm /tmp/$hostname.san"
 
-sudo chown 10001:10001 ./server.key ./server.crt ./ca.crt
+    scp vst@shion.stuyckv.local:$scrt $type.crt
+    rm $type.csr
+    cat ca.crt >> $type.crt
+}
+
+scp vst@shion.stuyckv.local:$scart ca.crt
+request-certificate client
+request-certificate server
+# openssl req -new -newkey rsa:2048 -nodes -keyout server.key -out server.csr -subj "/CN=$(hostname).stuyckv.local"
+# openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 365
 
-rm server.csr
 cd ..
+
+sudo chown -R 10001:10001 ./ssl