add labels for gitea and otel collector

- Use ssl/ certificates for gRPC and HTTP TLS
- Add Prometheus scrape configs for local and gitea targets
  Keep matrix and caddy scrape targets commented for future use
- Switch exporters to OTLP gRPC and enable TLS certs
- Remove Tempo, Mimir, Loki exporters
- Use batch + resource_detection/docker in all pipelines
- Enable telemetry metrics
- Remove top-level volumes from docker-compose
- Mount docker socket and add docker group
- Drop unused depends_on entries
- Fix cert script to chown ssl dir with sudo -R

.gitignore

@@ -1 +1,3 @@
 .env
+ssl
+/

config/otel-collector.yaml

@@ -0,0 +1,90 @@
+# config/otel-collector.yaml
+# Collector configuration routing to the LGTM stack
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+        tls:
+          ca_file: /etc/otelcol-contrib/ssl/ca.crt
+          cert_file: /etc/otelcol-contrib/ssl/server.crt
+          key_file: /etc/otelcol-contrib/ssl/server.key
+      http:
+        endpoint: 0.0.0.0:4318
+        tls:
+          ca_file: /etc/otelcol-contrib/ssl/ca.crt
+          cert_file: /etc/otelcol-contrib/ssl/server.crt
+          key_file: /etc/otelcol-contrib/ssl/server.key
+
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: 'otel-collector'
+          scrape_interval: 5s
+          static_configs:
+            - targets:
+              - '127.0.0.1:8888'
+              labels:
+                service.name: otel-collector
+                service.version: latest
+                host.name: charybdis.stuyckv.local
+                container.name: otel-collector
+        - job_name: 'gitea'
+          scrape_interval: 5s
+          static_configs:
+            - targets:
+              - 'gitea_server:3000'
+              labels:
+                service.name: gitea_server
+                service.version: latest
+                host.name: charybdis.stuyckv.local
+                container.name: gitea_server
+        # - job_name: 'matrix'
+        #   scrape_interval: 5s
+        #   static_configs:
+        #     - targets: ['matrix-synapse:8888']
+        # - job_name: 'caddy'
+        #   scrape_interval: 5s
+        #   static_configs:
+        #     - targets: ['caddy-proxy:8888']
+
+
+processors:
+  # Batch processor for efficient data transfer
+  batch:
+    timeout: 5s
+    send_batch_size: 512
+
+  resource_detection/docker:
+    detectors: [env, docker]
+    timeout: 2s
+    override: false
+
+exporters:
+  otlp_grpc:
+    endpoint: shion.stuyckv.local:4317
+    tls:
+      ca_file: /etc/otelcol-contrib/ssl/ca.crt
+      cert_file: /etc/otelcol-contrib/ssl/client.crt
+      key_file: /etc/otelcol-contrib/ssl/client.key
+  debug:
+    verbosity: basic
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch, resource_detection/docker]
+      exporters: [otlp_grpc]
+    metrics:
+      receivers: [otlp, prometheus]
+      processors: [batch, resource_detection/docker]
+      exporters: [otlp_grpc]
+    logs:
+      receivers: [otlp]
+      processors: [batch, resource_detection/docker]
+      exporters: [otlp_grpc]
+
+  telemetry:
+    metrics:

docker-compose.yaml

@@ -0,0 +1,24 @@
+
+networks:
+  gitea_database:
+    external: true
+
+
+services:
+
+  # OpenTelemetry Collector - receives and routes all telemetry
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib
+    container_name: otel-collector
+    restart: unless-stopped
+    group_add:
+      - 987 # docker group
+    ports:
+      - 4317:4317 # gprc
+      - 4318:4318 # http
+    volumes:
+      - ./config/otel-collector.yaml:/etc/otelcol-contrib/config.yaml
+      - ./ssl:/etc/otelcol-contrib/ssl
+      - /var/run/docker.sock:/var/run/docker.sock
+    networks:
+      - gitea_database

generate-certificates.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+sudo rm -rf ./ssl ; mkdir ssl
+cd ssl
+
+lgtm_ssl=/home/vst/lgtm-stack/ssl
+scart=$lgtm_ssl/ca.crt
+hostname=$(hostname)
+subj="/CN=$hostname.stuyckv.local"
+san="subjectAltName=DNS:$hostname.stuyckv.local,DNS:localhost,IP:127.0.0.1"
+function request-certificate {
+    type=$1
+    scsr=$lgtm_ssl/$hostname.$type.csr
+    scrt=$lgtm_ssl/$hostname.$type.crt
+
+    openssl req -new -newkey rsa:2048 \
+        -keyout $type.key -out $type.csr \
+        -nodes  -subj $subj
+    scp $type.csr vst@shion.stuyckv.local:$scsr
+
+    ssh vst@shion.stuyckv.local \
+        "echo '$san' > /tmp/$hostname.san &&
+         openssl x509 -req \
+            -in $scsr -out $scrt -days 365 \
+            -CA $scart -CAkey $lgtm_ssl/ca.key \
+            -CAcreateserial \
+            -extfile /tmp/$hostname.san && \
+         rm /tmp/$hostname.san"
+
+    scp vst@shion.stuyckv.local:$scrt $type.crt
+    rm $type.csr
+    cat ca.crt >> $type.crt
+}
+
+scp vst@shion.stuyckv.local:$scart ca.crt
+request-certificate client
+request-certificate server
+# openssl req -new -newkey rsa:2048 -nodes -keyout server.key -out server.csr -subj "/CN=$(hostname).stuyckv.local"
+# openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 365
+
+cd ..
+
+sudo chown -R 10001:10001 ./ssl