split docker-compose into core and observability stacks; add ntfy, alertmanager, blackbox exporter, and otel-collector services; configure prometheus alerting pipeline

.devcontainer/devcontainer.json

@@ -1,7 +1,6 @@
 {
     "workspaceFolder": "/workspace",
     "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,Z",
-    "runArgs": ["--userns=keep-id", "--security-opt", "label=disable"],
     "mounts": [
         "source=${localEnv:HOME}/.ssh,target=/root/.ssh,type=bind,readonly"
     ],

Caddyfile

@@ -2,6 +2,9 @@
     servers {
         metrics
     }
+    metrics {
+        per_host
+    }
     admin :2019
 }
 
@@ -20,3 +23,7 @@ gitea.qpismont.fr {
 woodpecker.qpismont.fr {
 	reverse_proxy woodpecker_server:8000
 }
+
+ntfy.qpismont.fr {
+    reverse_proxy ntfy:80
+}

blackbox/blackbox.yml

@@ -0,0 +1,10 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      valid_status_codes: []
+      method: GET
+      preferred_ip_protocol: "ip4"
+      tls_config:
+        insecure_skip_verify: true

deploy.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
 
-DATETIME=$(date +%Y%m%d%H%M%S)  docker --context prod stack deploy -c docker-compose.yml core --with-registry-auth
+DATETIME=$(date +%Y%m%d%H%M%S)  docker --context prod stack deploy -c docker-compose.core.yml core --with-registry-auth
+DATETIME=$(date +%Y%m%d%H%M%S)  docker --context prod stack deploy -c docker-compose.observability.yml observability --with-registry-auth

docker-compose.core.yml

@@ -18,7 +18,7 @@ services:
             - prometheus_network
 
     woodpecker_server:
-        image: woodpeckerci/woodpecker-server:v3.15-alpine
+        image: woodpeckerci/woodpecker-server:v3.16-alpine
         volumes:
             - woodpeckerdata:/var/lib/woodpecker/
         entrypoint: /bin/sh -c "export WOODPECKER_GITEA_CLIENT=$$(cat /run/secrets/woodpecker_gitea_client) && export WOODPECKER_GITEA_SECRET=$$(cat /run/secrets/woodpecker_gitea_secret) && export WOODPECKER_AGENT_SECRET=$$(cat /run/secrets/woodpecker_agent_secret) && /bin/woodpecker-server"
@@ -35,7 +35,7 @@ services:
             - woodpecker_gitea_secret
 
     woodpecker_agent:
-        image: woodpeckerci/woodpecker-agent:v3.15-alpine
+        image: woodpeckerci/woodpecker-agent:v3.16-alpine
         volumes:
             - /var/run/docker.sock:/var/run/docker.sock
         entrypoint: /bin/sh -c "export WOODPECKER_AGENT_SECRET=$$(cat /run/secrets/woodpecker_agent_secret) && /bin/woodpecker-agent"
@@ -49,7 +49,7 @@ services:
             - woodpecker_agent_secret
 
     gitea:
-        image: docker.gitea.com/gitea:1.26.2
+        image: docker.gitea.com/gitea:1.27.0
         environment:
             - USER_UID=1000
             - USER_GID=1000
@@ -61,53 +61,6 @@ services:
         networks:
             - reverse_network
 
-    wireguard_server:
-        image: linuxserver/wireguard
-        cap_add:
-            - NET_ADMIN
-            - SYS_MODULE
-        environment:
-            PUID: 1000
-            PGID: 1000
-            TZ: Europe/Paris
-            SERVEURURL: wireguard.qpismont.fr
-            SERVERPORT: 51820
-            PEERS: 1
-            PEERDNS: auto
-        volumes:
-            - /home/user/wireguard-config:/config
-            - /lib/modules:/lib/modules
-        ports:
-            - 51820:51820/udp
-        sysctls:
-            - net.ipv4.conf.all.src_valid_mark=1
-        networks:
-            - wireguard_network
-
-    perses:
-        image: persesdev/perses:latest
-        networks:
-            - wireguard_network
-            - prometheus_network
-
-    prometheus:
-        image: prom/prometheus:v3.9.1
-        configs:
-            - source: prometheus_config
-              target: /etc/prometheus/prometheus.yml
-        networks:
-            - prometheus_network
-
-    cadvisor:
-        image: gcr.io/cadvisor/cadvisor:latest
-        volumes:
-            - /:/rootfs:ro
-            - /var/run:/var/run:rw
-            - /sys:/sys:ro
-            - /var/lib/docker/:/var/lib/docker:ro
-        networks:
-            - prometheus_network
-
     herald:
         image: tintounn/herald:1.0
         entrypoint:
@@ -135,6 +88,40 @@ services:
             OPEN_ROUTER_MODEL: deepseek/deepseek-v4-flash
             OPEN_ROUTER_TIMEOUT: 600
 
+    wireguard_server:
+        image: linuxserver/wireguard
+        cap_add:
+            - NET_ADMIN
+            - SYS_MODULE
+        configs:
+            - source: wireguard_coredns_config
+              target: /coredns-source/Corefile
+        entrypoint:
+            - /bin/sh
+            - -c
+            - >-
+                mkdir -p /config/coredns &&
+                cp /coredns-source/Corefile /config/coredns/Corefile &&
+                exec /init
+        environment:
+            PUID: 1000
+            PGID: 1000
+            TZ: Europe/Paris
+            SERVEURURL: wireguard.qpismont.fr
+            SERVERPORT: 51820
+            PEERS: 1
+            PEERDNS: auto
+            INTERNAL_SUBNET: 10.13.13.0/24
+        volumes:
+            - /home/user/wireguard-config:/config
+            - /lib/modules:/lib/modules
+        ports:
+            - 51820:51820/udp
+        sysctls:
+            - net.ipv4.conf.all.src_valid_mark=1
+        networks:
+            - wireguard_network
+
 secrets:
     woodpecker_agent_secret:
         name: woodpecker_agent_secret_${DATETIME}
@@ -162,9 +149,9 @@ configs:
     caddy_config:
         name: caddy_config_${DATETIME}
         file: ./Caddyfile
-    prometheus_config:
+    wireguard_coredns_config:
-        name: prometheus_config_${DATETIME}
+        name: wireguard_coredns_config_${DATETIME}
-        file: ./prometheus.yml
+        file: ./wireguard/CoreDns
 
 volumes:
     giteadata:

docker-compose.observability.yml

@@ -0,0 +1,110 @@
+version: "3"
+
+services:
+    perses:
+        image: persesdev/perses:latest
+        volumes:
+            - persesdata:/perses
+        networks:
+            - prometheus_network
+            - wireguard_network
+
+    blackbox_exporter:
+        image: prom/blackbox-exporter:v0.28.0
+        configs:
+            - source: blackbox_config
+              target: /etc/blackbox_exporter/config.yml
+        networks:
+            - prometheus_network
+
+    alertmanager:
+        image: prom/alertmanager:v0.33.1
+        configs:
+            - source: alertmanager_config
+              target: /etc/alertmanager/alertmanager.yml
+        volumes:
+            - alertmanagerdata:/alertmanager
+        networks:
+            - prometheus_network
+
+    ntfy:
+        image: binwiederhier/ntfy
+        command:
+            - serve
+        environment:
+            TZ: Europe/Paris
+        volumes:
+            - ntfydata:/var/lib/ntfy
+        networks:
+            - prometheus_network
+            - reverse_network
+
+    otel-collector:
+        image: otel/opentelemetry-collector-contrib:0.156.0
+        configs:
+            - source: otel_collector_config
+              target: /etc/otelcol-contrib/config.yaml
+        deploy:
+            resources:
+                limits:
+                    memory: 256M
+        networks:
+            - prometheus_network
+
+    prometheus:
+        image: prom/prometheus:v3.9.1
+        command:
+            - --config.file=/etc/prometheus/prometheus.yml
+            - --storage.tsdb.path=/prometheus
+            - --web.enable-remote-write-receiver
+        configs:
+            - source: prometheus_config
+              target: /etc/prometheus/prometheus.yml
+            - source: prometheus_rules_config
+              target: /etc/prometheus/alert.rules.yml
+        volumes:
+            - prometheusdata:/prometheus
+        networks:
+            - prometheus_network
+            - wireguard_network
+
+    cadvisor:
+        image: gcr.io/cadvisor/cadvisor:latest
+        volumes:
+            - /:/rootfs:ro
+            - /var/run:/var/run:rw
+            - /sys:/sys:ro
+            - /var/lib/docker/:/var/lib/docker:ro
+        networks:
+            - prometheus_network
+
+configs:
+    prometheus_config:
+        name: prometheus_config_${DATETIME}
+        file: ./prometheus/prometheus.yml
+    blackbox_config:
+        name: blackbox_config_${DATETIME}
+        file: ./blackbox/blackbox.yml
+    alertmanager_config:
+        name: alertmanager_config_${DATETIME}
+        file: ./prometheus/alertmanager/alertmanager.yml
+    prometheus_rules_config:
+        name: prometheus_rules_config_${DATETIME}
+        file: ./prometheus/alert.rules.yml
+    otel_collector_config:
+        name: otel_collector_config_${DATETIME}
+        file: ./otel-collector/config.yaml
+
+volumes:
+    persesdata:
+    prometheusdata:
+    ntfydata:
+    alertmanagerdata:
+
+networks:
+    reverse_network:
+        external: true
+    wireguard_network:
+        external: true
+    prometheus_network:
+        external: true

otel-collector/config.yaml

@@ -0,0 +1,36 @@
+receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: 0.0.0.0:4318
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: reverse_caddy
+          scrape_interval: 10s
+          static_configs:
+            - targets: ['reverse_caddy:2019']
+        - job_name: cadvisor
+          scrape_interval: 10s
+          static_configs:
+            - targets: ['cadvisor:8080']
+
+processors:
+  memory_limiter:
+    check_interval: 5s
+    limit_mib: 200
+    spike_limit_mib: 50
+  batch:
+    timeout: 10s
+    send_batch_size: 1000
+
+exporters:
+  prometheusremotewrite:
+    endpoint: http://prometheus:9090/api/v1/write
+
+service:
+  pipelines:
+    metrics:
+      receivers: [otlp, prometheus]
+      processors: [memory_limiter, batch]
+      exporters: [prometheusremotewrite]

prometheus.yml

@@ -1,15 +0,0 @@
-global:
-  scrape_interval: 10s
-scrape_configs:
- - job_name: prometheus
-   static_configs:
-    - targets:
-      - prometheus:9090
- - job_name: reverse_caddy
-   static_configs:
-    - targets:
-      - reverse_caddy:2019
- - job_name: cadvisor
-   static_configs:
-    - targets:
-      - cadvisor:8080

prometheus/alert.rules.yml

@@ -0,0 +1,11 @@
+groups:
+  - name: uptime_alerts
+    rules:
+      - alert: ServiceDown
+        expr: probe_success{job="blackbox_http"} == 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "{{ $labels.instance }} est down"
+          description: "Le service {{ $labels.instance }} ne répond plus depuis plus d'1 minute."

prometheus/alertmanager/alertmanager.yml

@@ -0,0 +1,11 @@
+route:
+  receiver: ntfy
+  group_by: ['alertname']
+  repeat_interval: 8760h
+  group_wait: 10s
+
+receivers:
+  - name: ntfy
+    webhook_configs:
+      - url: http://ntfy:80/alerts?template=alertmanager
+        send_resolved: true

prometheus/prometheus.yml

@@ -0,0 +1,33 @@
+global:
+  scrape_interval: 10s
+
+rule_files:
+  - /etc/prometheus/alert.rules.yml
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+          - alertmanager:9093
+
+scrape_configs:
+ - job_name: alertmanager
+   static_configs:
+    - targets:
+      - alertmanager:9093
+ - job_name: blackbox_http
+   metrics_path: /probe
+   params:
+     module: [http_2xx]
+   static_configs:
+    - targets:
+      - https://www.qpismont.fr
+      - https://gitea.qpismont.fr
+      - https://woodpecker.qpismont.fr
+   relabel_configs:
+    - source_labels: [__address__]
+      target_label: __param_target
+    - source_labels: [__param_target]
+      target_label: instance
+    - target_label: __address__
+      replacement: blackbox_exporter:9115

wireguard/CoreDns

@@ -0,0 +1,7 @@
+.:53 {
+    rewrite name perses.internal perses
+    rewrite name prometheus.internal prometheus
+    forward . 127.0.0.11
+    errors
+    cache 10
+}