From e13bfe45023ee4c8d261e22dff366eeaa1ec5660 Mon Sep 17 00:00:00 2001 From: qpismont Date: Wed, 15 Jul 2026 18:21:50 +0000 Subject: [PATCH] split docker-compose into core and observability stacks; add ntfy, alertmanager, blackbox exporter, and otel-collector services; configure prometheus alerting pipeline --- .devcontainer/devcontainer.json | 1 - Caddyfile | 7 ++ blackbox/blackbox.yml | 10 ++ deploy.sh | 3 +- docker-compose.yml => docker-compose.core.yml | 93 +++++++-------- docker-compose.observability.yml | 110 ++++++++++++++++++ otel-collector/config.yaml | 36 ++++++ prometheus.yml | 15 --- prometheus/alert.rules.yml | 11 ++ prometheus/alertmanager/alertmanager.yml | 11 ++ prometheus/prometheus.yml | 33 ++++++ wireguard/CoreDns | 7 ++ 12 files changed, 267 insertions(+), 70 deletions(-) create mode 100644 blackbox/blackbox.yml rename docker-compose.yml => docker-compose.core.yml (84%) create mode 100644 docker-compose.observability.yml create mode 100644 otel-collector/config.yaml delete mode 100644 prometheus.yml create mode 100644 prometheus/alert.rules.yml create mode 100644 prometheus/alertmanager/alertmanager.yml create mode 100644 prometheus/prometheus.yml create mode 100644 wireguard/CoreDns diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 36b566c..655be80 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,6 @@ { "workspaceFolder": "/workspace", "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,Z", - "runArgs": ["--userns=keep-id", "--security-opt", "label=disable"], "mounts": [ "source=${localEnv:HOME}/.ssh,target=/root/.ssh,type=bind,readonly" ], diff --git a/Caddyfile b/Caddyfile index 2c0ecd8..6715bf7 100644 --- a/Caddyfile +++ b/Caddyfile @@ -2,6 +2,9 @@ servers { metrics } + metrics { + per_host + } admin :2019 } @@ -19,4 +22,8 @@ gitea.qpismont.fr { woodpecker.qpismont.fr { reverse_proxy woodpecker_server:8000 +} + +ntfy.qpismont.fr { + reverse_proxy ntfy:80 } \ No newline at end of file diff --git a/blackbox/blackbox.yml b/blackbox/blackbox.yml new file mode 100644 index 0000000..ad94a31 --- /dev/null +++ b/blackbox/blackbox.yml @@ -0,0 +1,10 @@ +modules: + http_2xx: + prober: http + timeout: 5s + http: + valid_status_codes: [] + method: GET + preferred_ip_protocol: "ip4" + tls_config: + insecure_skip_verify: true diff --git a/deploy.sh b/deploy.sh index 5b70649..c4ce3d2 100644 --- a/deploy.sh +++ b/deploy.sh @@ -1,3 +1,4 @@ #!/bin/bash -DATETIME=$(date +%Y%m%d%H%M%S) docker --context prod stack deploy -c docker-compose.yml core --with-registry-auth \ No newline at end of file +DATETIME=$(date +%Y%m%d%H%M%S) docker --context prod stack deploy -c docker-compose.core.yml core --with-registry-auth +DATETIME=$(date +%Y%m%d%H%M%S) docker --context prod stack deploy -c docker-compose.observability.yml observability --with-registry-auth \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.core.yml similarity index 84% rename from docker-compose.yml rename to docker-compose.core.yml index c2090d6..d255a87 100644 --- a/docker-compose.yml +++ b/docker-compose.core.yml @@ -18,7 +18,7 @@ services: - prometheus_network woodpecker_server: - image: woodpeckerci/woodpecker-server:v3.15-alpine + image: woodpeckerci/woodpecker-server:v3.16-alpine volumes: - woodpeckerdata:/var/lib/woodpecker/ entrypoint: /bin/sh -c "export WOODPECKER_GITEA_CLIENT=$$(cat /run/secrets/woodpecker_gitea_client) && export WOODPECKER_GITEA_SECRET=$$(cat /run/secrets/woodpecker_gitea_secret) && export WOODPECKER_AGENT_SECRET=$$(cat /run/secrets/woodpecker_agent_secret) && /bin/woodpecker-server" @@ -35,7 +35,7 @@ services: - woodpecker_gitea_secret woodpecker_agent: - image: woodpeckerci/woodpecker-agent:v3.15-alpine + image: woodpeckerci/woodpecker-agent:v3.16-alpine volumes: - /var/run/docker.sock:/var/run/docker.sock entrypoint: /bin/sh -c "export WOODPECKER_AGENT_SECRET=$$(cat /run/secrets/woodpecker_agent_secret) && /bin/woodpecker-agent" @@ -49,7 +49,7 @@ services: - woodpecker_agent_secret gitea: - image: docker.gitea.com/gitea:1.26.2 + image: docker.gitea.com/gitea:1.27.0 environment: - USER_UID=1000 - USER_GID=1000 @@ -61,53 +61,6 @@ services: networks: - reverse_network - wireguard_server: - image: linuxserver/wireguard - cap_add: - - NET_ADMIN - - SYS_MODULE - environment: - PUID: 1000 - PGID: 1000 - TZ: Europe/Paris - SERVEURURL: wireguard.qpismont.fr - SERVERPORT: 51820 - PEERS: 1 - PEERDNS: auto - volumes: - - /home/user/wireguard-config:/config - - /lib/modules:/lib/modules - ports: - - 51820:51820/udp - sysctls: - - net.ipv4.conf.all.src_valid_mark=1 - networks: - - wireguard_network - - perses: - image: persesdev/perses:latest - networks: - - wireguard_network - - prometheus_network - - prometheus: - image: prom/prometheus:v3.9.1 - configs: - - source: prometheus_config - target: /etc/prometheus/prometheus.yml - networks: - - prometheus_network - - cadvisor: - image: gcr.io/cadvisor/cadvisor:latest - volumes: - - /:/rootfs:ro - - /var/run:/var/run:rw - - /sys:/sys:ro - - /var/lib/docker/:/var/lib/docker:ro - networks: - - prometheus_network - herald: image: tintounn/herald:1.0 entrypoint: @@ -135,6 +88,40 @@ services: OPEN_ROUTER_MODEL: deepseek/deepseek-v4-flash OPEN_ROUTER_TIMEOUT: 600 + wireguard_server: + image: linuxserver/wireguard + cap_add: + - NET_ADMIN + - SYS_MODULE + configs: + - source: wireguard_coredns_config + target: /coredns-source/Corefile + entrypoint: + - /bin/sh + - -c + - >- + mkdir -p /config/coredns && + cp /coredns-source/Corefile /config/coredns/Corefile && + exec /init + environment: + PUID: 1000 + PGID: 1000 + TZ: Europe/Paris + SERVEURURL: wireguard.qpismont.fr + SERVERPORT: 51820 + PEERS: 1 + PEERDNS: auto + INTERNAL_SUBNET: 10.13.13.0/24 + volumes: + - /home/user/wireguard-config:/config + - /lib/modules:/lib/modules + ports: + - 51820:51820/udp + sysctls: + - net.ipv4.conf.all.src_valid_mark=1 + networks: + - wireguard_network + secrets: woodpecker_agent_secret: name: woodpecker_agent_secret_${DATETIME} @@ -162,9 +149,9 @@ configs: caddy_config: name: caddy_config_${DATETIME} file: ./Caddyfile - prometheus_config: - name: prometheus_config_${DATETIME} - file: ./prometheus.yml + wireguard_coredns_config: + name: wireguard_coredns_config_${DATETIME} + file: ./wireguard/CoreDns volumes: giteadata: diff --git a/docker-compose.observability.yml b/docker-compose.observability.yml new file mode 100644 index 0000000..15ac3a2 --- /dev/null +++ b/docker-compose.observability.yml @@ -0,0 +1,110 @@ +version: "3" + +services: + perses: + image: persesdev/perses:latest + volumes: + - persesdata:/perses + networks: + - prometheus_network + - wireguard_network + + blackbox_exporter: + image: prom/blackbox-exporter:v0.28.0 + configs: + - source: blackbox_config + target: /etc/blackbox_exporter/config.yml + networks: + - prometheus_network + + alertmanager: + image: prom/alertmanager:v0.33.1 + configs: + - source: alertmanager_config + target: /etc/alertmanager/alertmanager.yml + volumes: + - alertmanagerdata:/alertmanager + networks: + - prometheus_network + + ntfy: + image: binwiederhier/ntfy + command: + - serve + environment: + TZ: Europe/Paris + volumes: + - ntfydata:/var/lib/ntfy + networks: + - prometheus_network + - reverse_network + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.156.0 + configs: + - source: otel_collector_config + target: /etc/otelcol-contrib/config.yaml + deploy: + resources: + limits: + memory: 256M + networks: + - prometheus_network + + prometheus: + image: prom/prometheus:v3.9.1 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --web.enable-remote-write-receiver + configs: + - source: prometheus_config + target: /etc/prometheus/prometheus.yml + - source: prometheus_rules_config + target: /etc/prometheus/alert.rules.yml + volumes: + - prometheusdata:/prometheus + networks: + - prometheus_network + - wireguard_network + + cadvisor: + image: gcr.io/cadvisor/cadvisor:latest + volumes: + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + networks: + - prometheus_network + +configs: + prometheus_config: + name: prometheus_config_${DATETIME} + file: ./prometheus/prometheus.yml + blackbox_config: + name: blackbox_config_${DATETIME} + file: ./blackbox/blackbox.yml + alertmanager_config: + name: alertmanager_config_${DATETIME} + file: ./prometheus/alertmanager/alertmanager.yml + prometheus_rules_config: + name: prometheus_rules_config_${DATETIME} + file: ./prometheus/alert.rules.yml + otel_collector_config: + name: otel_collector_config_${DATETIME} + file: ./otel-collector/config.yaml + +volumes: + persesdata: + prometheusdata: + ntfydata: + alertmanagerdata: + +networks: + reverse_network: + external: true + wireguard_network: + external: true + prometheus_network: + external: true diff --git a/otel-collector/config.yaml b/otel-collector/config.yaml new file mode 100644 index 0000000..9f05a33 --- /dev/null +++ b/otel-collector/config.yaml @@ -0,0 +1,36 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + prometheus: + config: + scrape_configs: + - job_name: reverse_caddy + scrape_interval: 10s + static_configs: + - targets: ['reverse_caddy:2019'] + - job_name: cadvisor + scrape_interval: 10s + static_configs: + - targets: ['cadvisor:8080'] + +processors: + memory_limiter: + check_interval: 5s + limit_mib: 200 + spike_limit_mib: 50 + batch: + timeout: 10s + send_batch_size: 1000 + +exporters: + prometheusremotewrite: + endpoint: http://prometheus:9090/api/v1/write + +service: + pipelines: + metrics: + receivers: [otlp, prometheus] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] \ No newline at end of file diff --git a/prometheus.yml b/prometheus.yml deleted file mode 100644 index 4220f5c..0000000 --- a/prometheus.yml +++ /dev/null @@ -1,15 +0,0 @@ -global: - scrape_interval: 10s -scrape_configs: - - job_name: prometheus - static_configs: - - targets: - - prometheus:9090 - - job_name: reverse_caddy - static_configs: - - targets: - - reverse_caddy:2019 - - job_name: cadvisor - static_configs: - - targets: - - cadvisor:8080 \ No newline at end of file diff --git a/prometheus/alert.rules.yml b/prometheus/alert.rules.yml new file mode 100644 index 0000000..6deb4d9 --- /dev/null +++ b/prometheus/alert.rules.yml @@ -0,0 +1,11 @@ +groups: + - name: uptime_alerts + rules: + - alert: ServiceDown + expr: probe_success{job="blackbox_http"} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "{{ $labels.instance }} est down" + description: "Le service {{ $labels.instance }} ne répond plus depuis plus d'1 minute." diff --git a/prometheus/alertmanager/alertmanager.yml b/prometheus/alertmanager/alertmanager.yml new file mode 100644 index 0000000..f1d36e4 --- /dev/null +++ b/prometheus/alertmanager/alertmanager.yml @@ -0,0 +1,11 @@ +route: + receiver: ntfy + group_by: ['alertname'] + repeat_interval: 8760h + group_wait: 10s + +receivers: + - name: ntfy + webhook_configs: + - url: http://ntfy:80/alerts?template=alertmanager + send_resolved: true diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml new file mode 100644 index 0000000..3a92b01 --- /dev/null +++ b/prometheus/prometheus.yml @@ -0,0 +1,33 @@ +global: + scrape_interval: 10s + +rule_files: + - /etc/prometheus/alert.rules.yml + +alerting: + alertmanagers: + - static_configs: + - targets: + - alertmanager:9093 + +scrape_configs: + - job_name: alertmanager + static_configs: + - targets: + - alertmanager:9093 + - job_name: blackbox_http + metrics_path: /probe + params: + module: [http_2xx] + static_configs: + - targets: + - https://www.qpismont.fr + - https://gitea.qpismont.fr + - https://woodpecker.qpismont.fr + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox_exporter:9115 \ No newline at end of file diff --git a/wireguard/CoreDns b/wireguard/CoreDns new file mode 100644 index 0000000..1dc1526 --- /dev/null +++ b/wireguard/CoreDns @@ -0,0 +1,7 @@ +.:53 { + rewrite name perses.internal perses + rewrite name prometheus.internal prometheus + forward . 127.0.0.11 + errors + cache 10 +}