Extend monitoring and update portfolio

Add hostmetrics receiver and OTel Collector telemetry, and move
Alertmanager and Blackbox scraping to the collector. Raise collector
memory limits, restrict Blackbox to HTTP 200, and add collector and
Caddy down alerts with inhibition. Update portfolio title and copy.

blackbox/blackbox.yml

@@ -3,7 +3,7 @@ modules:
     prober: http
     timeout: 5s
     http:
-      valid_status_codes: []
+      valid_status_codes: [200]
       method: GET
       preferred_ip_protocol: "ip4"
       tls_config:

docker-compose.core.yml

@@ -20,7 +20,7 @@ services:
             - prometheus_network
 
     woodpecker_server:
-        image: woodpeckerci/woodpecker-server:v3.16-alpine
+        image: woodpeckerci/woodpecker-server:v3.17-alpine
         volumes:
             - woodpeckerdata:/var/lib/woodpecker/
         entrypoint: /bin/sh -c "export WOODPECKER_GITEA_CLIENT=$$(cat /run/secrets/woodpecker_gitea_client) && export WOODPECKER_GITEA_SECRET=$$(cat /run/secrets/woodpecker_gitea_secret) && export WOODPECKER_AGENT_SECRET=$$(cat /run/secrets/woodpecker_agent_secret) && /bin/woodpecker-server"
@@ -37,7 +37,7 @@ services:
             - woodpecker_gitea_secret
 
     woodpecker_agent:
-        image: woodpeckerci/woodpecker-agent:v3.16-alpine
+        image: woodpeckerci/woodpecker-agent:v3.17-alpine
         volumes:
             - /var/run/docker.sock:/var/run/docker.sock
         entrypoint: /bin/sh -c "export WOODPECKER_AGENT_SECRET=$$(cat /run/secrets/woodpecker_agent_secret) && /bin/woodpecker-agent"
@@ -51,7 +51,7 @@ services:
             - woodpecker_agent_secret
 
     gitea:
-        image: docker.gitea.com/gitea:1.27.0
+        image: docker.gitea.com/gitea:1.27.1
         environment:
             - USER_UID=1000
             - USER_GID=1000

docker-compose.observability.yml

@@ -57,10 +57,16 @@ services:
         configs:
             - source: otel_collector_config
               target: /etc/otelcol-contrib/config.yaml
+        volumes:
+            - /:/hostfs:ro
+            - /proc:/hostfs/proc:ro
+            - /sys:/hostfs/sys:ro
+            - /dev:/hostfs/dev:ro
+            - /run:/hostfs/run:ro
         deploy:
             resources:
                 limits:
-                    memory: 256M
+                    memory: 512M
         networks:
             - prometheus_network
 

otel-collector/config.yaml

@@ -3,6 +3,17 @@ receivers:
     protocols:
       http:
         endpoint: 0.0.0.0:4318
+  hostmetrics:
+    root_path: /hostfs
+    collection_interval: 10s
+    scrapers:
+      cpu:
+      memory:
+      disk:
+      filesystem:
+      network:
+      load:
+      processes:
   prometheus:
     config:
       scrape_configs:
@@ -18,11 +29,32 @@ receivers:
           scrape_interval: 10s
           static_configs:
             - targets: ['herald:9100']
+        - job_name: alertmanager
+          scrape_interval: 10s
+          static_configs:
+            - targets: ['alertmanager:9093']
+        - job_name: blackbox_http
+          scrape_interval: 10s
+          metrics_path: /probe
+          params:
+            module: [http_2xx]
+          static_configs:
+            - targets:
+              - https://www.qpismont.fr
+              - https://gitea.qpismont.fr
+              - https://woodpecker.qpismont.fr
+          relabel_configs:
+            - source_labels: [__address__]
+              target_label: __param_target
+            - source_labels: [__param_target]
+              target_label: instance
+            - target_label: __address__
+              replacement: blackbox_exporter:9115
 
 processors:
   memory_limiter:
     check_interval: 5s
-    limit_mib: 200
+    limit_mib: 400
     spike_limit_mib: 50
   batch:
     timeout: 10s
@@ -35,9 +67,17 @@ exporters:
     endpoint: http://tempo:4318
 
 service:
+  telemetry:
+    metrics:
+      readers:
+        - pull:
+            exporter:
+              prometheus:
+                host: 0.0.0.0
+                port: 8888
   pipelines:
     metrics:
-      receivers: [otlp, prometheus]
+      receivers: [otlp, prometheus, hostmetrics]
       processors: [memory_limiter, batch]
       exporters: [prometheusremotewrite]
     traces:

portfolio/index.html

@@ -3,11 +3,11 @@
 <head>
 <meta charset="UTF-8">
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
-<meta name="description" content="Portfolio of a platform / infrastructure engineer: distributed systems, observability, security and self-hosted CI/CD, tested in production.">
-<title>Portfolio — Platform Engineer</title>
+<meta name="description" content="Portfolio of Quentin Pismont, senior/lead platform engineer: distributed systems, observability, security and self-hosted CI/CD, tested in production.">
+<title>Quentin Pismont — Senior/Lead Platform Engineer</title>
 <!-- Open Graph -->
 <meta property="og:type" content="website">
-<meta property="og:title" content="Portfolio — Platform Engineer">
+<meta property="og:title" content="Quentin Pismont — Senior/Lead Platform Engineer">
 <meta property="og:description" content="Self-hosted infrastructure running in production, a security-first AI agent in Rust, and a distributed media pipeline. Source code available.">
 <style>
   :root {
@@ -371,8 +371,7 @@
     <div class="about-grid reveal">
       <div>
         <p>
-          Developer for <strong>over 20 years</strong> (first lines of C at age 10),
-          working as an engineer for <strong>10 years</strong>. My specialty: designing coherent
+          Working as an engineer for <strong>10 years</strong>. My specialty: designing coherent
           distributed systems and keeping them operable over time.
         </p>
         <p>

prometheus/alert.rules.yml

@@ -9,3 +9,19 @@ groups:
         annotations:
           summary: "{{ $labels.instance }} est down"
           description: "Le service {{ $labels.instance }} ne répond plus depuis plus d'1 minute."
+      - alert: OtelCollectorDown
+        expr: up{job="otel-collector"} == 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "OTel Collector est down"
+          description: "L'OTel Collector ne répond plus depuis plus d'1 minute."
+      - alert: CaddyDown
+        expr: up{job="reverse_caddy"} == 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Caddy est down"
+          description: "Le reverse proxy Caddy ne répond plus depuis plus d'1 minute."

prometheus/alertmanager/alertmanager.yml

@@ -4,6 +4,12 @@ route:
   repeat_interval: 8760h
   group_wait: 10s
 
+inhibit_rules:
+  - source_matchers:
+      - alertname = "CaddyDown"
+    target_matchers:
+      - alertname = "ServiceDown"
+
 receivers:
   - name: ntfy
     webhook_configs:

prometheus/prometheus.yml

@@ -11,23 +11,7 @@ alerting:
           - alertmanager:9093
 
 scrape_configs:
- - job_name: alertmanager
+ - job_name: otel-collector
    static_configs:
     - targets:
-      - alertmanager:9093
- - job_name: blackbox_http
-   metrics_path: /probe
-   params:
-     module: [http_2xx]
-   static_configs:
-    - targets:
-      - https://www.qpismont.fr
-      - https://gitea.qpismont.fr
-      - https://woodpecker.qpismont.fr
-   relabel_configs:
-    - source_labels: [__address__]
-      target_label: __param_target
-    - source_labels: [__param_target]
-      target_label: instance
-    - target_label: __address__
-      replacement: blackbox_exporter:9115
+      - otel-collector:8888