Observability #6

Merged
qpismont merged 6 commits from 1.1 into main 2026-07-26 21:55:42 +02:00
10 changed files with 332 additions and 16 deletions
Showing only changes of commit 6ffd88927c - Show all commits
+3
View File
@@ -17,3 +17,6 @@ GITEA_TIMEOUT=30
SENTRY_DSN=
RUST_LOG=info
RUST_BACKTRACE=1
METRICS_BIND_ADDR=
qpismont marked this conversation as resolved
Review

La variable d'environnement METRICS_BIND_ADDR est définie sans valeur par défaut ni commentaire. Il serait judicieux d'ajouter un exemple de format (ex: 0.0.0.0:9090) ou un commentaire indiquant le format attendu, afin d'éviter des erreurs de configuration.

La variable d'environnement `METRICS_BIND_ADDR` est définie sans valeur par défaut ni commentaire. Il serait judicieux d'ajouter un exemple de format (ex: `0.0.0.0:9090`) ou un commentaire indiquant le format attendu, afin d'éviter des erreurs de configuration.
Generated
+229 -7
View File
@@ -34,7 +34,7 @@ dependencies = [
"bytestring",
"derive_more",
"encoding_rs",
"foldhash",
"foldhash 0.1.5",
"futures-core",
"http 0.2.12",
"httparse",
@@ -129,7 +129,7 @@ dependencies = [
"cfg-if",
"derive_more",
"encoding_rs",
"foldhash",
"foldhash 0.1.5",
"futures-core",
"futures-util",
"impl-more",
@@ -357,6 +357,21 @@ version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "crossbeam-epoch"
version = "0.9.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
[[package]]
name = "darling"
version = "0.20.11"
@@ -545,6 +560,17 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "evmap"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b8874945f036109c72242964c1174cf99434e30cfa45bf45fedc983f50046f8"
dependencies = [
"hashbag",
"left-right",
"smallvec",
]
[[package]]
name = "fastrand"
version = "2.4.1"
@@ -581,6 +607,12 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "foldhash"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
[[package]]
name = "foreign-types"
version = "0.3.2"
@@ -666,6 +698,21 @@ dependencies = [
"slab",
]
[[package]]
name = "generator"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae"
dependencies = [
"cc",
"cfg-if",
"libc",
"log",
"rustversion",
"windows-link",
"windows-result",
]
[[package]]
name = "getrandom"
version = "0.2.17"
@@ -718,6 +765,21 @@ dependencies = [
"tracing",
]
[[package]]
name = "hashbag"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064"
[[package]]
name = "hashbrown"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
dependencies = [
"foldhash 0.2.0",
]
[[package]]
name = "hashbrown"
version = "0.17.1"
@@ -734,6 +796,8 @@ dependencies = [
"dotenvy",
"futures-util",
"hex",
"metrics",
"metrics-exporter-prometheus",
"openrouter-rs",
"reqwest 0.12.28",
"ring",
@@ -1023,7 +1087,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
dependencies = [
"equivalent",
"hashbrown",
"hashbrown 0.17.1",
]
[[package]]
@@ -1062,6 +1126,17 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]]
name = "left-right"
version = "0.11.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bc015ded5d9b3054dbbdb63332cdd6ee42352ccef19e911e25117490e2f48ee"
dependencies = [
"crossbeam-utils",
"loom",
"slab",
]
[[package]]
name = "libc"
version = "0.2.186"
@@ -1101,6 +1176,19 @@ version = "0.4.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
[[package]]
name = "loom"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca"
dependencies = [
"cfg-if",
"generator",
"scoped-tls",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "lru-slab"
version = "0.1.2"
@@ -1128,12 +1216,70 @@ version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
[[package]]
name = "metrics"
version = "0.24.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2"
dependencies = [
"portable-atomic",
"rapidhash",
]
[[package]]
name = "metrics-exporter-prometheus"
version = "0.18.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108"
dependencies = [
"base64",
"evmap",
"http-body-util",
"hyper",
"hyper-util",
"indexmap",
"ipnet",
"metrics",
"metrics-util",
"quanta",
"thiserror 2.0.18",
"tokio",
"tracing",
]
[[package]]
name = "metrics-util"
version = "0.20.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96f8722f8562635f92f8ed992f26df0532266eb03d5202607c20c0d7e9745e13"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
"hashbrown 0.16.1",
"metrics",
"quanta",
"rand",
"rand_xoshiro",
"rapidhash",
"sketches-ddsketch",
]
[[package]]
name = "mime"
version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "mime_guess"
version = "2.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
dependencies = [
"mime",
"unicase",
]
[[package]]
name = "miniz_oxide"
version = "0.8.9"
@@ -1375,9 +1521,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "openrouter-rs"
version = "0.10.0"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f7101578df2f54d9013594e94367adbe656521ef6002f03e4577f1e00e57cb4"
checksum = "25918cfeef40cfd3adea348b268496ee5206eb0408f2aacfe4bdfef2bfbe3bf9"
dependencies = [
"derive_builder",
"dotenvy_macro",
@@ -1522,6 +1668,12 @@ version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
[[package]]
name = "portable-atomic"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3"
[[package]]
name = "potential_utf"
version = "0.1.5"
@@ -1555,6 +1707,21 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "quanta"
version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7"
dependencies = [
"crossbeam-utils",
"libc",
"once_cell",
"raw-cpuid",
"wasi",
"web-sys",
"winapi",
]
[[package]]
name = "quinn"
version = "0.11.9"
@@ -1654,6 +1821,33 @@ dependencies = [
"getrandom 0.3.4",
]
[[package]]
name = "rand_xoshiro"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41"
dependencies = [
"rand_core",
]
[[package]]
name = "rapidhash"
version = "4.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e"
dependencies = [
"rustversion",
]
[[package]]
name = "raw-cpuid"
version = "11.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
dependencies = [
"bitflags",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
@@ -1736,6 +1930,7 @@ dependencies = [
"hyper-util",
"js-sys",
"log",
"mime_guess",
"percent-encoding",
"pin-project-lite",
"quinn",
@@ -1927,6 +2122,12 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "scoped-tls"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
[[package]]
name = "scopeguard"
version = "1.2.0"
@@ -2213,6 +2414,12 @@ dependencies = [
"libc",
]
[[package]]
name = "sketches-ddsketch"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b"
[[package]]
name = "slab"
version = "0.4.12"
@@ -2425,9 +2632,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.52.3"
version = "1.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
dependencies = [
"bytes",
"libc",
@@ -2619,6 +2826,12 @@ dependencies = [
"libc",
]
[[package]]
name = "unicase"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
[[package]]
name = "unicode-ident"
version = "1.0.24"
@@ -2884,6 +3097,15 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
[[package]]
name = "windows-result"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
dependencies = [
"windows-link",
]
[[package]]
name = "windows-sys"
version = "0.52.0"
+4 -2
View File
@@ -8,7 +8,7 @@ debug = 1
[dependencies]
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
tokio = { version = "1.52", features = ["full"] }
tokio = { version = "1.53", features = ["full"] }
tokio-stream = "0.1"
tokio-util = "0.7"
futures-util = "0.3"
@@ -16,7 +16,7 @@ serde_json = "1.0"
serde = { version = "1.0", features = ["derive"] }
sentry = { version = "0.48", features = ["tower-axum-matched-path"] }
sentry-anyhow = { version = "0.48", features = ["backtrace"] }
openrouter-rs = "0.10"
openrouter-rs = "0.12"
dotenvy = "0.15"
tower = "0.5"
tower-http = {version = "0.6", features = ["trace"] }
@@ -28,3 +28,5 @@ thiserror = "2.0"
ring = "0.17"
hex = "0.4"
bytes = "1.1"
metrics = "0.24"
metrics-exporter-prometheus = { version = "0.18", default-features = false, features = ["http-listener"] }
+6
View File
@@ -17,6 +17,7 @@ use tokio_util::sync::CancellationToken;
use crate::consts::{GITEA_EVENT_TYPE_HEADER_NAME, GITEA_SIG_HEADER_NAME, MAX_WEBHOOK_BODY_SIZE};
use crate::errors::AppError;
use crate::gitea::WebhookType;
use crate::metrics;
use crate::state::AppState;
pub async fn start(app_state: AppState, shutdown: CancellationToken) -> anyhow::Result<()> {
@@ -56,13 +57,18 @@ async fn webhook(
) -> Result<impl IntoResponse, AppError> {
tracing::Span::current().record("webhook_type", tracing::field::debug(&wb));
let event_type = wb.event_type_str();
metrics::webhook_received(event_type);
let event_id = wb.event_id();
if app_state.bot.check_and_mark(event_id).await {
metrics::webhook_duplicate(event_type);
return Err(AppError::AlreadyProcessedErr);
}
if app_state.bot_tx.try_send(wb).is_err() {
app_state.bot.unmark(event_id).await;
metrics::webhook_channel_full(event_type);
return Err(AppError::ChannelFullErr);
}
+10 -3
View File
@@ -1,6 +1,5 @@
use crate::{
gitea::{GiteaAPI, WebhookType},
open_router::OpenRouterClient,
gitea::{GiteaAPI, WebhookType}, metrics, open_router::OpenRouterClient,
};
use serde::Deserialize;
use std::{collections::HashSet, sync::Arc};
1
@@ -87,10 +86,12 @@ impl Bot {
info!(queued = rx.len(), active = tasks.len(), "Webhook received");
let permit = sem.clone().acquire_owned().await?;
let self_clone = self.clone();
metrics::increment_task_active();
tasks.spawn(async move {
self_clone.exec(wb).await;
drop(permit);
metrics::decrement_task_active();
});
}
@@ -102,6 +103,8 @@ impl Bot {
#[instrument(skip(self, webhook), fields(repo, pr))]
pub async fn exec(&self, webhook: WebhookType) {
let event_type_str = webhook.event_type_str();
match &webhook {
WebhookType::Review(p) => {
tracing::Span::current().record("repo", &p.repository.full_name);
@@ -121,8 +124,12 @@ impl Bot {
.await;
match exec_result {
Ok(_) => info!("Task completed"),
Ok(_) => {
metrics::task_completed(event_type_str);
info!("Task completed");
}
Err(err) => {
metrics::task_failed(event_type_str);
error!(%err, "Task error");
sentry_anyhow::capture_anyhow(&err);
}
+4 -4
View File
@@ -4,10 +4,7 @@ use tokio_util::io::StreamReader;
use tracing::instrument;
use crate::{
bot::ReviewResult,
consts::{BOT_PROCESS_MSG, MAX_DIFF_SIZE, REVIEW_PROMPT},
gitea::{GiteaAPI, ReviewPayload},
open_router::OpenRouterClient,
bot::ReviewResult, consts::{BOT_PROCESS_MSG, MAX_DIFF_SIZE, REVIEW_PROMPT}, gitea::{GiteaAPI, ReviewPayload}, metrics, open_router::OpenRouterClient,
};
#[instrument(skip(gitea_api, open_router_client, http_client, review_payload))]
@@ -48,6 +45,9 @@ pub async fn exec_review(
let mut review_result = serde_json::from_str::<ReviewResult>(&chat_result.message)?;
review_result.cost = chat_result.cost;
if let Some(cost) = review_result.cost {
metrics::openrouter_cost_usd((cost / 100.0).round() as u64);
}
let final_review_markdown = review_result_to_markdown(&review_result);
+3
View File
@@ -11,6 +11,7 @@ pub struct EnvConfig {
pub gitea_url: String,
pub gitea_token: String,
pub gitea_timeout: u64,
pub metrics_bind_addr: Option<String>,
}
pub fn load_config() -> anyhow::Result<EnvConfig> {
@@ -23,6 +24,7 @@ pub fn load_config() -> anyhow::Result<EnvConfig> {
let gitea_url = try_get_env("GITEA_URL")?;
let gitea_token = try_get_env("GITEA_TOKEN")?;
let gitea_timeout = try_get_env("GITEA_TIMEOUT")?.parse()?;
let metrics_bind_addr = std::env::var("METRICS_BIND_ADDR").ok();
Ok(EnvConfig {
http_port,
@@ -34,6 +36,7 @@ pub fn load_config() -> anyhow::Result<EnvConfig> {
gitea_url,
gitea_token,
gitea_timeout,
metrics_bind_addr,
})
}
+6
View File
@@ -179,6 +179,12 @@ impl WebhookType {
WebhookType::Review(payload) => payload.comment.id,
}
}
pub fn event_type_str(&self) -> &'static str {
match self {
WebhookType::Review(_) => "review",
}
}
}
#[derive(Deserialize, Debug)]
+5
View File
@@ -13,6 +13,7 @@ mod consts;
mod env;
mod errors;
mod gitea;
mod metrics;
mod open_router;
mod state;
@@ -49,6 +50,10 @@ fn main() -> anyhow::Result<()> {
async fn run() -> anyhow::Result<()> {
let config = env::load_config()?;
if let Some(metric_bind_addr) = &config.metrics_bind_addr {
qpismont marked this conversation as resolved
Review

Le démarrage conditionnel de l'exportateur Prometheus est une bonne pratique. Toutefois, si l'installation échoue, l'erreur est propagée via ?, ce qui met fin à l'application. Cela peut être acceptable, mais un message d'erreur plus explicite (ex: avec tracing::error) aiderait au débogage.

Le démarrage conditionnel de l'exportateur Prometheus est une bonne pratique. Toutefois, si l'installation échoue, l'erreur est propagée via `?`, ce qui met fin à l'application. Cela peut être acceptable, mais un message d'erreur plus explicite (ex: avec `tracing::error`) aiderait au débogage.
metrics::install(metric_bind_addr)?;
}
let gitea_api = GiteaAPI::new(&config.gitea_url, &config.gitea_token, config.gitea_timeout)?;
let gitea_user = gitea_api.get_authorized_user().await?;
+62
View File
@@ -0,0 +1,62 @@
use std::{net::SocketAddr, str::FromStr};
use metrics::{Unit, describe_counter, describe_gauge, counter, gauge};
pub fn webhook_received(event_type: &str) {
counter!("herald_webhooks_received_total", "event_type" => event_type.to_string())
.increment(1);
}
pub fn webhook_duplicate(event_type: &str) {
counter!("herald_webhooks_duplicate_total", "event_type" => event_type.to_string())
.increment(1);
}
pub fn webhook_channel_full(event_type: &str) {
counter!("herald_webhooks_channel_full_total", "event_type" => event_type.to_string())
.increment(1);
}
pub fn increment_task_active() {
gauge!("herald_bot_tasks_active").increment(1.0);
}
pub fn decrement_task_active() {
gauge!("herald_bot_tasks_active").decrement(1.0);
}
pub fn task_completed(event_type: &str) {
counter!("herald_bot_tasks_completed_total", "event_type" => event_type.to_string())
.increment(1);
}
pub fn task_failed(event_type: &str) {
counter!("herald_bot_tasks_failed_total", "event_type" => event_type.to_string())
.increment(1);
}
pub fn openrouter_cost_usd(cost: u64) {
qpismont marked this conversation as resolved Outdated
Outdated
Review

La métrique herald_openrouter_cost_usd_total utilise un compteur de type Count, mais elle représente un coût en dollars. Bien que cela soit fonctionnel, il serait plus clair d'utiliser un type Gauge ou d'ajouter une unité personnalisée si possible. Actuellement l'unité Count peut prêter à confusion pour les opérateurs.

La métrique `herald_openrouter_cost_usd_total` utilise un compteur de type `Count`, mais elle représente un coût en dollars. Bien que cela soit fonctionnel, il serait plus clair d'utiliser un type `Gauge` ou d'ajouter une unité personnalisée si possible. Actuellement l'unité `Count` peut prêter à confusion pour les opérateurs.
counter!("herald_openrouter_cost_usd_total").increment(cost);
}
pub fn describe() {
describe_counter!("herald_webhooks_received_total", Unit::Count, "Total webhooks received");
describe_counter!("herald_webhooks_duplicate_total", Unit::Count, "Webhooks rejected as duplicates");
describe_counter!("herald_webhooks_channel_full_total", Unit::Count, "Webhooks dropped because the bot channel was full");
describe_gauge!("herald_bot_tasks_active", Unit::Count, "Bot tasks currently in progress");
describe_counter!("herald_bot_tasks_completed_total", Unit::Count, "Bot tasks completed successfully");
describe_counter!("herald_bot_tasks_failed_total", Unit::Count, "Bot tasks that failed");
describe_counter!("herald_openrouter_cost_usd_total", Unit::Count, "Total OpenRouter cost in USD");
}
pub fn install(bind_addr: &str) -> anyhow::Result<()> {
describe();
let builder = metrics_exporter_prometheus::PrometheusBuilder::new();
builder
.with_http_listener(SocketAddr::from_str(bind_addr)?)
.install()?;
tracing::info!(bind_addr, "Prometheus metrics exporter installed");
Ok(())
}