From 6ffd88927c79b9b73ac92f6aafbaa971aeb12ef4 Mon Sep 17 00:00:00 2001 From: qpismont Date: Sun, 26 Jul 2026 11:02:34 +0000 Subject: [PATCH] add prometheus metrics --- .env.example | 3 + Cargo.lock | 236 ++++++++++++++++++++++++++++++++++++-- Cargo.toml | 6 +- src/api.rs | 6 + src/bot.rs | 13 ++- src/bot_actions/review.rs | 8 +- src/env.rs | 3 + src/gitea.rs | 6 + src/main.rs | 5 + src/metrics.rs | 62 ++++++++++ 10 files changed, 332 insertions(+), 16 deletions(-) create mode 100644 src/metrics.rs diff --git a/.env.example b/.env.example index 9478415..06a8c62 100644 --- a/.env.example +++ b/.env.example @@ -17,3 +17,6 @@ GITEA_TIMEOUT=30 SENTRY_DSN= RUST_LOG=info RUST_BACKTRACE=1 + + +METRICS_BIND_ADDR= \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index fc3fa28..d88ac2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,7 +34,7 @@ dependencies = [ "bytestring", "derive_more", "encoding_rs", - "foldhash", + "foldhash 0.1.5", "futures-core", "http 0.2.12", "httparse", @@ -129,7 +129,7 @@ dependencies = [ "cfg-if", "derive_more", "encoding_rs", - "foldhash", + "foldhash 0.1.5", "futures-core", "futures-util", "impl-more", @@ -357,6 +357,21 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "darling" version = "0.20.11" @@ -545,6 +560,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "evmap" +version = "11.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8874945f036109c72242964c1174cf99434e30cfa45bf45fedc983f50046f8" +dependencies = [ + "hashbag", + "left-right", + "smallvec", +] + [[package]] name = "fastrand" version = "2.4.1" @@ -581,6 +607,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -666,6 +698,21 @@ dependencies = [ "slab", ] +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -718,6 +765,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbag" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064" + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -734,6 +796,8 @@ dependencies = [ "dotenvy", "futures-util", "hex", + "metrics", + "metrics-exporter-prometheus", "openrouter-rs", "reqwest 0.12.28", "ring", @@ -1023,7 +1087,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.17.1", ] [[package]] @@ -1062,6 +1126,17 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "left-right" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc015ded5d9b3054dbbdb63332cdd6ee42352ccef19e911e25117490e2f48ee" +dependencies = [ + "crossbeam-utils", + "loom", + "slab", +] + [[package]] name = "libc" version = "0.2.186" @@ -1101,6 +1176,19 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -1128,12 +1216,70 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108" +dependencies = [ + "base64", + "evmap", + "http-body-util", + "hyper", + "hyper-util", + "indexmap", + "ipnet", + "metrics", + "metrics-util", + "quanta", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-util" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96f8722f8562635f92f8ed992f26df0532266eb03d5202607c20c0d7e9745e13" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.16.1", + "metrics", + "quanta", + "rand", + "rand_xoshiro", + "rapidhash", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1375,9 +1521,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "openrouter-rs" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7101578df2f54d9013594e94367adbe656521ef6002f03e4577f1e00e57cb4" +checksum = "25918cfeef40cfd3adea348b268496ee5206eb0408f2aacfe4bdfef2bfbe3bf9" dependencies = [ "derive_builder", "dotenvy_macro", @@ -1522,6 +1668,12 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "portable-atomic" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" + [[package]] name = "potential_utf" version = "0.1.5" @@ -1555,6 +1707,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quinn" version = "0.11.9" @@ -1654,6 +1821,33 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rapidhash" +version = "4.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1736,6 +1930,7 @@ dependencies = [ "hyper-util", "js-sys", "log", + "mime_guess", "percent-encoding", "pin-project-lite", "quinn", @@ -1927,6 +2122,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -2213,6 +2414,12 @@ dependencies = [ "libc", ] +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + [[package]] name = "slab" version = "0.4.12" @@ -2425,9 +2632,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -2619,6 +2826,12 @@ dependencies = [ "libc", ] +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -2884,6 +3097,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index 4192d13..c92a520 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ debug = 1 [dependencies] reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } -tokio = { version = "1.52", features = ["full"] } +tokio = { version = "1.53", features = ["full"] } tokio-stream = "0.1" tokio-util = "0.7" futures-util = "0.3" @@ -16,7 +16,7 @@ serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } sentry = { version = "0.48", features = ["tower-axum-matched-path"] } sentry-anyhow = { version = "0.48", features = ["backtrace"] } -openrouter-rs = "0.10" +openrouter-rs = "0.12" dotenvy = "0.15" tower = "0.5" tower-http = {version = "0.6", features = ["trace"] } @@ -28,3 +28,5 @@ thiserror = "2.0" ring = "0.17" hex = "0.4" bytes = "1.1" +metrics = "0.24" +metrics-exporter-prometheus = { version = "0.18", default-features = false, features = ["http-listener"] } diff --git a/src/api.rs b/src/api.rs index 13e3505..1b5a436 100644 --- a/src/api.rs +++ b/src/api.rs @@ -17,6 +17,7 @@ use tokio_util::sync::CancellationToken; use crate::consts::{GITEA_EVENT_TYPE_HEADER_NAME, GITEA_SIG_HEADER_NAME, MAX_WEBHOOK_BODY_SIZE}; use crate::errors::AppError; use crate::gitea::WebhookType; +use crate::metrics; use crate::state::AppState; pub async fn start(app_state: AppState, shutdown: CancellationToken) -> anyhow::Result<()> { @@ -56,13 +57,18 @@ async fn webhook( ) -> Result { tracing::Span::current().record("webhook_type", tracing::field::debug(&wb)); + let event_type = wb.event_type_str(); + metrics::webhook_received(event_type); + let event_id = wb.event_id(); if app_state.bot.check_and_mark(event_id).await { + metrics::webhook_duplicate(event_type); return Err(AppError::AlreadyProcessedErr); } if app_state.bot_tx.try_send(wb).is_err() { app_state.bot.unmark(event_id).await; + metrics::webhook_channel_full(event_type); return Err(AppError::ChannelFullErr); } diff --git a/src/bot.rs b/src/bot.rs index 55e0cab..470990f 100644 --- a/src/bot.rs +++ b/src/bot.rs @@ -1,6 +1,5 @@ use crate::{ - gitea::{GiteaAPI, WebhookType}, - open_router::OpenRouterClient, + gitea::{GiteaAPI, WebhookType}, metrics, open_router::OpenRouterClient, }; use serde::Deserialize; use std::{collections::HashSet, sync::Arc}; @@ -87,10 +86,12 @@ impl Bot { info!(queued = rx.len(), active = tasks.len(), "Webhook received"); let permit = sem.clone().acquire_owned().await?; let self_clone = self.clone(); + metrics::increment_task_active(); tasks.spawn(async move { self_clone.exec(wb).await; drop(permit); + metrics::decrement_task_active(); }); } @@ -102,6 +103,8 @@ impl Bot { #[instrument(skip(self, webhook), fields(repo, pr))] pub async fn exec(&self, webhook: WebhookType) { + let event_type_str = webhook.event_type_str(); + match &webhook { WebhookType::Review(p) => { tracing::Span::current().record("repo", &p.repository.full_name); @@ -121,8 +124,12 @@ impl Bot { .await; match exec_result { - Ok(_) => info!("Task completed"), + Ok(_) => { + metrics::task_completed(event_type_str); + info!("Task completed"); + } Err(err) => { + metrics::task_failed(event_type_str); error!(%err, "Task error"); sentry_anyhow::capture_anyhow(&err); } diff --git a/src/bot_actions/review.rs b/src/bot_actions/review.rs index 4bf840f..534dd3d 100644 --- a/src/bot_actions/review.rs +++ b/src/bot_actions/review.rs @@ -4,10 +4,7 @@ use tokio_util::io::StreamReader; use tracing::instrument; use crate::{ - bot::ReviewResult, - consts::{BOT_PROCESS_MSG, MAX_DIFF_SIZE, REVIEW_PROMPT}, - gitea::{GiteaAPI, ReviewPayload}, - open_router::OpenRouterClient, + bot::ReviewResult, consts::{BOT_PROCESS_MSG, MAX_DIFF_SIZE, REVIEW_PROMPT}, gitea::{GiteaAPI, ReviewPayload}, metrics, open_router::OpenRouterClient, }; #[instrument(skip(gitea_api, open_router_client, http_client, review_payload))] @@ -48,6 +45,9 @@ pub async fn exec_review( let mut review_result = serde_json::from_str::(&chat_result.message)?; review_result.cost = chat_result.cost; + if let Some(cost) = review_result.cost { + metrics::openrouter_cost_usd((cost / 100.0).round() as u64); + } let final_review_markdown = review_result_to_markdown(&review_result); diff --git a/src/env.rs b/src/env.rs index 4a558a6..8aed750 100644 --- a/src/env.rs +++ b/src/env.rs @@ -11,6 +11,7 @@ pub struct EnvConfig { pub gitea_url: String, pub gitea_token: String, pub gitea_timeout: u64, + pub metrics_bind_addr: Option, } pub fn load_config() -> anyhow::Result { @@ -23,6 +24,7 @@ pub fn load_config() -> anyhow::Result { let gitea_url = try_get_env("GITEA_URL")?; let gitea_token = try_get_env("GITEA_TOKEN")?; let gitea_timeout = try_get_env("GITEA_TIMEOUT")?.parse()?; + let metrics_bind_addr = std::env::var("METRICS_BIND_ADDR").ok(); Ok(EnvConfig { http_port, @@ -34,6 +36,7 @@ pub fn load_config() -> anyhow::Result { gitea_url, gitea_token, gitea_timeout, + metrics_bind_addr, }) } diff --git a/src/gitea.rs b/src/gitea.rs index 2459a8e..8477b4b 100644 --- a/src/gitea.rs +++ b/src/gitea.rs @@ -179,6 +179,12 @@ impl WebhookType { WebhookType::Review(payload) => payload.comment.id, } } + + pub fn event_type_str(&self) -> &'static str { + match self { + WebhookType::Review(_) => "review", + } + } } #[derive(Deserialize, Debug)] diff --git a/src/main.rs b/src/main.rs index 34d92fe..27d0554 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,6 +13,7 @@ mod consts; mod env; mod errors; mod gitea; +mod metrics; mod open_router; mod state; @@ -49,6 +50,10 @@ fn main() -> anyhow::Result<()> { async fn run() -> anyhow::Result<()> { let config = env::load_config()?; + if let Some(metric_bind_addr) = &config.metrics_bind_addr { + metrics::install(metric_bind_addr)?; + } + let gitea_api = GiteaAPI::new(&config.gitea_url, &config.gitea_token, config.gitea_timeout)?; let gitea_user = gitea_api.get_authorized_user().await?; diff --git a/src/metrics.rs b/src/metrics.rs new file mode 100644 index 0000000..38c35d9 --- /dev/null +++ b/src/metrics.rs @@ -0,0 +1,62 @@ +use std::{net::SocketAddr, str::FromStr}; + +use metrics::{Unit, describe_counter, describe_gauge, counter, gauge}; + +pub fn webhook_received(event_type: &str) { + counter!("herald_webhooks_received_total", "event_type" => event_type.to_string()) + .increment(1); +} + +pub fn webhook_duplicate(event_type: &str) { + counter!("herald_webhooks_duplicate_total", "event_type" => event_type.to_string()) + .increment(1); +} + +pub fn webhook_channel_full(event_type: &str) { + counter!("herald_webhooks_channel_full_total", "event_type" => event_type.to_string()) + .increment(1); +} + +pub fn increment_task_active() { + gauge!("herald_bot_tasks_active").increment(1.0); +} + +pub fn decrement_task_active() { + gauge!("herald_bot_tasks_active").decrement(1.0); +} + +pub fn task_completed(event_type: &str) { + counter!("herald_bot_tasks_completed_total", "event_type" => event_type.to_string()) + .increment(1); +} + +pub fn task_failed(event_type: &str) { + counter!("herald_bot_tasks_failed_total", "event_type" => event_type.to_string()) + .increment(1); +} + +pub fn openrouter_cost_usd(cost: u64) { + counter!("herald_openrouter_cost_usd_total").increment(cost); +} + +pub fn describe() { + describe_counter!("herald_webhooks_received_total", Unit::Count, "Total webhooks received"); + describe_counter!("herald_webhooks_duplicate_total", Unit::Count, "Webhooks rejected as duplicates"); + describe_counter!("herald_webhooks_channel_full_total", Unit::Count, "Webhooks dropped because the bot channel was full"); + describe_gauge!("herald_bot_tasks_active", Unit::Count, "Bot tasks currently in progress"); + describe_counter!("herald_bot_tasks_completed_total", Unit::Count, "Bot tasks completed successfully"); + describe_counter!("herald_bot_tasks_failed_total", Unit::Count, "Bot tasks that failed"); + describe_counter!("herald_openrouter_cost_usd_total", Unit::Count, "Total OpenRouter cost in USD"); +} + +pub fn install(bind_addr: &str) -> anyhow::Result<()> { + describe(); + + let builder = metrics_exporter_prometheus::PrometheusBuilder::new(); + builder + .with_http_listener(SocketAddr::from_str(bind_addr)?) + .install()?; + + tracing::info!(bind_addr, "Prometheus metrics exporter installed"); + Ok(()) +}