Postgres persistance + job queue + api with sse #9

Merged
qpismont merged 8 commits from 1.3 into main 2026-10-26 21:19:32 +01:00
45 changed files with 2139 additions and 1880 deletions
Showing only changes of commit 3078ba5705 - Show all commits

No files matched your search

-1
View File
@@ -11,7 +11,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
curl \ curl \
git \ git \
build-essential \ build-essential \
libssl-dev \
cmake \ cmake \
pkg-config \ pkg-config \
clang \ clang \
+6
View File
@@ -23,6 +23,12 @@ steps:
- rustup component add clippy - rustup component add clippy
- cargo clippy --workspace --all-targets --all-features -- -D warnings - cargo clippy --workspace --all-targets --all-features -- -D warnings
- name: deny
image: rust:1.99
commands:
- cargo install cargo-deny --locked --version 0.18.9
Review

[performance] cargo install cargo-deny --locked recompile l'outil à chaque pipeline, ce qui ajoute plusieurs minutes de CI. Il vaut mieux utiliser une image préconstruite (ghcr.io/embarkstudios/cargo-deny) ou un binaire précompilé. Par ailleurs les étapes CI utilisent rust:1.99 alors que le Containerfile build avec rust:1.98-trixie : les versions de toolchain divergent.

**[performance]** `cargo install cargo-deny --locked` recompile l'outil à chaque pipeline, ce qui ajoute plusieurs minutes de CI. Il vaut mieux utiliser une image préconstruite (`ghcr.io/embarkstudios/cargo-deny`) ou un binaire précompilé. Par ailleurs les étapes CI utilisent `rust:1.99` alors que le `Containerfile` build avec `rust:1.98-trixie` : les versions de toolchain divergent.
- cargo deny check
- name: wait-for-postgres - name: wait-for-postgres
image: postgres:18 image: postgres:18
commands: commands:
Generated
+15 -93
View File
@@ -1392,21 +1392,6 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
[[package]]
name = "foreign-types"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
dependencies = [
"foreign-types-shared",
]
[[package]]
name = "foreign-types-shared"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]] [[package]]
name = "form_urlencoded" name = "form_urlencoded"
version = "1.2.2" version = "1.2.2"
@@ -1811,18 +1796,17 @@ dependencies = [
] ]
[[package]] [[package]]
name = "hyper-tls" name = "hyper-rustls"
version = "0.6.0" version = "0.27.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53"
dependencies = [ dependencies = [
"bytes", "http 1.5.0",
"http-body-util",
"hyper", "hyper",
"hyper-util", "hyper-util",
"native-tls", "rustls",
"tokio", "tokio",
"tokio-native-tls", "tokio-rustls",
"tower-service", "tower-service",
] ]
@@ -2579,23 +2563,6 @@ dependencies = [
"windows-sys 0.61.2", "windows-sys 0.61.2",
] ]
[[package]]
name = "native-tls"
version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2"
dependencies = [
"libc",
"log",
"openssl",
"openssl-probe",
"openssl-sys",
"schannel",
"security-framework",
"security-framework-sys",
"tempfile",
]
[[package]] [[package]]
name = "nix" name = "nix"
version = "0.31.3" version = "0.31.3"
@@ -2856,49 +2823,12 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
[[package]]
name = "openssl"
version = "0.10.81"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45"
dependencies = [
"bitflags 2.13.1",
"cfg-if",
"foreign-types",
"libc",
"openssl-macros",
"openssl-sys",
]
[[package]]
name = "openssl-macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]] [[package]]
name = "openssl-probe" name = "openssl-probe"
version = "0.2.1" version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
[[package]]
name = "openssl-sys"
version = "0.9.117"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695"
dependencies = [
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]] [[package]]
name = "os_info" name = "os_info"
version = "3.15.0" version = "3.15.0"
@@ -3444,19 +3374,20 @@ dependencies = [
"http-body", "http-body",
"http-body-util", "http-body-util",
"hyper", "hyper",
"hyper-tls", "hyper-rustls",
"hyper-util", "hyper-util",
"js-sys", "js-sys",
"log", "log",
"native-tls",
"percent-encoding", "percent-encoding",
"pin-project-lite", "pin-project-lite",
"rustls",
"rustls-pki-types", "rustls-pki-types",
"rustls-platform-verifier",
"serde", "serde",
"serde_json", "serde_json",
"sync_wrapper", "sync_wrapper",
"tokio", "tokio",
"tokio-native-tls", "tokio-rustls",
"tokio-util", "tokio-util",
"tower", "tower",
"tower-http 0.6.11", "tower-http 0.6.11",
@@ -3604,6 +3535,7 @@ version = "0.23.45"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
dependencies = [ dependencies = [
"log",
"once_cell", "once_cell",
"ring", "ring",
"rustls-pki-types", "rustls-pki-types",
@@ -3759,8 +3691,8 @@ checksum = "76a88b65feb368d9dde5d531a2b59b25016e36fd6e4978432371a3ee77746ca2"
dependencies = [ dependencies = [
"cfg_aliases", "cfg_aliases",
"httpdate", "httpdate",
"native-tls",
"reqwest", "reqwest",
"rustls",
"sentry-actix", "sentry-actix",
"sentry-backtrace", "sentry-backtrace",
"sentry-contexts", "sentry-contexts",
@@ -4281,6 +4213,7 @@ dependencies = [
"metrics-exporter-prometheus", "metrics-exporter-prometheus",
"reqwest", "reqwest",
"ring", "ring",
"rustls",
"sentry", "sentry",
"sentry-anyhow", "sentry-anyhow",
"serde", "serde",
@@ -4721,16 +4654,6 @@ dependencies = [
"syn 3.0.4", "syn 3.0.4",
] ]
[[package]]
name = "tokio-native-tls"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
dependencies = [
"native-tls",
"tokio",
]
[[package]] [[package]]
name = "tokio-postgres" name = "tokio-postgres"
version = "0.7.18" version = "0.7.18"
@@ -5318,14 +5241,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d" checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d"
dependencies = [ dependencies = [
"base64 0.23.1", "base64 0.23.1",
"der 0.8.2",
"log", "log",
"native-tls",
"percent-encoding", "percent-encoding",
"rustls",
"rustls-pki-types", "rustls-pki-types",
"ureq-proto", "ureq-proto",
"utf8-zero", "utf8-zero",
"webpki-root-certs", "webpki-roots",
] ]
[[package]] [[package]]
+22 -2
View File
@@ -6,14 +6,31 @@ members = [
resolver = "3" resolver = "3"
[workspace.dependencies] [workspace.dependencies]
reqwest = { version = "0.13", default-features = false, features = ["json", "native-tls", "stream"] } # TLS is pure Rust (rustls + ring): no libssl is needed at build or run time.
# `rustls-no-provider` leaves the crypto provider to the binary, which installs
# `ring` at startup (see `main`), so that a single provider is ever compiled in.
reqwest = { version = "0.13", default-features = false, features = ["json", "rustls-no-provider", "stream"] }
rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] }
tokio = { version = "1.53", features = ["full"] } tokio = { version = "1.53", features = ["full"] }
tokio-stream = "0.1" tokio-stream = "0.1"
tokio-util = "0.7" tokio-util = "0.7"
futures-util = "0.3" futures-util = "0.3"
serde_json = "1.0" serde_json = "1.0"
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
sentry = { version = "0.49", features = ["tower-axum-matched-path"] } # Sentry's default `transport` feature pulls `native-tls`; the same set of features
# is spelled out here with rustls instead.
sentry = { version = "0.49", default-features = false, features = [
"backtrace",
"contexts",
"debug-images",
"logs",
"metrics",
"panic",
"release-health",
"reqwest",
"rustls-no-provider",
"tower-axum-matched-path",
] }
sentry-anyhow = { version = "0.49", features = ["backtrace"] } sentry-anyhow = { version = "0.49", features = ["backtrace"] }
dotenvy = "0.15" dotenvy = "0.15"
tower = "0.5" tower = "0.5"
@@ -31,6 +48,9 @@ metrics-exporter-prometheus = { version = "0.18", default-features = false, feat
toasty = { version = "0.11", features = ["postgresql", "serde", "jiff"] } toasty = { version = "0.11", features = ["postgresql", "serde", "jiff"] }
toasty-cli = "0.11" toasty-cli = "0.11"
jiff = { version = "0.2", features = ["serde"] } jiff = { version = "0.2", features = ["serde"] }
tempfile = "3"
bollard = "0.21"
tar = "0.4"
[profile.release] [profile.release]
debug = 1 debug = 1
-3
View File
@@ -14,9 +14,6 @@ FROM debian:trixie-slim
RUN apt-get update && apt-get install -y --no-install-recommends \ RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \ ca-certificates \
git \ git \
libssl3t64 \
libzstd1 \
zlib1g \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
+19 -1
View File
@@ -42,7 +42,7 @@ Steep reads its configuration from environment variables (a `.env` file is suppo
| `METRICS_BIND_ADDR` | *(optional)* Bind address for the Prometheus metrics endpoint (e.g. `0.0.0.0:9100`). If unset, the metrics exporter is disabled. | | `METRICS_BIND_ADDR` | *(optional)* Bind address for the Prometheus metrics endpoint (e.g. `0.0.0.0:9100`). If unset, the metrics exporter is disabled. |
| `SENTRY_DSN` | *(optional)* Sentry DSN for error tracking | | `SENTRY_DSN` | *(optional)* Sentry DSN for error tracking |
| `RUST_LOG` | *(optional)* Log level, defaults to `info` | | `RUST_LOG` | *(optional)* Log level, defaults to `info` |
| `SANDBOX_MAX_ITERATIONS` | *(optional)* Maximum number of tool-calling iterations per sandboxed review. Defaults to `8` | | `SANDBOX_MAX_ITERATIONS` | *(optional)* Maximum number of tool-calling iterations per sandboxed review. Defaults to `8`; a value that is set but is not a number stops the startup |
| `DOCKER_HOST` | *(optional)* Container daemon socket Steep drives, e.g. `unix:///run/user/1000/podman/podman.sock` for a rootless podman. Defaults to `unix:///var/run/docker.sock` | | `DOCKER_HOST` | *(optional)* Container daemon socket Steep drives, e.g. `unix:///run/user/1000/podman/podman.sock` for a rootless podman. Defaults to `unix:///var/run/docker.sock` |
## Sandboxed reviews ## Sandboxed reviews
@@ -139,6 +139,24 @@ cargo run -p steep-server
Copy `.env.example` to `.env` and fill in your values before running. Copy `.env.example` to `.env` and fill in your values before running.
## Job queue
Steep is meant to run as a **single instance**. Webhooks are stored in the `jobs` table
of PostgreSQL, and the worker is woken up as soon as a job is pushed (no polling).
A job still `processing` at startup was interrupted by the previous process: it is put
back in the queue and reviewed again. Do not run two instances on the same database,
they would both take over each other's running jobs.
## Dependencies
TLS is handled by [rustls](https://github.com/rustls/rustls) (with `ring`), so neither
the build nor the image needs OpenSSL. The `Containerfile` runtime image only
installs `ca-certificates` and `git`.
`cargo deny check` audits the dependencies (security advisories, licenses, banned
crates, sources) with the policy of `deny.toml`. It runs in the CI, and it also fails
if `openssl-sys` or `native-tls` comes back into the dependency tree.
## Metrics ## Metrics
Steep optionally exposes a Prometheus metrics endpoint, useful for scraping with an [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) or any Prometheus-compatible scraper. Steep optionally exposes a Prometheus metrics endpoint, useful for scraping with an [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) or any Prometheus-compatible scraper.
+4 -3
View File
@@ -2,12 +2,13 @@
name = "devcontainer-rs" name = "devcontainer-rs"
version = "0.1.0" version = "0.1.0"
edition = "2024" edition = "2024"
publish = false
[dependencies] [dependencies]
bollard = "0.21" bollard = { workspace = true }
bytes = { workspace = true } bytes = { workspace = true }
futures-util = { workspace = true } futures-util = { workspace = true }
tar = "0.4" tar = { workspace = true }
tokio = { workspace = true } tokio = { workspace = true }
tokio-stream = { workspace = true } tokio-stream = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
@@ -17,4 +18,4 @@ anyhow = { workspace = true }
thiserror = { workspace = true } thiserror = { workspace = true }
[dev-dependencies] [dev-dependencies]
tempfile = "3" tempfile = { workspace = true }
+8 -9
View File
@@ -1,25 +1,24 @@
//! Constantes de la crate : endpoint du daemon, timeouts et découpage du //! Crate constants: daemon endpoint, timeouts and build context chunking.
//! contexte de build.
use std::time::Duration; use std::time::Duration;
/// Endpoint affiché dans les logs quand `DOCKER_HOST` n'est pas défini. /// Endpoint shown in the logs when `DOCKER_HOST` is not set.
pub(crate) const DEFAULT_ENDPOINT: &str = "unix:///var/run/docker.sock"; pub(crate) const DEFAULT_ENDPOINT: &str = "unix:///var/run/docker.sock";
/// Timeout appliqué aux opérations de build/run/stop/remove. /// Timeout applied to build/run/stop/remove operations.
pub(crate) const DEFAULT_COMMAND_TIMEOUT: Duration = Duration::from_secs(600); pub(crate) const DEFAULT_COMMAND_TIMEOUT: Duration = Duration::from_secs(600);
/// Timeout appliqué aux commandes exécutées dans un container en cours d'exécution. /// Timeout applied to commands run inside a running container.
pub(crate) const DEFAULT_EXEC_TIMEOUT: Duration = Duration::from_secs(60); pub(crate) const DEFAULT_EXEC_TIMEOUT: Duration = Duration::from_secs(60);
/// Taille des morceaux du contexte de build envoyés au daemon. /// Size of the build context chunks sent to the daemon.
pub(crate) const CONTEXT_CHUNK_SIZE: usize = 64 * 1024; pub(crate) const CONTEXT_CHUNK_SIZE: usize = 64 * 1024;
/// Nombre de morceaux pouvant attendre dans le canal : c'est ce qui borne la /// Number of chunks that can wait in the channel: this is what bounds the
/// mémoire occupée par le contexte, quelle que soit sa taille. /// memory held by the context, regardless of its size.
pub(crate) const CONTEXT_CHUNKS: usize = 4; pub(crate) const CONTEXT_CHUNKS: usize = 4;
/// Lectures successives du code de sortie d'un exec, et attente entre elles. /// Successive reads of an exec's exit code, and the delay between them.
pub(crate) const EXIT_CODE_ATTEMPTS: usize = 10; pub(crate) const EXIT_CODE_ATTEMPTS: usize = 10;
pub(crate) const EXIT_CODE_DELAY: Duration = Duration::from_millis(20); pub(crate) const EXIT_CODE_DELAY: Duration = Duration::from_millis(20);
+11 -11
View File
@@ -1,4 +1,4 @@
//! Un devcontainer en cours d'exécution : inspection et exécution de commandes. //! A running devcontainer: inspection and command execution.
use std::time::Duration; use std::time::Duration;
@@ -7,7 +7,7 @@ use crate::{
runtime::ContainerRuntime, runtime::ContainerRuntime,
}; };
/// Un devcontainer en cours d'exécution. /// A running devcontainer.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct Container { pub struct Container {
runtime: ContainerRuntime, runtime: ContainerRuntime,
@@ -19,8 +19,8 @@ pub struct Container {
} }
impl Container { impl Container {
/// Assemble un container démarré, avec son réseau et, le cas échéant, l'image /// Assembles a started container, with its network and, if any, the image
/// construite pour lui et à nettoyer. /// built for it that must be cleaned up.
pub(crate) fn new( pub(crate) fn new(
runtime: ContainerRuntime, runtime: ContainerRuntime,
name: String, name: String,
@@ -47,10 +47,10 @@ impl Container {
&self.workspace_folder &self.workspace_folder
} }
/// Exécute une commande dans le container et renvoie sa sortie. /// Runs a command in the container and returns its output.
/// ///
/// La commande est transmise sous forme de vecteur d'arguments (argv, sans shell), /// The command is passed as an argument vector (argv, no shell), so no
/// donc aucun échappement ni interpolation n'est effectué. /// escaping or interpolation is performed.
pub async fn exec(&self, cmd: &[&str]) -> Result<ExecOutput, ContainerError> { pub async fn exec(&self, cmd: &[&str]) -> Result<ExecOutput, ContainerError> {
self.exec_with_timeout(cmd, DEFAULT_EXEC_TIMEOUT).await self.exec_with_timeout(cmd, DEFAULT_EXEC_TIMEOUT).await
} }
@@ -65,19 +65,19 @@ impl Container {
.await .await
} }
/// Exécute un script shell dans le container via `sh -c`. /// Runs a shell script in the container via `sh -c`.
pub async fn exec_shell(&self, script: &str) -> Result<ExecOutput, ContainerError> { pub async fn exec_shell(&self, script: &str) -> Result<ExecOutput, ContainerError> {
self.exec(&["sh", "-c", script]).await self.exec(&["sh", "-c", script]).await
} }
/// Arrête le container. /// Stops the container.
pub async fn stop(&self) -> Result<(), ContainerError> { pub async fn stop(&self) -> Result<(), ContainerError> {
self.runtime.stop_container(&self.name).await self.runtime.stop_container(&self.name).await
} }
/// Supprime le container, son réseau et son image. /// Removes the container, its network and its image.
/// ///
/// La suppression du réseau et de l'image est best-effort : ils peuvent déjà être absents. /// Removing the network and the image is best-effort: they may already be gone.
pub async fn remove(&self) -> Result<(), ContainerError> { pub async fn remove(&self) -> Result<(), ContainerError> {
self.runtime.remove_container(&self.name).await?; self.runtime.remove_container(&self.name).await?;
+14 -14
View File
@@ -1,4 +1,4 @@
//! Empaquetage en flux du contexte de build envoyé au daemon. //! Streaming packaging of the build context sent to the daemon.
use std::{ use std::{
io::{BufWriter, Write}, io::{BufWriter, Write},
@@ -12,13 +12,13 @@ use tokio_stream::wrappers::ReceiverStream;
use crate::consts::{CONTEXT_CHUNK_SIZE, CONTEXT_CHUNKS}; use crate::consts::{CONTEXT_CHUNK_SIZE, CONTEXT_CHUNKS};
/// Empaquette le contexte de build dans un tar, **en flux**. /// Packs the build context into a tar, **as a stream**.
/// ///
/// Le CLI `docker build` empaquette le contexte puis l'envoie ; le faire d'un coup /// The `docker build` CLI packs the context and then sends it; doing that in one go
/// chargerait tout le dossier en mémoire, ce qui devient intenable dès que le /// would load the whole folder into memory, which becomes untenable as soon as the
/// `.devcontainer` embarque des binaires. Ici l'archive est écrite par une tâche /// `.devcontainer` ships binaries. Here the archive is written by a blocking task
/// bloquante et poussée morceau par morceau, avec la contre-pression du canal : /// and pushed chunk by chunk, with the channel's backpressure: if the daemon reads
/// si le daemon lit lentement, l'empaquetage ralentit. /// slowly, packing slows down.
pub(crate) fn tar_directory_stream( pub(crate) fn tar_directory_stream(
dir: &Path, dir: &Path,
) -> impl Stream<Item = Result<Bytes, std::io::Error>> + Send + 'static { ) -> impl Stream<Item = Result<Bytes, std::io::Error>> + Send + 'static {
@@ -33,13 +33,13 @@ pub(crate) fn tar_directory_stream(
let result = builder let result = builder
.append_dir_all(".", &dir) .append_dir_all(".", &dir)
.and_then(|()| builder.finish()) .and_then(|()| builder.finish())
// `finish` écrit les blocs de fin d'archive ; il reste à vider le tampon // `finish` writes the end-of-archive blocks; the buffer still has to be
// pour que le daemon reçoive tout, y compris une archive vide. // flushed so the daemon receives everything, even an empty archive.
.and_then(|()| builder.get_mut().flush()); .and_then(|()| builder.get_mut().flush());
if let Err(error) = result { if let Err(error) = result {
// L'échec est remonté au daemon par le flux : sinon il ne verrait qu'un // The failure is reported to the daemon through the stream: otherwise it
// tar tronqué et signalerait une erreur incompréhensible. // would only see a truncated tar and report a confusing error.
let _ = failures.blocking_send(Err(error)); let _ = failures.blocking_send(Err(error));
} }
}); });
@@ -47,9 +47,9 @@ pub(crate) fn tar_directory_stream(
ReceiverStream::new(receiver) ReceiverStream::new(receiver)
} }
/// Écrit les morceaux du tar dans un canal, en attendant qu'il se vide. /// Writes the tar chunks into a channel, waiting for it to drain.
/// ///
/// Utilisé depuis une tâche bloquante, seul contexte où `blocking_send` est autorisé. /// Used from a blocking task, the only context where `blocking_send` is allowed.
struct ChannelWriter { struct ChannelWriter {
sender: mpsc::Sender<Result<Bytes, std::io::Error>>, sender: mpsc::Sender<Result<Bytes, std::io::Error>>,
} }
@@ -80,7 +80,7 @@ mod tests {
use futures_util::StreamExt; use futures_util::StreamExt;
/// Rassemble le flux du contexte en un tar complet, pour l'inspecter. /// Collects the context stream into a complete tar, for inspection.
async fn context_tar(dir: &Path) -> Vec<u8> { async fn context_tar(dir: &Path) -> Vec<u8> {
let mut tar = Vec::new(); let mut tar = Vec::new();
let mut chunks = tar_directory_stream(dir); let mut chunks = tar_directory_stream(dir);
+9 -9
View File
@@ -1,5 +1,5 @@
//! Représentation analysée d'un `devcontainer.json` et son cycle de vie : //! Parsed representation of a `devcontainer.json` and its lifecycle: naming,
//! nommage, build, démarrage et hooks. //! build, startup and hooks.
use std::{ use std::{
collections::HashMap, collections::HashMap,
@@ -29,29 +29,29 @@ pub struct DevContainer {
} }
impl DevContainer { impl DevContainer {
/// Dossier de workspace dans le container, par défaut `/workspace`. /// Workspace folder inside the container, `/workspace` by default.
pub fn workspace_folder(&self) -> String { pub fn workspace_folder(&self) -> String {
self.workspace_folder self.workspace_folder
.clone() .clone()
.unwrap_or_else(|| "/workspace".to_string()) .unwrap_or_else(|| "/workspace".to_string())
} }
/// Tag de l'image construite localement à partir du Dockerfile. /// Tag of the image built locally from the Dockerfile.
fn local_image_tag(&self, id: &str) -> String { fn local_image_tag(&self, id: &str) -> String {
format!("devcontainer-rs/{}:{}", self.sanitized_name(), id) format!("devcontainer-rs/{}:{}", self.sanitized_name(), id)
} }
/// Nom du container de cette sandbox. /// Name of this sandbox's container.
fn container_name(&self, id: &str) -> String { fn container_name(&self, id: &str) -> String {
format!("devcontainer-rs-{}-{}", self.sanitized_name(), id) format!("devcontainer-rs-{}-{}", self.sanitized_name(), id)
} }
/// Nom du devcontainer nettoyé pour servir de nom d'image/container. /// Devcontainer name, sanitized to be usable as an image/container name.
fn sanitized_name(&self) -> String { fn sanitized_name(&self) -> String {
sanitize(self.name.as_deref().unwrap_or("devcontainer")) sanitize(self.name.as_deref().unwrap_or("devcontainer"))
} }
/// Construit l'image devcontainer sous `image_tag`. /// Builds the devcontainer image as `image_tag`.
async fn build( async fn build(
&self, &self,
runtime: &ContainerRuntime, runtime: &ContainerRuntime,
@@ -239,7 +239,7 @@ impl Default for DevContainer {
} }
} }
/// Nettoie une chaîne pour qu'elle puisse servir de nom d'image/container docker. /// Sanitizes a string so it can be used as a docker image/container name.
fn sanitize(input: &str) -> String { fn sanitize(input: &str) -> String {
let sanitized: String = input let sanitized: String = input
.chars() .chars()
@@ -260,7 +260,7 @@ fn sanitize(input: &str) -> String {
} }
} }
/// Identifiant unique à une exécution de sandbox, combinant l'identifiant de processus et un horodatage. /// Identifier unique to a sandbox run, combining the process id and a timestamp.
fn unique_id() -> String { fn unique_id() -> String {
let nanos = SystemTime::now() let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH) .duration_since(UNIX_EPOCH)
+5 -6
View File
@@ -1,5 +1,4 @@
//! Erreurs de la crate : échecs du daemon de containers et échecs d'analyse du //! Crate errors: container daemon failures and `devcontainer.json` parsing failures.
//! `devcontainer.json`.
use std::{path::PathBuf, time::Duration}; use std::{path::PathBuf, time::Duration};
@@ -7,25 +6,25 @@ use bollard::errors::Error as BollardError;
#[derive(Debug, thiserror::Error)] #[derive(Debug, thiserror::Error)]
pub enum ContainerError { pub enum ContainerError {
/// Le daemon est injoignable, ou a refusé une requête. /// The daemon is unreachable, or rejected a request.
#[error("container daemon request failed: {source}")] #[error("container daemon request failed: {source}")]
Request { Request {
#[from] #[from]
source: BollardError, source: BollardError,
}, },
/// Une opération a dépassé son timeout. /// An operation exceeded its timeout.
#[error("`{operation}` timed out after {timeout:?}")] #[error("`{operation}` timed out after {timeout:?}")]
Timeout { Timeout {
operation: String, operation: String,
timeout: Duration, timeout: Duration,
}, },
/// La construction de l'image a échoué. /// The image build failed.
#[error("image build failed: {message}")] #[error("image build failed: {message}")]
Build { message: String }, Build { message: String },
/// Le daemon a répondu autre chose que ce qui était attendu. /// The daemon replied with something other than what was expected.
#[error("{0}")] #[error("{0}")]
Unexpected(String), Unexpected(String),
} }
+3 -3
View File
@@ -1,9 +1,9 @@
//! Résultat d'une commande exécutée dans un container. //! Result of a command run in a container.
/// Résultat d'une commande exécutée dans un container. /// Result of a command run in a container.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct ExecOutput { pub struct ExecOutput {
/// Code de sortie, ou `-1` si le processus a été terminé par un signal. /// Exit code, or `-1` if the process was terminated by a signal.
pub status: i32, pub status: i32,
pub stdout: String, pub stdout: String,
pub stderr: String, pub stderr: String,
+18 -19
View File
@@ -1,28 +1,27 @@
//! Primitives de cycle de vie de container pour un [`DevContainer`] analysé. //! Container lifecycle primitives for a parsed [`DevContainer`].
//! //!
//! Cette crate pilote l'API du daemon de containers pour préparer l'image //! This crate drives the container daemon API to prepare the devcontainer image
//! devcontainer (construite depuis un Dockerfile, ou tirée d'un registre), //! (built from a Dockerfile, or pulled from a registry), start a container and
//! démarrer un container et y copier le workspace, exécuter les //! copy the workspace into it, run the `postCreateCommand` / `postStartCommand`
//! hooks `postCreateCommand` / `postStartCommand` et lancer des commandes à //! hooks, and run commands inside the running container.
//! l'intérieur du container en cours d'exécution.
//! //!
//! Le daemon est joint par son **socket** (celui de l'hôte, monté dans le //! The daemon is reached through its **socket** (the host's, mounted into the
//! container de Steep) : [`ContainerRuntime::connect`] suit `DOCKER_HOST` comme //! Steep container): [`ContainerRuntime::connect`] follows `DOCKER_HOST` like the
//! le fait le CLI `docker`, et retombe sur le socket local. Aucun binaire de //! `docker` CLI does, and falls back to the local socket. No runtime binary is
//! runtime n'est donc requis dans l'image, et `podman` fonctionne de la même //! therefore required in the image, and `podman` works the same way as long as it
//! façon dès lors qu'il expose son socket compatible Docker. //! exposes its Docker-compatible socket.
//! //!
//! # Isolation //! # Isolation
//! //!
//! Chaque sandbox dispose de son propre tag d'image, de son propre container et de //! Each sandbox has its own image tag, its own container and its own network. The
//! son propre réseau. Le container démarre attaché à ce réseau afin que les hooks //! container starts attached to that network so that the `postCreateCommand` /
//! `postCreateCommand` / `postStartCommand` puissent récupérer des dépendances //! `postStartCommand` hooks can fetch dependencies (e.g. `npm install`); once the
//! (par ex. `npm install`) ; une fois les hooks exécutés, le container est déconnecté //! hooks have run, the container is disconnected from the network for the rest of
//! du réseau pour le reste de sa durée de vie. Chaque commande est bornée par un timeout. //! its lifetime. Every command is bounded by a timeout.
//! //!
//! Les `runArgs` du `devcontainer.json` sont ignorés : ils viennent du dépôt, donc //! The `devcontainer.json` `runArgs` are ignored: they come from the repository,
//! d'une pull request non fiable, et pourraient rattacher le container à un autre //! hence from an untrusted pull request, and could attach the container to another
//! réseau ou lui donner des privilèges qui annuleraient cette isolation. //! network or grant it privileges that would defeat this isolation.
mod consts; mod consts;
mod container; mod container;
+3 -3
View File
@@ -1,9 +1,9 @@
//! Normalisation lexicale de chemins, sans accès au système de fichiers. //! Lexical path normalization, without accessing the filesystem.
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
/// Normalise lexicalement un chemin, en résolvant `.` et `..` sans toucher au /// Lexically normalizes a path, resolving `.` and `..` without touching the
/// système de fichiers. Renvoie `None` si le chemin sort de sa racine. /// filesystem. Returns `None` if the path escapes its root.
pub fn normalize(path: &Path) -> Option<PathBuf> { pub fn normalize(path: &Path) -> Option<PathBuf> {
use std::path::Component; use std::path::Component;
+25 -25
View File
@@ -1,4 +1,4 @@
//! Client de l'API du daemon de containers et opérations de cycle de vie. //! Container daemon API client and lifecycle operations.
use std::{collections::HashMap, path::Path, time::Duration}; use std::{collections::HashMap, path::Path, time::Duration};
@@ -24,7 +24,7 @@ use crate::{
exec::ExecOutput, exec::ExecOutput,
}; };
/// Un client de l'API du daemon de containers. /// A container daemon API client.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct ContainerRuntime { pub struct ContainerRuntime {
docker: Docker, docker: Docker,
@@ -33,7 +33,7 @@ pub struct ContainerRuntime {
} }
impl ContainerRuntime { impl ContainerRuntime {
/// Se connecte au daemon désigné par `DOCKER_HOST`, ou au socket local par défaut. /// Connects to the daemon designated by `DOCKER_HOST`, or to the default local socket.
pub fn connect() -> Result<Self, ContainerError> { pub fn connect() -> Result<Self, ContainerError> {
let endpoint = let endpoint =
std::env::var("DOCKER_HOST").unwrap_or_else(|_| String::from(DEFAULT_ENDPOINT)); std::env::var("DOCKER_HOST").unwrap_or_else(|_| String::from(DEFAULT_ENDPOINT));
@@ -45,13 +45,13 @@ impl ContainerRuntime {
}) })
} }
/// Remplace le timeout appliqué aux opérations de build/run/stop/remove. /// Replaces the timeout applied to build/run/stop/remove operations.
pub fn with_timeout(mut self, timeout: Duration) -> Self { pub fn with_timeout(mut self, timeout: Duration) -> Self {
self.timeout = timeout; self.timeout = timeout;
self self
} }
/// Endpoint du daemon, pour les logs. /// Daemon endpoint, for logging.
pub fn endpoint(&self) -> &str { pub fn endpoint(&self) -> &str {
&self.endpoint &self.endpoint
} }
@@ -60,12 +60,12 @@ impl ContainerRuntime {
self.timeout self.timeout
} }
/// Vérifie que le daemon est joignable et répond. /// Checks that the daemon is reachable and responding.
pub async fn available(&self) -> bool { pub async fn available(&self) -> bool {
self.docker.ping().await.is_ok() self.docker.ping().await.is_ok()
} }
/// Exécute une requête du daemon en appliquant le timeout des commandes. /// Runs a daemon request, applying the command timeout.
async fn request<T>( async fn request<T>(
&self, &self,
operation: &str, operation: &str,
@@ -81,7 +81,7 @@ impl ContainerRuntime {
} }
} }
/// Construit l'image `tag` depuis le contexte `context_dir`. /// Builds the image `tag` from the `context_dir` context.
pub(crate) async fn build_image( pub(crate) async fn build_image(
&self, &self,
context_dir: &Path, context_dir: &Path,
@@ -102,8 +102,8 @@ impl ContainerRuntime {
.docker .docker
.build_image(options, None, Some(body_try_stream(context))); .build_image(options, None, Some(body_try_stream(context)));
// Le flux porte la progression du build : la dernière erreur signalée par le // The stream carries the build progress: the last error reported by the
// daemon fait échouer l'opération. // daemon makes the operation fail.
let consume = async { let consume = async {
let mut failure = None; let mut failure = None;
@@ -162,7 +162,7 @@ impl ContainerRuntime {
} }
} }
/// Crée un container, sans le démarrer. /// Creates a container without starting it.
pub(crate) async fn create_container( pub(crate) async fn create_container(
&self, &self,
name: &str, name: &str,
@@ -217,7 +217,7 @@ impl ContainerRuntime {
Ok(()) Ok(())
} }
/// Exécute une commande dans un container et renvoie sa sortie. /// Runs a command in a container and returns its output.
pub(crate) async fn exec( pub(crate) async fn exec(
&self, &self,
container: &str, container: &str,
@@ -254,8 +254,8 @@ impl ContainerRuntime {
let mut stdout = String::new(); let mut stdout = String::new();
let mut stderr = String::new(); let mut stderr = String::new();
// Le timeout couvre l'exécution de la commande elle-même, pas seulement sa // The timeout covers the execution of the command itself, not just its
// mise en place. // setup.
let collect = async { let collect = async {
while let Some(message) = output.next().await { while let Some(message) = output.next().await {
match message? { match message? {
@@ -276,10 +276,10 @@ impl ContainerRuntime {
Ok(Ok(())) => {} Ok(Ok(())) => {}
Ok(Err(source)) => return Err(ContainerError::Request { source }), Ok(Err(source)) => return Err(ContainerError::Request { source }),
Err(_) => { Err(_) => {
// Le processus continue de tourner dans le container : sans arrêt, il // The process keeps running in the container: left alone, it would
// consommerait CPU et mémoire et pourrait encore modifier le workspace // consume CPU and memory and could still modify the workspace for the
// pendant toute la durée de vie de la sandbox. L'API n'offre pas de // whole lifetime of the sandbox. The API offers no way to kill an exec,
// moyen de tuer un exec, on arrête donc le container qui le porte. // so we stop the container that hosts it.
let _ = self.stop_container(container).await; let _ = self.stop_container(container).await;
return Err(ContainerError::Timeout { return Err(ContainerError::Timeout {
@@ -289,10 +289,10 @@ impl ContainerRuntime {
} }
} }
// Le code de sortie n'est pas forcément publié au moment où le flux se ferme : // The exit code is not necessarily published by the time the stream closes:
// on laisse au daemon le temps de le renseigner, sans bloquer indéfiniment. // we give the daemon time to fill it in, without blocking indefinitely.
// Sans cela, une commande réussie peut être rapportée en échec (code -1) et // Without this, a successful command can be reported as failed (code -1) and
// l'outil renvoie une erreur au modèle à la place du contenu. // the tool returns an error to the model instead of the content.
let mut inspected = self let mut inspected = self
.request("inspect exec", self.docker.inspect_exec(&created.id)) .request("inspect exec", self.docker.inspect_exec(&created.id))
.await?; .await?;
@@ -331,7 +331,7 @@ impl ContainerRuntime {
Ok(()) Ok(())
} }
/// Supprime un container, ses volumes anonymes et le processus qui y tourne. /// Removes a container, its anonymous volumes and the process running in it.
pub(crate) async fn remove_container(&self, name: &str) -> Result<(), ContainerError> { pub(crate) async fn remove_container(&self, name: &str) -> Result<(), ContainerError> {
let options = RemoveContainerOptions { let options = RemoveContainerOptions {
force: true, force: true,
@@ -363,7 +363,7 @@ impl ContainerRuntime {
Ok(()) Ok(())
} }
/// Crée le réseau dédié d'une sandbox. /// Creates a sandbox's dedicated network.
pub(crate) async fn create_network(&self, name: &str) -> Result<(), ContainerError> { pub(crate) async fn create_network(&self, name: &str) -> Result<(), ContainerError> {
let request = NetworkCreateRequest { let request = NetworkCreateRequest {
name: String::from(name), name: String::from(name),
@@ -376,7 +376,7 @@ impl ContainerRuntime {
Ok(()) Ok(())
} }
/// Détache un container d'un réseau, ce qui coupe sa connectivité. /// Detaches a container from a network, which cuts off its connectivity.
pub(crate) async fn disconnect_network( pub(crate) async fn disconnect_network(
&self, &self,
network: &str, network: &str,
+16 -16
View File
@@ -1,5 +1,5 @@
//! Schéma du `devcontainer.json`, analyse du fichier et résolution des //! `devcontainer.json` schema, file parsing and resolution of `${localEnv:…}`
//! références `${localEnv:…}`. //! references.
use std::{ use std::{
collections::HashMap, collections::HashMap,
@@ -23,7 +23,7 @@ struct DevContainerSchema {
#[serde(default)] #[serde(default)]
name: Option<String>, name: Option<String>,
/// Image de base tirée du registre, alternative à `build`. /// Base image pulled from the registry, alternative to `build`.
#[serde(default)] #[serde(default)]
image: Option<String>, image: Option<String>,
@@ -106,16 +106,16 @@ impl TryFrom<(DevContainerSchema, PathBuf)> for DevContainer {
} }
} }
/// Résout les références `${localEnv:VAR}` et `${localEnv:VAR:default}` d'un /// Resolves the `${localEnv:VAR}` and `${localEnv:VAR:default}` references of a
/// `devcontainer.json`. /// `devcontainer.json`.
/// ///
/// La spécification veut que `${localEnv:VAR}` soit lu dans l'environnement du /// The specification says `${localEnv:VAR}` is read from the client's environment;
/// client ; ici le `devcontainer.json` vient d'une **pull request**, donc de code /// here the `devcontainer.json` comes from a **pull request**, hence from untrusted
/// non fiable : lire l'environnement du processus lui permettrait de récupérer /// code: reading the process environment would let it grab `GITEA_TOKEN`,
/// `GITEA_TOKEN`, `OPEN_ROUTER_API_KEY` ou n'importe quel autre secret de Steep et /// `OPEN_ROUTER_API_KEY` or any other Steep secret and exfiltrate it through a
/// de l'exfiltrer depuis un `build.args`, un `containerEnv`, son Dockerfile ou un /// `build.args`, a `containerEnv`, its Dockerfile or a hook. The environment is
/// hook. L'environnement n'est donc **jamais** consulté : seule la valeur par /// therefore **never** consulted: only the default value is used, and a reference
/// défaut est utilisée, et une référence sans défaut devient une chaîne vide. /// without a default becomes an empty string.
fn substitute_local_env(input: &str) -> String { fn substitute_local_env(input: &str) -> String {
const PREFIX: &str = "${localEnv:"; const PREFIX: &str = "${localEnv:";
@@ -129,7 +129,7 @@ fn substitute_local_env(input: &str) -> String {
match after.find('}') { match after.find('}') {
Some(end) => { Some(end) => {
let inner = &after[..end]; let inner = &after[..end];
// La clé est lue pour délimiter la référence, pas pour la résoudre. // The key is read to delimit the reference, not to resolve it.
let (_key, default) = match inner.split_once(':') { let (_key, default) = match inner.split_once(':') {
Some((key, default)) => (key, Some(default)), Some((key, default)) => (key, Some(default)),
None => (inner, None), None => (inner, None),
@@ -239,8 +239,8 @@ mod tests {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let devcontainer_path = dir.path().join("devcontainer.json"); let devcontainer_path = dir.path().join("devcontainer.json");
// `runArgs` vient d'une pull request non fiable : il doit être accepté par // `runArgs` comes from an untrusted pull request: it must be accepted by
// l'analyse mais jamais repris dans le devcontainer. // parsing but never carried over into the devcontainer.
fs::write( fs::write(
&devcontainer_path, &devcontainer_path,
r#"{ r#"{
@@ -271,8 +271,8 @@ mod tests {
fn local_env_references_are_never_read_from_the_environment() { fn local_env_references_are_never_read_from_the_environment() {
unsafe { std::env::set_var("DEVCONTAINER_TEST_SECRET", "s3cret") }; unsafe { std::env::set_var("DEVCONTAINER_TEST_SECRET", "s3cret") };
// Le `devcontainer.json` vient d'une pull request : résoudre la référence // The `devcontainer.json` comes from a pull request: resolving the reference
// depuis l'environnement de Steep y exposerait ses jetons. // from Steep's environment would expose its tokens to it.
assert_eq!( assert_eq!(
substitute_local_env("${localEnv:DEVCONTAINER_TEST_SECRET}"), substitute_local_env("${localEnv:DEVCONTAINER_TEST_SECRET}"),
"" ""
+3 -1
View File
@@ -2,6 +2,7 @@
name = "steep-server" name = "steep-server"
version = "1.3.0" version = "1.3.0"
edition = "2024" edition = "2024"
publish = false
[dependencies] [dependencies]
reqwest = { workspace = true } reqwest = { workspace = true }
@@ -26,7 +27,8 @@ hex = { workspace = true }
metrics = { workspace = true } metrics = { workspace = true }
metrics-exporter-prometheus = { workspace = true } metrics-exporter-prometheus = { workspace = true }
devcontainer-rs = { path = "../devcontainer-rs" } devcontainer-rs = { path = "../devcontainer-rs" }
tempfile = "3" tempfile = { workspace = true }
rustls = { workspace = true }
toasty = { workspace = true } toasty = { workspace = true }
toasty-cli = { workspace = true } toasty-cli = { workspace = true }
jiff = { workspace = true } jiff = { workspace = true }
+15 -9
View File
@@ -28,8 +28,8 @@ pub async fn start(app_state: AppState) -> anyhow::Result<()> {
let http_port = app_state.config.http_port; let http_port = app_state.config.http_port;
let shutdown = app_state.shutdown.clone(); let shutdown = app_state.shutdown.clone();
// Le dashboard est servi depuis un dossier statique ; les routes API // The dashboard is served from a static directory; the API routes below take
// ci-dessous restent prioritaires (le dossier n'est que le fallback). // precedence (the directory is only the fallback).
let front_dir = std::env::var("FRONT_DIR").unwrap_or_else(|_| String::from("front")); let front_dir = std::env::var("FRONT_DIR").unwrap_or_else(|_| String::from("front"));
if !std::path::Path::new(&front_dir).is_dir() { if !std::path::Path::new(&front_dir).is_dir() {
warn!(dir = %front_dir, "front directory not found, the dashboard will 404"); warn!(dir = %front_dir, "front directory not found, the dashboard will 404");
@@ -38,7 +38,7 @@ pub async fn start(app_state: AppState) -> anyhow::Result<()> {
let app = Router::new() let app = Router::new()
.route("/webhook", post(webhook)) .route("/webhook", post(webhook))
.route("/dedup", get(dedup_list)) .route("/reviews", get(reviews_list))
Review

[security] Les routes /reviews, /jobs, /events et /jobs/{id}/events sont publiques, sans authentification. Elles exposent les noms de dépôts privés, les numéros de PR, le payload complet des webhooks (corps des commentaires, clone_url), le markdown des reviews et le flux des appels d'outils de l'agent. Ce flux contient des extraits du code des dépôts, ce qui est sensible. Le service tourne sur 0.0.0.0 et le dashboard est servi par le même processus. Il faut protéger ces routes (token ou basic auth, ou reverse proxy authentifié), ou au moins les placer sur un port interne distinct du /webhook. Il n'y a aussi aucune limite sur le nombre de connexions SSE ouvertes.

**[security]** Les routes `/reviews`, `/jobs`, `/events` et `/jobs/{id}/events` sont publiques, sans authentification. Elles exposent les noms de dépôts privés, les numéros de PR, le payload complet des webhooks (corps des commentaires, clone_url), le markdown des reviews et le flux des appels d'outils de l'agent. Ce flux contient des extraits du code des dépôts, ce qui est sensible. Le service tourne sur `0.0.0.0` et le dashboard est servi par le même processus. Il faut protéger ces routes (token ou basic auth, ou reverse proxy authentifié), ou au moins les placer sur un port interne distinct du `/webhook`. Il n'y a aussi aucune limite sur le nombre de connexions SSE ouvertes.
.route("/jobs", get(jobs_list)) .route("/jobs", get(jobs_list))
qpismont marked this conversation as resolved
Review

[security] Ces routes de consultation et les flux SSE sont accessibles sans authentification. Elles exposent notamment les noms de dépôts, les reviews et les sorties des outils exécutés sur le code, qui peuvent contenir des données de dépôts privés. Protégez ces endpoints ou limitez explicitement leur accès.

**[security]** Ces routes de consultation et les flux SSE sont accessibles sans authentification. Elles exposent notamment les noms de dépôts, les reviews et les sorties des outils exécutés sur le code, qui peuvent contenir des données de dépôts privés. Protégez ces endpoints ou limitez explicitement leur accès.
.route("/events", get(lifecycle_events)) .route("/events", get(lifecycle_events))
.route("/jobs/{id}/events", get(job_events)) .route("/jobs/{id}/events", get(job_events))
@@ -63,8 +63,8 @@ pub async fn start(app_state: AppState) -> anyhow::Result<()> {
Ok(()) Ok(())
} }
async fn dedup_list(State(app_state): State<AppState>) -> Result<impl IntoResponse, AppError> { async fn reviews_list(State(app_state): State<AppState>) -> Result<impl IntoResponse, AppError> {
let reviews = app_state.dedup.list_reviews().await?; let reviews = app_state.reviews.list().await?;
Ok(Json(json!({ Ok(Json(json!({
"reviews": reviews "reviews": reviews
@@ -81,10 +81,10 @@ async fn jobs_list(State(app_state): State<AppState>) -> Result<impl IntoRespons
}))) })))
} }
/// Flux SSE des évènements de cycle de vie global (liste des jobs). /// SSE stream of the global lifecycle events (the job list).
/// ///
/// Le flux est clos dès que l'arrêt gracieux est lancé, sans quoi une connexion /// The stream is closed as soon as the graceful shutdown starts: a live SSE
/// SSE vivante empêcherait le serveur de s'arrêter. /// connection would otherwise keep the server from stopping.
async fn lifecycle_events( async fn lifecycle_events(
State(app_state): State<AppState>, State(app_state): State<AppState>,
) -> Sse<impl Stream<Item = Result<Event, axum::Error>>> { ) -> Sse<impl Stream<Item = Result<Event, axum::Error>>> {
@@ -143,7 +143,7 @@ async fn webhook(
metrics::webhook_received(event_type); metrics::webhook_received(event_type);
let event_id = wb.event_id(); let event_id = wb.event_id();
match app_state.dedup.is_known(event_id).await { match is_known(&app_state, event_id).await {
qpismont marked this conversation as resolved
Review

[bug] Le test is_known et l’insertion dans la file ne sont pas atomiques. Deux livraisons simultanées du même webhook peuvent toutes deux passer le test ; la contrainte unique fait ensuite échouer l’une et renvoie 503 au lieu de traiter ce doublon comme déjà accepté. Gérez explicitement ce conflit d’insertion.

**[bug]** Le test `is_known` et l’insertion dans la file ne sont pas atomiques. Deux livraisons simultanées du même webhook peuvent toutes deux passer le test ; la contrainte unique fait ensuite échouer l’une et renvoie 503 au lieu de traiter ce doublon comme déjà accepté. Gérez explicitement ce conflit d’insertion.
Ok(true) => { Ok(true) => {
metrics::webhook_duplicate(event_type); metrics::webhook_duplicate(event_type);
return Err(AppError::AlreadyProcessedErr); return Err(AppError::AlreadyProcessedErr);
1
@@ -172,6 +172,12 @@ async fn webhook(
Ok((StatusCode::CREATED, "Task started")) Ok((StatusCode::CREATED, "Task started"))
} }
/// Whether an event is already queued, or already reviewed.
async fn is_known(app_state: &AppState, event_id: u64) -> toasty::Result<bool> {
Ok(app_state.queue.is_queued(event_id).await?
|| app_state.reviews.is_reviewed(event_id).await?)
}
pub struct WebhookExtract(pub WebhookType); pub struct WebhookExtract(pub WebhookType);
impl<S> FromRequest<S> for WebhookExtract impl<S> FromRequest<S> for WebhookExtract
+89 -202
View File
@@ -1,200 +1,95 @@
use std::sync::Arc; use std::{sync::Arc, time::Duration};
use futures_util::StreamExt; use tokio::{sync::Semaphore, task::JoinSet};
use serde::{Deserialize, Deserializer};
use tokio::sync::Semaphore;
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use tracing::{error, info, instrument}; use tracing::{error, info, instrument};
use crate::{ use crate::{
bot_actions::review::ReviewOutcome, bot_actions::review::{self, ReviewContext, ReviewOutcome},
dedup::{Dedup, ReviewRecord},
event_bus::{EventBus, Reporter}, event_bus::{EventBus, Reporter},
gitea::{GiteaAPI, WebhookType}, gitea::{GiteaAPI, WebhookType},
job_queue::JobQueue, job_queue::JobQueue,
metrics, metrics,
open_router::OpenRouterClient, open_router::OpenRouterClient,
sandbox::SandboxConfig, review_store::{ReviewRecord, ReviewStore},
sandbox::{SandboxConfig, tools},
schemas::Job, schemas::Job,
}; };
#[derive(Deserialize, Debug)] /// Delay before claiming again after the queue could not be reached.
pub struct ReviewResult { const CLAIM_RETRY_DELAY: Duration = Duration::from_secs(1);
pub reviews: Vec<ReviewItem>,
pub comment: String,
pub cost: Option<f64>,
}
#[derive(Deserialize, Debug)] /// Everything the bot works with: the external services and the stores.
pub struct ReviewItem { #[derive(Clone)]
pub filename: String, pub struct BotDeps {
pub line: Option<u64>, pub gitea: GiteaAPI,
#[serde(default, deserialize_with = "deserialize_side")] pub open_router: OpenRouterClient,
pub side: Option<ReviewSide>, pub sandbox: SandboxConfig,
#[serde(default, deserialize_with = "deserialize_severity")] pub reviews: ReviewStore,
pub severity: Option<ReviewSeverity>, pub queue: JobQueue,
pub message: String, pub event_bus: EventBus,
}
/// Which version of the file a review comment is anchored on.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReviewSide {
/// The line was added by the pull request: `line` is a line number of the
/// new version of the file.
Added,
/// The line was removed by the pull request: `line` is a line number of the
/// old version of the file.
Removed,
}
impl ReviewSide {
/// Reads the side the model asked for, tolerating casing and synonyms.
fn parse(raw: &str) -> Option<Self> {
match raw.trim().to_ascii_lowercase().as_str() {
"added" | "add" | "new" | "right" => Some(Self::Added),
"removed" | "remove" | "deleted" | "delete" | "old" | "left" => Some(Self::Removed),
_ => None,
}
}
}
/// What kind of problem a review reports.
///
/// The categories are the ones the prompt asks for; a review whose severity is
/// missing or unreadable falls back to [`ReviewSeverity::Maintainability`], the
/// least alarming bucket, instead of failing the whole review.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReviewSeverity {
/// Wrong behaviour.
Bug,
/// A vulnerability.
Security,
/// A resource problem.
Performance,
/// Everything else: readability, structure, naming, tests.
Maintainability,
}
impl ReviewSeverity {
/// Reads the severity the model asked for, tolerating casing and synonyms.
fn parse(raw: &str) -> Option<Self> {
match raw.trim().to_ascii_lowercase().as_str() {
"bug" | "bugs" | "correctness" | "error" => Some(Self::Bug),
"security" | "vulnerability" => Some(Self::Security),
"performance" | "perf" => Some(Self::Performance),
"maintainability" | "quality" | "style" => Some(Self::Maintainability),
_ => None,
}
}
/// Lower-case label used in the review markdown.
pub fn label(self) -> &'static str {
match self {
Self::Bug => "bug",
Self::Security => "security",
Self::Performance => "performance",
Self::Maintainability => "maintainability",
}
}
}
/// Reads the side the model asked for. An unreadable value is ignored instead of
/// failing the whole review: the side is then resolved from the changed lines.
fn deserialize_side<'de, D>(deserializer: D) -> Result<Option<ReviewSide>, D::Error>
where
D: Deserializer<'de>,
{
let raw = Option::<serde_json::Value>::deserialize(deserializer)?;
Ok(raw
.as_ref()
.and_then(serde_json::Value::as_str)
.and_then(ReviewSide::parse))
}
/// Reads the severity the model asked for, ignoring an unreadable value: the
/// review is then reported as [`ReviewSeverity::Maintainability`].
fn deserialize_severity<'de, D>(deserializer: D) -> Result<Option<ReviewSeverity>, D::Error>
where
D: Deserializer<'de>,
{
let raw = Option::<serde_json::Value>::deserialize(deserializer)?;
Ok(raw
.as_ref()
.and_then(serde_json::Value::as_str)
.and_then(ReviewSeverity::parse))
} }
#[derive(Clone)] #[derive(Clone)]
pub struct Bot { pub struct Bot {
gitea_api: GiteaAPI, deps: BotDeps,
open_router_client: OpenRouterClient,
max_concurrent: usize, max_concurrent: usize,
open_router_model: String,
sandbox: SandboxConfig,
dedup: Dedup,
queue: JobQueue,
event_bus: EventBus,
} }
impl Bot { impl Bot {
#[allow(clippy::too_many_arguments)] pub fn new(deps: BotDeps, max_concurrent: usize) -> Self {
pub fn new(
gitea_api: GiteaAPI,
open_router_client: OpenRouterClient,
max_concurrent: usize,
open_router_model: String,
sandbox: SandboxConfig,
dedup: Dedup,
queue: JobQueue,
event_bus: EventBus,
) -> Self {
Self { Self {
gitea_api, deps,
open_router_client,
max_concurrent, max_concurrent,
open_router_model,
sandbox,
dedup,
queue,
event_bus,
} }
} }
/// Consomme la file jusqu'à l'annulation de `shutdown`. /// Consumes the queue until `shutdown` is cancelled.
/// ///
/// Un sémaphore borne la concurrence ; les jobs en cours sont laissés se /// A semaphore bounds the concurrency, and a job is only claimed once a slot is
/// terminer. Les jobs non terminés restent en base et seront repris au /// free, so the jobs waiting for one stay `pending`. Running jobs are left to
/// prochain démarrage. /// finish; the unfinished ones stay `processing` and are requeued at the next
/// startup.
pub async fn start(&self, shutdown: CancellationToken) -> anyhow::Result<()> { pub async fn start(&self, shutdown: CancellationToken) -> anyhow::Result<()> {
info!("Bot started"); info!("Bot started");
let semaphore = Arc::new(Semaphore::new(self.max_concurrent)); let semaphore = Arc::new(Semaphore::new(self.max_concurrent));
let mut tasks = tokio::task::JoinSet::new(); let mut tasks = JoinSet::new();
let mut pending = std::pin::pin!(self.queue.pending());
loop { 'consume: loop {
let job = tokio::select! { let permit = tokio::select! {
biased; biased;
_ = shutdown.cancelled() => break, () = shutdown.cancelled() => break,
job = pending.next() => match job { permit = semaphore.clone().acquire_owned() => permit?,
Some(job) => job,
None => break,
},
}; };
while let Some(res) = tasks.try_join_next() { let job = loop {
if let Err(err) = res { while let Some(res) = tasks.try_join_next() {
error!("Task panicked: {err}"); if let Err(err) = res {
error!("Task panicked: {err}");
}
} }
}
let permit = semaphore.clone().acquire_owned().await?; match self.deps.queue.claim_next().await {
Ok(Some(job)) => break job,
Ok(None) => {
tokio::select! {
biased;
() = shutdown.cancelled() => break 'consume,
() = self.deps.queue.wait_for_work() => {}
}
}
Err(err) => {
error!(%err, "failed to claim a job");
if let Err(err) = self.queue.mark_processing(job.id).await { tokio::select! {
error!(%err, "failed to mark job as processing"); biased;
} () = shutdown.cancelled() => break 'consume,
() = tokio::time::sleep(CLAIM_RETRY_DELAY) => {}
}
}
}
};
let bot = self.clone(); let bot = self.clone();
metrics::increment_task_active(); metrics::increment_task_active();
@@ -212,30 +107,35 @@ impl Bot {
Ok(()) Ok(())
} }
/// Runs a claimed job and records its outcome.
async fn process(&self, job: Job) { async fn process(&self, job: Job) {
self.run_job(&job).await;
if let Err(err) = self.deps.queue.remove(job.id).await {
error!(%err, "failed to remove job");
}
}
async fn run_job(&self, job: &Job) {
let deps = &self.deps;
let event_type = job.payload.event_type_str(); let event_type = job.payload.event_type_str();
match self.dedup.is_reviewed(job.event_id).await { match deps.reviews.is_reviewed(job.event_id).await {
Ok(true) => { Ok(true) => {
metrics::task_skipped(event_type); metrics::task_skipped(event_type);
info!("Review already done, skipping"); info!("Review already done, skipping");
self.event_bus.skip(job.id, &job.repo, job.pr); deps.event_bus.skip(job.id, &job.repo, job.pr);
if let Err(err) = self.queue.remove(job.id).await {
error!(%err, "failed to remove job");
}
return; return;
} }
Ok(false) => {} Ok(false) => {}
Err(err) => error!(%err, "Dedup lookup failed, reviewing anyway"), Err(err) => error!(%err, "Review lookup failed, reviewing anyway"),
} }
let reporter = self.event_bus.start(job.id, job.repo.clone(), job.pr); let reporter = deps.event_bus.start(job.id, job.repo.clone(), job.pr);
let payload = job.payload.0.clone();
let started = std::time::Instant::now(); let started = std::time::Instant::now();
let result = self.exec(payload, &reporter).await; let result = self.exec(&job.payload, &reporter).await;
let duration_ms = started.elapsed().as_millis() as u64; let duration_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX);
let (ok, outcome) = match result { let (ok, outcome) = match result {
Ok(outcome) => { Ok(outcome) => {
@@ -251,60 +151,47 @@ impl Bot {
} }
}; };
let comments = outcome.comments as u64;
let record = ReviewRecord { let record = ReviewRecord {
event_id: job.event_id, event_id: job.event_id,
repo: job.repo.clone(), repo: job.repo.clone(),
pr: job.pr, pr: job.pr,
ok, ok,
model: self.open_router_model.clone(), model: deps.open_router.model().to_string(),
markdown: outcome.markdown, markdown: outcome.markdown,
cost: outcome.cost, cost: outcome.cost,
comments, comments: outcome.comments,
duration_ms, duration_ms,
}; };
if let Err(err) = self.dedup.mark_reviewed(record).await { if let Err(err) = deps.reviews.record(record).await {
error!(%err, "failed to record review"); error!(%err, "failed to record review");
} }
self.event_bus deps.event_bus
.finish(job.id, &job.repo, job.pr, ok, comments, duration_ms); .finish(job.id, &job.repo, job.pr, ok, outcome.comments, duration_ms);
if let Err(err) = self.queue.remove(job.id).await {
error!(%err, "failed to remove job");
}
} }
#[instrument(skip(self, webhook, reporter), fields(repo, pr))] /// Dispatches a webhook to the action it asks for.
pub async fn exec( #[instrument(
skip_all,
fields(repo = %webhook.repository_full_name(), pr = webhook.pull_request_number())
)]
async fn exec(
&self, &self,
webhook: WebhookType, webhook: &WebhookType,
reporter: &Reporter, reporter: &Reporter,
) -> anyhow::Result<ReviewOutcome> { ) -> anyhow::Result<ReviewOutcome> {
match &webhook { let deps = &self.deps;
WebhookType::Review(p) => { let ctx = ReviewContext {
tracing::Span::current().record("repo", &p.repository.full_name); gitea: &deps.gitea,
tracing::Span::current().record("pr", p.pull_request.number); open_router: &deps.open_router,
} sandbox: &deps.sandbox,
qpismont marked this conversation as resolved Outdated
Outdated
Review

[bug] BOT_MAX_CONCURRENT=0 est accepté par le parsing de configuration, mais acquire_owned() attendra alors indéfiniment un permis. Comme cette attente n'est pas incluse dans un select! avec l'annulation, l'arrêt peut rester bloqué jusqu'au délai forcé de 60 secondes. Il faut rejeter zéro au chargement de configuration ou rendre l'attente annulable.

**[bug]** `BOT_MAX_CONCURRENT=0` est accepté par le parsing de configuration, mais `acquire_owned()` attendra alors indéfiniment un permis. Comme cette attente n'est pas incluse dans un `select!` avec l'annulation, l'arrêt peut rester bloqué jusqu'au délai forcé de 60 secondes. Il faut rejeter zéro au chargement de configuration ou rendre l'attente annulable.
Review

[bug] L’attente d’un permis n’écoute pas shutdown. Si tous les slots sont occupés au moment de l’arrêt, le bot attend qu’un slot se libère puis lance quand même le job en attente, et peut continuer ainsi à vider la file. Cela contredit le comportement annoncé où seuls les jobs déjà en cours terminent ; rendez aussi cette attente interruptible.

**[bug]** L’attente d’un permis n’écoute pas `shutdown`. Si tous les slots sont occupés au moment de l’arrêt, le bot attend qu’un slot se libère puis lance quand même le job en attente, et peut continuer ainsi à vider la file. Cela contredit le comportement annoncé où seuls les jobs déjà en cours terminent ; rendez aussi cette attente interruptible.
reporter,
}; };
qpismont marked this conversation as resolved Outdated
Outdated
Review

[bug] En cas d'échec de mark_processing, le code journalise l'erreur mais lance quand même le traitement. Le job reste alors Pending et peut être repris par le poller pendant que la review est déjà en cours, ce qui crée des traitements et commentaires en double. Il faut abandonner ce job (ou le remettre explicitement dans un état contrôlé) lorsque la réservation échoue.

**[bug]** En cas d'échec de `mark_processing`, le code journalise l'erreur mais lance quand même le traitement. Le job reste alors `Pending` et peut être repris par le poller pendant que la review est déjà en cours, ce qui crée des traitements et commentaires en double. Il faut abandonner ce job (ou le remettre explicitement dans un état contrôlé) lorsque la réservation échoue.
Review

[bug] Si la mise à jour du statut échoue, le job est quand même traité. Il reste alors Pending et sera récupéré au prochain sondage, ce qui peut lancer plusieurs reviews concurrentes pour le même événement. Ne lancez pas le traitement si cette transition échoue, ou garantissez une prise en charge atomique.

**[bug]** Si la mise à jour du statut échoue, le job est quand même traité. Il reste alors `Pending` et sera récupéré au prochain sondage, ce qui peut lancer plusieurs reviews concurrentes pour le même événement. Ne lancez pas le traitement si cette transition échoue, ou garantissez une prise en charge atomique.
let tools = tools::for_webhook(webhook);
let tools = crate::sandbox::tools::for_webhook(&webhook);
match webhook { match webhook {
WebhookType::Review(review_payload) => { WebhookType::Review(payload) => review::exec_review(&ctx, tools, payload).await,
crate::bot_actions::review::exec_review(
&self.gitea_api,
&self.open_router_client,
&self.open_router_model,
&self.sandbox,
tools,
review_payload,
reporter,
)
.await
}
} }
qpismont marked this conversation as resolved
Review

[bug] En cas d’échec de mark_processing, le job est tout de même traité. Il reste alors Pending en base et pourra être repris après redémarrage — ou par une autre instance — pendant que cette exécution peut déjà publier la review. N’exécutez pas le job si sa transition d’état échoue.

**[bug]** En cas d’échec de `mark_processing`, le job est tout de même traité. Il reste alors Pending en base et pourra être repris après redémarrage — ou par une autre instance — pendant que cette exécution peut déjà publier la review. N’exécutez pas le job si sa transition d’état échoue.
} }
} }
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,177 @@
//! Anchoring of the model's reviews on the lines a pull request changes.
use tracing::warn;
use super::diff::ChangedLines;
use crate::domain::review::ReviewResult;
/// Resolves the side each review is anchored on and drops the reviews that do
/// not match a line the pull request changes.
///
/// The model is asked to pick a line and a side from the provided lists, but
/// nothing forces it to, and Gitea accepts any position: a wrong one places the
/// comment on an unrelated line of the file instead of failing. A review that
/// omits its side is resolved from the lists, and one that matches no changed
/// line is dropped.
pub(super) fn resolve_review_sides(review_result: &mut ReviewResult, changed_lines: &ChangedLines) {
let mut dropped = Vec::new();
review_result.reviews.retain_mut(|review| {
let side = review.line.and_then(|line| {
let file = changed_lines
.iter()
.find(|file| file.filename == review.filename)?;
let side = review.side.or_else(|| file.side_of(line))?;
file.lines(side).contains(&line).then_some(side)
});
match side {
Some(side) => {
review.side = Some(side);
true
}
None => {
dropped.push(match review.line {
Some(line) => format!("{}:{line}", review.filename),
None => format!("{}:no line", review.filename),
});
false
}
}
});
if !dropped.is_empty() {
warn!(
dropped = dropped.len(),
reviews = %dropped.join(", "),
"Dropped reviews that are not anchored on a changed line"
);
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
bot_actions::review::{diff::parse_changed_lines, fixtures::*},
domain::review::ReviewSide,
};
#[test]
fn reviews_keep_their_changed_line_and_resolve_their_side() {
let changed_lines = parse_changed_lines(DIFF);
let mut review_result = review_result(vec![
review("src/foo.rs", Some(4), Some(ReviewSide::Added)),
review("src/foo.rs", Some(2), Some(ReviewSide::Removed)),
review("src/bar.rs", Some(13), None),
]);
resolve_review_sides(&mut review_result, &changed_lines);
let sides = review_result
.reviews
.iter()
.map(|review| review.side)
.collect::<Vec<_>>();
assert_eq!(
sides,
vec![
Some(ReviewSide::Added),
Some(ReviewSide::Removed),
Some(ReviewSide::Added)
]
);
}
#[test]
fn a_line_changed_on_both_sides_defaults_to_added() {
let changed_lines = parse_changed_lines(DIFF);
let mut review_result = review_result(vec![review("src/foo.rs", Some(2), None)]);
resolve_review_sides(&mut review_result, &changed_lines);
assert_eq!(review_result.reviews[0].side, Some(ReviewSide::Added));
}
#[test]
fn reviews_outside_changed_lines_are_dropped() {
let changed_lines = parse_changed_lines(DIFF);
let mut review_result = review_result(vec![
// valid
review("src/foo.rs", Some(2), Some(ReviewSide::Added)),
// a line that exists but is not part of the change
review("src/foo.rs", Some(3), None),
// a line beyond the change
review("src/foo.rs", Some(999), None),
// a changed line, but on the wrong side
review("src/foo.rs", Some(4), Some(ReviewSide::Removed)),
// a line changed in another file
review("src/foo.rs", Some(11), None),
// an unknown file
review("src/baz.rs", Some(1), None),
// no line at all
review("src/bar.rs", None, None),
]);
resolve_review_sides(&mut review_result, &changed_lines);
assert_eq!(review_result.reviews.len(), 1);
assert_eq!(review_result.reviews[0].filename, "src/foo.rs");
assert_eq!(review_result.reviews[0].line, Some(2));
assert_eq!(review_result.reviews[0].side, Some(ReviewSide::Added));
}
#[test]
fn a_deleted_line_is_kept_as_a_removed_anchor() {
let changed_lines = parse_changed_lines(DELETION_ONLY);
let mut review_result = review_result(vec![
review("src/old.rs", Some(2), Some(ReviewSide::Removed)),
review("src/old.rs", Some(2), Some(ReviewSide::Added)),
]);
resolve_review_sides(&mut review_result, &changed_lines);
assert_eq!(review_result.reviews.len(), 1);
assert_eq!(review_result.reviews[0].side, Some(ReviewSide::Removed));
}
#[test]
fn odd_sides_from_the_model_are_tolerated() {
let changed_lines = parse_changed_lines(DIFF);
let mut review_result: ReviewResult = serde_json::from_str(
r#"{
"reviews": [
{ "filename": "src/foo.rs", "line": 4, "side": "Added", "message": "a" },
{ "filename": "src/foo.rs", "line": 2, "side": "new", "message": "b" },
{ "filename": "src/foo.rs", "line": 2, "side": "REMOVED", "message": "c" },
{ "filename": "src/foo.rs", "line": 2, "side": "banana", "message": "d" },
{ "filename": "src/foo.rs", "line": 5, "message": "e" }
],
"comment": ""
}"#,
)
.unwrap();
resolve_review_sides(&mut review_result, &changed_lines);
let sides = review_result
.reviews
.iter()
.map(|review| review.side)
.collect::<Vec<_>>();
assert_eq!(
sides,
vec![
Some(ReviewSide::Added),
Some(ReviewSide::Added),
Some(ReviewSide::Removed),
Some(ReviewSide::Added),
Some(ReviewSide::Added)
]
);
}
}
@@ -0,0 +1,389 @@
//! Parsing of a pull request unified diff into the lines it changes.
use tracing::warn;
use crate::{domain::review::ReviewSide, gitea::PullRequestFile};
/// The lines a pull request changed, per file.
///
/// Line numbers are the ones Gitea expects to place a review comment:
/// [`ReviewSide::Added`] numbers refer to the new version of the file (sent as
/// `new_position`), [`ReviewSide::Removed`] numbers to the old version (sent as
/// `old_position`).
pub(super) type ChangedLines = Vec<ChangedFile>;
pub(super) struct ChangedFile {
pub(super) filename: String,
pub(super) added: Vec<u64>,
pub(super) removed: Vec<u64>,
}
impl ChangedFile {
fn new(filename: &str) -> Self {
Self {
filename: String::from(filename),
added: Vec::new(),
removed: Vec::new(),
}
}
/// Lines that can be commented on for the given side.
pub(super) fn lines(&self, side: ReviewSide) -> &[u64] {
match side {
ReviewSide::Added => &self.added,
ReviewSide::Removed => &self.removed,
}
}
/// Side a line belongs to, used when the model did not state one.
pub(super) fn side_of(&self, line: u64) -> Option<ReviewSide> {
if self.added.contains(&line) {
Some(ReviewSide::Added)
} else if self.removed.contains(&line) {
Some(ReviewSide::Removed)
} else {
None
}
}
}
/// Lists the lines changed by the diff, per file, on both sides.
///
/// Only the line numbers are kept: the model reads the code itself through the
/// sandbox tools.
pub(super) fn parse_changed_lines(git_diff: &str) -> ChangedLines {
let mut files = Vec::new();
let mut current_file: Option<String> = None;
let mut in_hunk = false;
let mut old_line: u64 = 0;
let mut new_line: u64 = 0;
for line in git_diff.lines() {
if line.starts_with("diff --git ") {
current_file = None;
in_hunk = false;
continue;
}
// `--- a/x` and `+++ b/x` only appear before the first hunk of a file.
// Inside a hunk, a line may legitimately start with them: removing
// `--x` gives `---x`, adding `++i;` gives `+++i;`.
if !in_hunk && (line.starts_with("--- ") || line.starts_with("+++ ")) {
// A header is never a content line: `+++ /dev/null` on a deleted
// file keeps the path of the other side in place.
if let Some(path) = header_file_path(line) {
current_file = Some(path);
}
continue;
}
if line.starts_with("@@") {
if let Some((old_start, new_start)) = parse_hunk_starts(line) {
old_line = old_start;
new_line = new_start;
in_hunk = true;
}
continue;
}
let Some(filename) = current_file.as_deref() else {
continue;
};
match line.as_bytes().first() {
Some(b' ') => {
old_line += 1;
new_line += 1;
}
Some(b'-') => {
changed_file(&mut files, filename).removed.push(old_line);
old_line += 1;
}
Some(b'+') => {
changed_file(&mut files, filename).added.push(new_line);
new_line += 1;
}
// `\ No newline at end of file`, and anything unexpected: a line
// that advances neither side.
_ => {}
}
}
files
}
/// Path of the file on one side of the diff, from a `--- a/<path>` or
/// `+++ b/<path>` header line.
///
/// These lines are the only unambiguous source for the path: the `diff --git`
/// line is cut at the first space, and it names the old path of a renamed file.
/// `None` for `/dev/null`, which leaves the path of the other side in place.
fn header_file_path(line: &str) -> Option<String> {
let (prefix, raw) = match line.strip_prefix("--- ") {
Some(raw) => ("a/", raw),
None => ("b/", line.strip_prefix("+++ ")?),
};
let path = decode_git_path(raw);
Some(String::from(path.strip_prefix(prefix)?))
}
/// Decodes a path as git writes it in a diff header: git wraps it in quotes and
/// escapes the bytes that need it (`\303\251` for `é`) when the path contains
/// non-printable or non-ASCII characters.
fn decode_git_path(raw: &str) -> String {
let Some(quoted) = raw.strip_prefix('"').and_then(|raw| raw.strip_suffix('"')) else {
return String::from(raw);
};
let bytes = quoted.as_bytes();
let mut decoded = Vec::with_capacity(bytes.len());
let mut index = 0;
while let Some(byte) = bytes.get(index) {
index += 1;
if *byte != b'\\' {
decoded.push(*byte);
continue;
}
match bytes.get(index) {
// Octal escapes are the ones that matter for a path: git uses them
// for every non-ASCII byte.
Some(digit @ b'0'..=b'7') => {
let mut value = u32::from(digit - b'0');
let mut digits = 1;
while digits < 3 {
match bytes.get(index + digits) {
Some(next @ b'0'..=b'7') => {
value = value * 8 + u32::from(next - b'0');
digits += 1;
}
_ => break,
}
}
decoded.push(u8::try_from(value).unwrap_or(b'?'));
index += digits;
}
Some(escaped) => {
decoded.push(match escaped {
b't' => b'\t',
b'n' => b'\n',
b'r' => b'\r',
other => *other,
});
index += 1;
}
None => decoded.push(b'\\'),
}
}
String::from_utf8_lossy(&decoded).into_owned()
}
/// Replaces the paths parsed from the diff with the exact paths reported by the
/// API, which are the ones the model sees in the sandbox.
pub(super) fn resolve_filenames(changed_lines: &mut ChangedLines, files: &[PullRequestFile]) {
for changed in changed_lines.iter_mut() {
let parsed = changed.filename.as_str();
let Some(file) = files.iter().find(|file| file.filename == parsed) else {
if !files.is_empty() {
warn!(path = %parsed, "Changed file is not in the pull request file list");
}
continue;
};
changed.filename = file.filename.clone();
}
}
/// Basenames of generated files that are never reviewed.
///
/// Their diffs are machine-written dependency churn: they run to thousands of
/// lines, flood the prompt with line numbers and drown the code the model should
/// look at. A lockfile is also never a place where a review comment belongs.
const IGNORED_FILE_NAMES: [&str; 16] = [
"Cargo.lock",
"package-lock.json",
"npm-shrinkwrap.json",
"yarn.lock",
"pnpm-lock.yaml",
"bun.lock",
"bun.lockb",
"composer.lock",
"Gemfile.lock",
"poetry.lock",
"uv.lock",
"Pipfile.lock",
"go.sum",
"packages.lock.json",
"flake.lock",
"pubspec.lock",
];
/// Whether a changed path is a generated file that is never reviewed.
///
/// The comparison is on the basename, so a lockfile nested in a workspace (for
/// example `crates/foo/Cargo.lock`) is matched too.
pub(super) fn is_ignored(filename: &str) -> bool {
let basename = filename.rsplit('/').next().unwrap_or(filename);
IGNORED_FILE_NAMES.contains(&basename)
}
/// Drops the generated files from the changed lines.
///
/// Removing them here keeps the prompt free of their line numbers, and also makes
/// [`resolve_review_sides`] reject any review the model anchors on one of them.
pub(super) fn drop_generated_files(changed_lines: &mut ChangedLines) {
changed_lines.retain(|file| !is_ignored(&file.filename));
}
/// Borrows the entry of `files` for a file, creating it on its first change.
fn changed_file<'a>(files: &'a mut ChangedLines, filename: &str) -> &'a mut ChangedFile {
let index = match files.iter().position(|file| file.filename == filename) {
Some(index) => index,
None => {
files.push(ChangedFile::new(filename));
files.len() - 1
}
};
&mut files[index]
}
/// Extracts the old and new starting line numbers of a hunk header such as
/// `@@ -12,3 +12,5 @@`. The counts are optional and git may append a section
/// heading after the closing `@@`.
fn parse_hunk_starts(hunk_header: &str) -> Option<(u64, u64)> {
let body = hunk_header.strip_prefix("@@ ")?;
let body = body.split(" @@").next()?;
let (old, new) = body.split_once(" +")?;
let old = old.strip_prefix('-')?.split(',').next()?;
let new = new.split(',').next()?;
Some((old.parse().ok()?, new.parse().ok()?))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::bot_actions::review::{fixtures::*, prompt::format_changed_lines};
#[test]
fn a_deletion_only_pull_request_lists_removed_lines() {
let expected = "src/old.rs: removed 1, 2, 3";
assert_eq!(
format_changed_lines(&parse_changed_lines(DELETION_ONLY)),
expected
);
}
#[test]
fn hunk_content_starting_with_plus_or_minus_is_counted() {
let expected = "src/tricky.js: added 2 / removed 2";
assert_eq!(
format_changed_lines(&parse_changed_lines(TRICKY_CONTENT)),
expected
);
}
#[test]
fn a_renamed_file_uses_its_new_path() {
let expected = "src/new.rs: added 1 / removed 1";
assert_eq!(
format_changed_lines(&parse_changed_lines(RENAMED)),
expected
);
}
#[test]
fn a_path_with_a_space_is_read_from_the_headers() {
let expected = "src/my file.rs: added 2";
assert_eq!(
format_changed_lines(&parse_changed_lines(PATH_WITH_SPACE)),
expected
);
}
#[test]
fn a_quoted_path_is_decoded() {
let expected = "docs/café.md: added 2";
assert_eq!(
format_changed_lines(&parse_changed_lines(QUOTED_PATH)),
expected
);
}
#[test]
fn filenames_are_resolved_against_the_api_list() {
let mut changed_lines = parse_changed_lines(DIFF);
let files = vec![
pull_request_file("src/bar.rs", None, "modified"),
pull_request_file("src/foo.rs", Some("src/renamed.rs"), "renamed"),
];
resolve_filenames(&mut changed_lines, &files);
assert_eq!(changed_lines[0].filename, "src/foo.rs");
assert_eq!(changed_lines[1].filename, "src/bar.rs");
}
#[test]
fn a_file_absent_from_the_api_list_keeps_the_diff_path() {
let mut changed_lines = parse_changed_lines(DIFF);
let files = vec![pull_request_file("src/bar.rs", None, "modified")];
resolve_filenames(&mut changed_lines, &files);
assert_eq!(changed_lines[0].filename, "src/foo.rs");
assert_eq!(changed_lines[1].filename, "src/bar.rs");
}
#[test]
fn generated_lockfiles_are_ignored() {
assert!(is_ignored("Cargo.lock"));
assert!(is_ignored("crates/steep-server/Cargo.lock"));
assert!(is_ignored("package-lock.json"));
assert!(is_ignored("web/yarn.lock"));
assert!(is_ignored("go.sum"));
assert!(!is_ignored("src/main.rs"));
assert!(!is_ignored("Cargo.toml"));
assert!(!is_ignored("docs/lockfiles.md"));
}
#[test]
fn generated_files_are_dropped_from_the_changed_lines() {
const DIFF_WITH_LOCKFILE: &str = concat!(
"diff --git a/Cargo.lock b/Cargo.lock\n",
"--- a/Cargo.lock\n",
"+++ b/Cargo.lock\n",
"@@ -1,1 +1,2 @@\n",
" [[package]]\n",
"+name = \"x\"\n",
"diff --git a/src/foo.rs b/src/foo.rs\n",
"--- a/src/foo.rs\n",
"+++ b/src/foo.rs\n",
"@@ -1,1 +1,2 @@\n",
" fn a() {}\n",
"+fn b() {}\n",
);
let mut changed_lines = parse_changed_lines(DIFF_WITH_LOCKFILE);
drop_generated_files(&mut changed_lines);
assert_eq!(format_changed_lines(&changed_lines), "src/foo.rs: added 2");
}
}
@@ -0,0 +1,119 @@
//! Shared fixtures of the review tests.
use crate::{
domain::review::{ReviewItem, ReviewResult, ReviewSide},
gitea::PullRequestFile,
};
/// Additions, a removal and a line changed on both sides.
pub(super) const DIFF: &str = concat!(
"diff --git a/src/foo.rs b/src/foo.rs\n",
"--- a/src/foo.rs\n",
"+++ b/src/foo.rs\n",
"@@ -1,4 +1,6 @@\n",
" fn main() {\n",
"+ let x = 1;\n",
"- let removed = 0;\n",
" println!(\"hello\");\n",
"+ let y = 2;\n",
"+ let z = 3;\n",
" }\n",
"diff --git a/src/bar.rs b/src/bar.rs\n",
"--- a/src/bar.rs\n",
"+++ b/src/bar.rs\n",
"@@ -10,4 +10,6 @@\n",
" old context\n",
"+ let a = 10;\n",
" more context\n",
"+ let b = 20;\n",
);
/// A pull request that only deletes a file.
pub(super) const DELETION_ONLY: &str = concat!(
"diff --git a/src/old.rs b/src/old.rs\n",
"deleted file mode 100644\n",
"--- a/src/old.rs\n",
"+++ /dev/null\n",
"@@ -1,3 +0,0 @@\n",
"-fn a() {}\n",
"-fn b() {}\n",
"-fn c() {}\n",
);
/// A hunk whose content starts with `+++` / `---`, which must not be taken
/// for the file headers.
pub(super) const TRICKY_CONTENT: &str = concat!(
"diff --git a/src/tricky.js b/src/tricky.js\n",
"--- a/src/tricky.js\n",
"+++ b/src/tricky.js\n",
"@@ -1,4 +1,4 @@\n",
" let i = 0;\n",
"+++i;\n",
"---x;\n",
" console.log(i);\n",
);
/// A pull request that renames a file.
pub(super) const RENAMED: &str = concat!(
"diff --git a/src/old.rs b/src/new.rs\n",
"similarity index 50%\n",
"rename from src/old.rs\n",
"rename to src/new.rs\n",
"--- a/src/old.rs\n",
"+++ b/src/new.rs\n",
"@@ -1,1 +1,1 @@\n",
"-fn old() {}\n",
"+fn new() {}\n",
);
/// A file whose path contains a space, which the `diff --git` line cannot
/// express without ambiguity.
pub(super) const PATH_WITH_SPACE: &str = concat!(
"diff --git a/src/my file.rs b/src/my file.rs\n",
"--- a/src/my file.rs\n",
"+++ b/src/my file.rs\n",
"@@ -1,1 +1,2 @@\n",
" fn a() {}\n",
"+fn b() {}\n",
);
/// A file whose path git quotes and escapes (`caf\\303\\251.md` is
/// `caf\u{e9}.md`).
pub(super) const QUOTED_PATH: &str = concat!(
"diff --git \"a/docs/caf\\303\\251.md\" \"b/docs/caf\\303\\251.md\"\n",
"--- \"a/docs/caf\\303\\251.md\"\n",
"+++ \"b/docs/caf\\303\\251.md\"\n",
"@@ -1,1 +1,2 @@\n",
" intro\n",
"+ajout\n",
);
pub(super) fn review(filename: &str, line: Option<u64>, side: Option<ReviewSide>) -> ReviewItem {
ReviewItem {
filename: String::from(filename),
line,
side,
severity: None,
message: String::from("message"),
}
}
pub(super) fn review_result(reviews: Vec<ReviewItem>) -> ReviewResult {
ReviewResult {
reviews,
comment: String::new(),
cost: None,
}
}
pub(super) fn pull_request_file(
filename: &str,
previous_filename: Option<&str>,
status: &str,
) -> PullRequestFile {
PullRequestFile {
filename: String::from(filename),
previous_filename: previous_filename.map(String::from),
status: String::from(status),
}
}
@@ -0,0 +1,210 @@
//! The pull request review action.
//!
//! [`exec_review`] drives one review end to end. The work is split by concern:
//! - [`diff`] parses the pull request diff into the lines it changes,
//! - [`prompt`] renders those changes for the model,
//! - [`anchor`] checks that the model's reviews land on changed lines,
//! - [`render`] builds the markdown summary posted on the pull request.
mod anchor;
mod diff;
mod prompt;
mod render;
#[cfg(test)]
mod fixtures;
use tracing::{info, instrument, warn};
use crate::{
consts::{BOT_PROCESS_MSG, REVIEW_PROMPT, SANDBOX_SYSTEM_PROMPT},
domain::review::ReviewResult,
event_bus::Reporter,
gitea::{GiteaAPI, ReviewPayload},
metrics,
open_router::{OpenRouterClient, Tool},
sandbox::{Sandbox, SandboxConfig, agent},
};
use self::{
anchor::resolve_review_sides,
diff::{drop_generated_files, parse_changed_lines, resolve_filenames},
prompt::format_changes,
render::review_result_to_markdown,
};
/// The services a review needs, borrowed from the bot for the duration of one job.
pub struct ReviewContext<'a> {
pub gitea: &'a GiteaAPI,
pub open_router: &'a OpenRouterClient,
pub sandbox: &'a SandboxConfig,
pub reporter: &'a Reporter,
}
pub struct ReviewOutcome {
pub markdown: String,
pub cost: Option<f64>,
pub comments: u64,
}
impl ReviewOutcome {
pub fn empty() -> Self {
Self {
markdown: String::new(),
cost: None,
comments: 0,
}
}
}
#[instrument(
skip_all,
fields(repo = %payload.repository.full_name, pr = payload.pull_request.number)
)]
pub async fn exec_review(
ctx: &ReviewContext<'_>,
tools: Vec<Tool>,
payload: &ReviewPayload,
) -> anyhow::Result<ReviewOutcome> {
info!(action = %payload.action, "Starting review");
let full_name = &payload.repository.full_name;
let index = payload.pull_request.number;
let progress_comment = ctx
.gitea
.comment(
&BOT_PROCESS_MSG.replace("{model}", ctx.open_router.model()),
full_name,
index,
)
.await?;
let review = review_pull_request(ctx, tools, payload).await;
// The progress comment is housekeeping: failing to update it must neither turn a
// posted review into a failure nor hide the error that made the review fail.
match review {
Ok((review_result, markdown)) => {
if let Err(err) = ctx
.gitea
.delete_comment(full_name, progress_comment.id)
.await
{
warn!(%err, "Failed to delete the progress comment");
}
Ok(ReviewOutcome {
markdown,
cost: review_result.cost,
comments: review_result.reviews.len() as u64,
})
}
Err(err) => {
if let Err(edit_err) = ctx
.gitea
.edit_comment(
&format!("Error while reviewing: {err}"),
full_name,
progress_comment.id,
)
.await
{
warn!(err = %edit_err, "Failed to report the error on the progress comment");
}
Err(err)
}
}
}
/// Runs the review and posts it, returning the result and the markdown that was
/// posted.
async fn review_pull_request(
ctx: &ReviewContext<'_>,
tools: Vec<Tool>,
payload: &ReviewPayload,
) -> anyhow::Result<(ReviewResult, String)> {
let full_name = &payload.repository.full_name;
let index = payload.pull_request.number;
let git_diff = ctx.gitea.pull_request_diff(full_name, index).await?;
// The file list only refines the paths and describes the changes: a failure is
// not fatal, the diff is the source of truth for the lines.
let files = match ctx.gitea.pull_request_files(full_name, index).await {
Ok(files) => files,
Err(err) => {
warn!(%err, "Failed to list the pull request files");
Vec::new()
}
};
let mut changed_lines = parse_changed_lines(&git_diff);
drop_generated_files(&mut changed_lines);
resolve_filenames(&mut changed_lines, &files);
let changes = format_changes(&files, &changed_lines);
let bot_request = REVIEW_PROMPT
.replace("{subject}", &payload.pull_request.title)
.replace("{comment}", &payload.comment.body)
.replace("{changes}", &changes);
let (mut review_result, cost) = run_sandboxed_review(ctx, tools, payload, &bot_request).await?;
resolve_review_sides(&mut review_result, &changed_lines);
review_result.cost = cost;
if let Some(cost) = review_result.cost {
metrics::openrouter_cost_usd(cost);
}
let markdown = review_result_to_markdown(&review_result);
ctx.gitea
.post_pull_request_review(&review_result, &markdown, full_name, index)
.await?;
Ok((review_result, markdown))
}
async fn run_sandboxed_review(
ctx: &ReviewContext<'_>,
tools: Vec<Tool>,
payload: &ReviewPayload,
bot_request: &str,
) -> anyhow::Result<(ReviewResult, Option<f64>)> {
let sandbox = Sandbox::create(
&ctx.sandbox.runtime,
&payload.repository.clone_url,
ctx.gitea.token(),
payload.pull_request.number,
ctx.reporter,
)
.await?;
let result = agent::run(
ctx.open_router,
&sandbox,
tools,
SANDBOX_SYSTEM_PROMPT,
bot_request,
ctx.sandbox.max_iterations,
ctx.reporter,
)
.await;
if let Err(err) = sandbox.cleanup().await {
warn!(%err, "Failed to clean up sandbox container");
}
let result = result?;
info!(
iterations = result.iterations,
cost = ?result.cost,
"Sandboxed review finished"
);
Ok((result.answer, result.cost))
}
@@ -0,0 +1,125 @@
//! Rendering of a pull request's changes for the model.
use super::diff::{ChangedLines, is_ignored};
use crate::gitea::PullRequestFile;
/// Renders the changes for the model: the files the pull request touches, then
/// the lines to review per file.
pub(super) fn format_changes(files: &[PullRequestFile], changed_lines: &ChangedLines) -> String {
let mut sections = Vec::new();
let described = files
.iter()
.filter(|file| !is_ignored(&file.filename))
.map(describe_file)
.collect::<Vec<_>>()
.join(", ");
if !described.is_empty() {
sections.push(format!("Files changed by the pull request: {described}"));
}
sections.push(format!(
"Lines to review, per file:\n{}",
format_changed_lines(changed_lines)
));
sections.join("\n\n")
}
/// Describes a changed file for the model, including how it changed.
fn describe_file(file: &PullRequestFile) -> String {
match &file.previous_filename {
Some(previous) => format!("{} ({} from {})", file.filename, file.status, previous),
None => format!("{} ({})", file.filename, file.status),
}
}
/// Renders the changed lines as `filename: added 1, 2 / removed 3`, one file per
/// line, keeping only the sides the file actually has.
pub(super) fn format_changed_lines(changed_lines: &ChangedLines) -> String {
changed_lines
.iter()
.map(|file| {
let mut sides = Vec::new();
for (label, lines) in [("added", &file.added), ("removed", &file.removed)] {
if !lines.is_empty() {
sides.push(format!("{label} {}", format_line_numbers(lines)));
}
}
format!("{}: {}", file.filename, sides.join(" / "))
})
.collect::<Vec<_>>()
.join("\n")
}
fn format_line_numbers(lines: &[u64]) -> String {
lines
.iter()
.map(u64::to_string)
.collect::<Vec<_>>()
.join(", ")
}
#[cfg(test)]
mod tests {
use super::*;
use crate::bot_actions::review::{diff::parse_changed_lines, fixtures::*};
#[test]
fn changed_lines_are_listed_per_file_and_side() {
let expected = concat!(
"src/foo.rs: added 2, 4, 5 / removed 2\n",
"src/bar.rs: added 11, 13"
);
assert_eq!(format_changed_lines(&parse_changed_lines(DIFF)), expected);
}
#[test]
fn changes_describe_the_files_then_the_lines() {
let changed_lines = parse_changed_lines(DELETION_ONLY);
let files = vec![pull_request_file("src/old.rs", None, "deleted")];
let expected = concat!(
"Files changed by the pull request: src/old.rs (deleted)\n",
"\n",
"Lines to review, per file:\n",
"src/old.rs: removed 1, 2, 3"
);
assert_eq!(format_changes(&files, &changed_lines), expected);
}
#[test]
fn changes_without_the_api_list_only_hold_the_lines() {
let changed_lines = parse_changed_lines(DELETION_ONLY);
let expected = "Lines to review, per file:\nsrc/old.rs: removed 1, 2, 3";
assert_eq!(format_changes(&[], &changed_lines), expected);
}
#[test]
fn a_generated_file_is_not_described_to_the_model() {
let files = vec![
pull_request_file("Cargo.lock", None, "modified"),
pull_request_file("src/foo.rs", None, "modified"),
];
let changed_lines = parse_changed_lines(DIFF);
let changes = format_changes(&files, &changed_lines);
assert!(!changes.contains("Cargo.lock"));
assert!(changes.contains("src/foo.rs (modified)"));
}
#[test]
fn a_renamed_file_is_described_with_its_previous_path() {
let file = pull_request_file("src/new.rs", Some("src/old.rs"), "renamed");
assert_eq!(describe_file(&file), "src/new.rs (renamed from src/old.rs)");
}
}
@@ -0,0 +1,110 @@
//! Markdown summary of a review, as posted on the pull request.
use crate::domain::review::{ReviewItem, ReviewResult, ReviewSeverity};
pub(super) fn review_result_to_markdown(review_result: &ReviewResult) -> String {
if review_result.reviews.is_empty() {
return String::from("No issues found. ✅");
}
let mut md = String::from("## Review Feedback\n\n");
md.push_str(&format!(
"### {} issues found.\n\n",
review_result.reviews.len()
));
let breakdown = severity_breakdown(&review_result.reviews);
if !breakdown.is_empty() {
md.push_str(&breakdown);
md.push('\n');
}
if !review_result.comment.is_empty() {
md.push_str("\n---\n\n");
md.push_str("### Summary\n\n");
md.push_str(&review_result.comment);
md.push('\n');
}
if let Some(cost) = review_result.cost {
md.push_str("\n---\n\n");
md.push_str(&format!("### Cost: ${}", cost));
md.push('\n');
}
md
}
/// Counts the reviews per severity, most severe first.
///
/// A review whose severity the model omitted or wrote unreadably is counted as
/// [`ReviewSeverity::Maintainability`], the same fallback used when posting it.
fn severity_breakdown(reviews: &[ReviewItem]) -> String {
[
ReviewSeverity::Bug,
ReviewSeverity::Security,
ReviewSeverity::Performance,
ReviewSeverity::Maintainability,
]
.into_iter()
.filter_map(|severity| {
let count = reviews
.iter()
.filter(|review| review.severity.unwrap_or(ReviewSeverity::Maintainability) == severity)
.count();
(count > 0).then(|| format!("- {} {}", count, severity.label()))
})
.collect::<Vec<_>>()
.join("\n")
}
#[cfg(test)]
mod tests {
use super::*;
use crate::bot_actions::review::fixtures::*;
#[test]
fn the_breakdown_counts_severities_most_severe_first() {
let reviews = vec![
ReviewItem {
severity: Some(ReviewSeverity::Maintainability),
..review("a", Some(1), None)
},
ReviewItem {
severity: Some(ReviewSeverity::Bug),
..review("b", Some(2), None)
},
ReviewItem {
severity: Some(ReviewSeverity::Bug),
..review("c", Some(3), None)
},
// A severity the model left out counts as maintainability.
review("d", Some(4), None),
];
assert_eq!(severity_breakdown(&reviews), "- 2 bug\n- 2 maintainability");
}
#[test]
fn the_markdown_breaks_the_issues_down_by_severity() {
let mut result = review_result(vec![
ReviewItem {
severity: Some(ReviewSeverity::Bug),
..review("a", Some(1), None)
},
ReviewItem {
severity: Some(ReviewSeverity::Performance),
..review("b", Some(2), None)
},
]);
result.comment = String::from("Le découpage en crates est propre.");
let markdown = review_result_to_markdown(&result);
assert!(markdown.contains("- 1 bug"));
assert!(markdown.contains("- 1 performance"));
assert!(markdown.contains("Le découpage en crates est propre."));
}
}
+35 -27
View File
@@ -1,7 +1,5 @@
use toasty::Db; use toasty::Db;
use crate::schemas::{Job, JobStatus};
pub async fn open(url: &str) -> toasty::Result<Db> { pub async fn open(url: &str) -> toasty::Result<Db> {
Db::builder() Db::builder()
.models(toasty::models!(crate::*)) .models(toasty::models!(crate::*))
@@ -9,35 +7,45 @@ pub async fn open(url: &str) -> toasty::Result<Db> {
.await .await
} }
pub async fn connect(url: &str) -> toasty::Result<Db> { /// A connection for a test, on the database `DATABASE_URL` points to.
let mut db = open(url).await?; ///
/// Every `#[tokio::test]` runs on its own runtime, and the connection tasks live on
Job::update_by_status(JobStatus::Processing) /// the runtime that opened the pool: sharing one `Db` between tests would leave
.status(JobStatus::Pending) /// them with a pool whose runtime is gone. Each test therefore opens its own, and
.exec(&mut db) /// only the migrations are applied once.
qpismont marked this conversation as resolved
Review

[bug] Au démarrage, tous les jobs Processing sont remis à Pending sans notion de propriétaire ou de lease. Si deux instances tournent simultanément, le démarrage de l'une peut réenfiler les jobs activement traités par l'autre, en plus de la course déjà présente dans le poller. La récupération après crash doit être limitée aux jobs dont le lease a expiré, et le traitement doit être atomiquement réservé.

**[bug]** Au démarrage, tous les jobs `Processing` sont remis à `Pending` sans notion de propriétaire ou de lease. Si deux instances tournent simultanément, le démarrage de l'une peut réenfiler les jobs activement traités par l'autre, en plus de la course déjà présente dans le poller. La récupération après crash doit être limitée aux jobs dont le lease a expiré, et le traitement doit être atomiquement réservé.
Review

[bug] Au démarrage, chaque instance remet à Pending tous les jobs Processing, y compris ceux qu’une autre instance est encore en train de traiter. Un redémarrage ou déploiement pendant une review peut donc la faire exécuter en double. La récupération doit distinguer les jobs réellement abandonnés de ceux détenus par des workers actifs.

**[bug]** Au démarrage, chaque instance remet à `Pending` tous les jobs `Processing`, y compris ceux qu’une autre instance est encore en train de traiter. Un redémarrage ou déploiement pendant une review peut donc la faire exécuter en double. La récupération doit distinguer les jobs réellement abandonnés de ceux détenus par des workers actifs.
.await?;
Ok(db)
}
#[cfg(test)] #[cfg(test)]
pub(crate) async fn test_connect() -> Db { pub(crate) async fn test_connect() -> Db {
static DB: tokio::sync::OnceCell<Db> = tokio::sync::OnceCell::const_new(); static MIGRATED: tokio::sync::OnceCell<()> = tokio::sync::OnceCell::const_new();
DB.get_or_init(|| async { let url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set");
let url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set");
let migrations = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("toasty"); // The migrations run on a runtime of their own, which outlives the first test.
let config = toasty_cli::Config::new() MIGRATED
.migration(toasty_cli::MigrationConfig::new().path(migrations)); .get_or_init(|| async {
let url = url.clone();
toasty_cli::ToastyCli::with_config(open(&url).await.expect("open pool"), config) std::thread::spawn(move || {
.parse_from(["toasty", "migration", "apply"]) tokio::runtime::Runtime::new()
.await .expect("migration runtime")
.expect("apply migrations"); .block_on(async {
let migrations =
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("toasty");
let config = toasty_cli::Config::new()
.migration(toasty_cli::MigrationConfig::new().path(migrations));
connect(&url).await.expect("connect to postgres") toasty_cli::ToastyCli::with_config(
}) open(&url).await.expect("open pool"),
.await config,
.clone() )
.parse_from(["toasty", "migration", "apply"])
.await
.expect("apply migrations");
});
})
.join()
.expect("migrations panicked");
})
.await;
open(&url).await.expect("connect to postgres")
} }
+1
View File
@@ -0,0 +1 @@
pub mod review;
+247
View File
@@ -0,0 +1,247 @@
//! Domain types of a review: what the model answers with, and how it is read.
use std::str::FromStr;
use serde::{Deserialize, Deserializer};
use tracing::warn;
use crate::text::excerpt;
#[derive(Deserialize, Debug)]
pub struct ReviewResult {
pub reviews: Vec<ReviewItem>,
pub comment: String,
pub cost: Option<f64>,
}
#[derive(Deserialize, Debug)]
pub struct ReviewItem {
pub filename: String,
pub line: Option<u64>,
#[serde(default, deserialize_with = "deserialize_side")]
pub side: Option<ReviewSide>,
#[serde(default, deserialize_with = "deserialize_severity")]
pub severity: Option<ReviewSeverity>,
pub message: String,
}
/// Which version of the file a review comment is anchored on.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReviewSide {
/// The line was added by the pull request: `line` is a line number of the
/// new version of the file.
Added,
/// The line was removed by the pull request: `line` is a line number of the
/// old version of the file.
Removed,
}
impl ReviewSide {
/// Reads the side the model asked for, tolerating casing and synonyms.
fn parse(raw: &str) -> Option<Self> {
match raw.trim().to_ascii_lowercase().as_str() {
"added" | "add" | "new" | "right" => Some(Self::Added),
"removed" | "remove" | "deleted" | "delete" | "old" | "left" => Some(Self::Removed),
_ => None,
}
}
}
/// What kind of problem a review reports.
///
/// The categories are the ones the prompt asks for; a review whose severity is
/// missing or unreadable falls back to [`ReviewSeverity::Maintainability`], the
/// least alarming bucket, instead of failing the whole review.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReviewSeverity {
/// Wrong behaviour.
Bug,
/// A vulnerability.
Security,
/// A resource problem.
Performance,
/// Everything else: readability, structure, naming, tests.
Maintainability,
}
impl ReviewSeverity {
/// Reads the severity the model asked for, tolerating casing and synonyms.
fn parse(raw: &str) -> Option<Self> {
match raw.trim().to_ascii_lowercase().as_str() {
"bug" | "bugs" | "correctness" | "error" => Some(Self::Bug),
"security" | "vulnerability" => Some(Self::Security),
"performance" | "perf" => Some(Self::Performance),
"maintainability" | "quality" | "style" => Some(Self::Maintainability),
_ => None,
}
}
/// Lower-case label used in the review markdown.
pub fn label(self) -> &'static str {
match self {
Self::Bug => "bug",
Self::Security => "security",
Self::Performance => "performance",
Self::Maintainability => "maintainability",
}
}
}
/// Reads the side the model asked for. An unreadable value is ignored instead of
/// failing the whole review: the side is then resolved from the changed lines.
fn deserialize_side<'de, D>(deserializer: D) -> Result<Option<ReviewSide>, D::Error>
where
D: Deserializer<'de>,
{
let raw = Option::<serde_json::Value>::deserialize(deserializer)?;
Ok(raw
.as_ref()
.and_then(serde_json::Value::as_str)
.and_then(ReviewSide::parse))
}
/// Reads the severity the model asked for, ignoring an unreadable value: the
/// review is then reported as [`ReviewSeverity::Maintainability`].
fn deserialize_severity<'de, D>(deserializer: D) -> Result<Option<ReviewSeverity>, D::Error>
where
D: Deserializer<'de>,
{
let raw = Option::<serde_json::Value>::deserialize(deserializer)?;
Ok(raw
.as_ref()
.and_then(serde_json::Value::as_str)
.and_then(ReviewSeverity::parse))
}
/// Number of characters of a model answer kept in the logs when it cannot be
/// parsed.
const MAX_LOGGED_ANSWER: usize = 500;
impl FromStr for ReviewResult {
type Err = anyhow::Error;
/// Parses the review the model answered with.
///
/// This is the contract the agent loop enforces: a rejected answer is sent
/// back to the model, with the reason, so that it can correct itself.
///
/// The contract is a raw JSON object, but models sometimes wrap it in a
/// markdown code fence or surround it with a sentence: the object is then
/// extracted from the answer before failing, and the answer is logged so a
/// breach of the contract can be diagnosed.
fn from_str(message: &str) -> Result<Self, Self::Err> {
let error = match serde_json::from_str::<Self>(message) {
Ok(review_result) => return Ok(review_result),
Err(error) => error,
};
// A markdown code fence or a sentence around the object is tolerated, with
// a warning: the contract asks for a raw JSON object.
if let Some(json) = json_object(message)
&& let Ok(review_result) = serde_json::from_str::<Self>(json)
{
warn!(
"Model answer is not a raw JSON object, it was extracted from the surrounding text"
);
return Ok(review_result);
}
// The reason of the rejection is logged along with the answer: without it,
// a broken answer is impossible to diagnose.
warn!(
answer = %excerpt(message, MAX_LOGGED_ANSWER),
reason = %error,
"Model answer is not the expected JSON"
);
anyhow::bail!("the answer is not valid JSON: {error}")
}
}
/// Returns the outermost `{...}` of an answer, which ignores a markdown code
/// fence or any text around it.
fn json_object(message: &str) -> Option<&str> {
let start = message.find('{')?;
let end = message.rfind('}')?;
(start < end).then(|| &message[start..=end])
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_raw_answer_is_parsed() {
let answer = r#"{"reviews": [], "comment": "ok"}"#;
assert_eq!(answer.parse::<ReviewResult>().unwrap().comment, "ok");
}
#[test]
fn a_fenced_answer_is_extracted() {
let answer = concat!(
"Voici ma review :\n",
"```json\n",
"{\"reviews\": [], \"comment\": \"rien à signaler\"}\n",
"```\n"
);
assert_eq!(
answer.parse::<ReviewResult>().unwrap().comment,
"rien à signaler"
);
}
#[test]
fn a_sentence_around_the_object_is_ignored() {
let answer = r#"Rien à signaler. {"reviews": [], "comment": "ok"} Bonne journée !"#;
assert_eq!(answer.parse::<ReviewResult>().unwrap().comment, "ok");
}
#[test]
fn an_answer_without_json_is_rejected() {
assert!("Je n'ai rien relevé.".parse::<ReviewResult>().is_err());
}
#[test]
fn an_answer_that_is_an_object_but_not_a_review_is_rejected() {
assert!(r#"{"message": "LGTM"}"#.parse::<ReviewResult>().is_err());
}
#[test]
fn severities_are_read_from_the_model() {
let result: ReviewResult = serde_json::from_str(
r#"{
"reviews": [
{ "filename": "a", "line": 1, "severity": "Bug", "message": "x" },
{ "filename": "b", "line": 2, "severity": "SECURITY", "message": "x" },
{ "filename": "c", "line": 3, "severity": "perf", "message": "x" },
{ "filename": "d", "line": 4, "severity": "banana", "message": "x" }
],
"comment": ""
}"#,
)
.unwrap();
let severities = result
.reviews
.iter()
.map(|review| review.severity)
.collect::<Vec<_>>();
assert_eq!(
severities,
vec![
Some(ReviewSeverity::Bug),
Some(ReviewSeverity::Security),
Some(ReviewSeverity::Performance),
None,
]
);
}
}
+97 -32
View File
@@ -1,4 +1,9 @@
use anyhow::anyhow; use std::{str::FromStr, time::Duration};
use anyhow::Context;
/// Number of tool-calling iterations a sandboxed review may take, unless configured.
const DEFAULT_SANDBOX_MAX_ITERATIONS: usize = 8;
#[derive(Clone)] #[derive(Clone)]
pub struct EnvConfig { pub struct EnvConfig {
@@ -6,46 +11,31 @@ pub struct EnvConfig {
pub webhook_secret: String, pub webhook_secret: String,
pub open_router_api_key: String, pub open_router_api_key: String,
pub open_router_model: String, pub open_router_model: String,
pub open_router_timeout: u64, pub open_router_timeout: Duration,
pub bot_max_concurrent: usize, pub bot_max_concurrent: usize,
pub gitea_url: String, pub gitea_url: String,
pub gitea_token: String, pub gitea_token: String,
pub gitea_timeout: u64, pub gitea_timeout: Duration,
pub database_url: String, pub database_url: String,
pub metrics_bind_addr: Option<String>, pub metrics_bind_addr: Option<String>,
pub sandbox_max_iterations: usize, pub sandbox_max_iterations: usize,
} }
pub fn load_config() -> anyhow::Result<EnvConfig> { pub fn load_config() -> anyhow::Result<EnvConfig> {
let http_port = try_get_env("HTTP_PORT")?.parse()?;
let webhook_secret = try_get_env("WEBHOOK_SIG_HEADER_SECRET")?;
let open_router_api_key = try_get_env("OPEN_ROUTER_API_KEY")?;
let open_router_model = try_get_env("OPEN_ROUTER_MODEL")?;
let open_router_timeout = try_get_env("OPEN_ROUTER_TIMEOUT")?.parse()?;
let bot_max_concurrent = try_get_env("BOT_MAX_CONCURRENT")?.parse()?;
let gitea_url = try_get_env("GITEA_URL")?;
let gitea_token = try_get_env("GITEA_TOKEN")?;
let gitea_timeout = try_get_env("GITEA_TIMEOUT")?.parse()?;
let database_url = try_get_env("DATABASE_URL")?;
let metrics_bind_addr = std::env::var("METRICS_BIND_ADDR").ok();
let sandbox_max_iterations = std::env::var("SANDBOX_MAX_ITERATIONS")
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(8);
Ok(EnvConfig { Ok(EnvConfig {
http_port, http_port: parse_env("HTTP_PORT")?,
webhook_secret, webhook_secret: try_get_env("WEBHOOK_SIG_HEADER_SECRET")?,
open_router_api_key, open_router_api_key: try_get_env("OPEN_ROUTER_API_KEY")?,
open_router_model, open_router_model: try_get_env("OPEN_ROUTER_MODEL")?,
open_router_timeout, open_router_timeout: parse_seconds_env("OPEN_ROUTER_TIMEOUT")?,
bot_max_concurrent, bot_max_concurrent: parse_env("BOT_MAX_CONCURRENT")?,
gitea_url, gitea_url: try_get_env("GITEA_URL")?,
gitea_token, gitea_token: try_get_env("GITEA_TOKEN")?,
gitea_timeout, gitea_timeout: parse_seconds_env("GITEA_TIMEOUT")?,
database_url, database_url: try_get_env("DATABASE_URL")?,
metrics_bind_addr, metrics_bind_addr: std::env::var("METRICS_BIND_ADDR").ok(),
sandbox_max_iterations, sandbox_max_iterations: parse_optional_env("SANDBOX_MAX_ITERATIONS")?
.unwrap_or(DEFAULT_SANDBOX_MAX_ITERATIONS),
}) })
} }
@@ -53,16 +43,57 @@ pub fn try_get_env(key: &str) -> anyhow::Result<String> {
let env_value = std::env::var(key).map_err(|e| anyhow::anyhow!("{}: {}", key, e))?; let env_value = std::env::var(key).map_err(|e| anyhow::anyhow!("{}: {}", key, e))?;
if env_value.trim().is_empty() { if env_value.trim().is_empty() {
return Err(anyhow!(format!("env var {} is empty", key))); anyhow::bail!("env var {key} is empty");
} }
Ok(env_value) Ok(env_value)
} }
/// Reads a required variable and parses it, naming the variable when it is invalid.
fn parse_env<T>(key: &str) -> anyhow::Result<T>
where
T: FromStr,
T::Err: std::error::Error + Send + Sync + 'static,
{
parse_value(key, &try_get_env(key)?)
}
/// Reads an optional variable and parses it. A missing or empty variable is `None`;
/// one that is set but invalid is an error, like it is for the required ones.
fn parse_optional_env<T>(key: &str) -> anyhow::Result<Option<T>>
where
T: FromStr,
T::Err: std::error::Error + Send + Sync + 'static,
{
match std::env::var(key) {
Ok(value) if !value.trim().is_empty() => parse_value(key, &value).map(Some),
_ => Ok(None),
}
}
/// Reads a required variable holding a number of seconds.
fn parse_seconds_env(key: &str) -> anyhow::Result<Duration> {
parse_env::<u64>(key).map(Duration::from_secs)
}
fn parse_value<T>(key: &str, value: &str) -> anyhow::Result<T>
where
T: FromStr,
T::Err: std::error::Error + Send + Sync + 'static,
{
value
.trim()
.parse()
.with_context(|| format!("env var {key} is invalid: `{value}`"))
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
// Every test uses its own variable: the environment is shared by the tests,
// which run in parallel.
#[test] #[test]
fn test_try_get_env_returns_value() { fn test_try_get_env_returns_value() {
unsafe { std::env::set_var("TEST_ENV_PRESENT", "hello") }; unsafe { std::env::set_var("TEST_ENV_PRESENT", "hello") };
@@ -88,4 +119,38 @@ mod tests {
let err = try_get_env("TEST_ENV_WHITESPACE").unwrap_err(); let err = try_get_env("TEST_ENV_WHITESPACE").unwrap_err();
assert!(err.to_string().contains("TEST_ENV_WHITESPACE")); assert!(err.to_string().contains("TEST_ENV_WHITESPACE"));
} }
#[test]
fn a_timeout_is_read_as_seconds() {
unsafe { std::env::set_var("TEST_ENV_SECONDS", "30") };
assert_eq!(
parse_seconds_env("TEST_ENV_SECONDS").unwrap(),
Duration::from_secs(30)
);
}
#[test]
fn an_invalid_number_names_its_variable() {
unsafe { std::env::set_var("TEST_ENV_NOT_A_NUMBER", "soon") };
let err = parse_env::<u16>("TEST_ENV_NOT_A_NUMBER").unwrap_err();
assert!(format!("{err:#}").contains("TEST_ENV_NOT_A_NUMBER"));
assert!(format!("{err:#}").contains("soon"));
}
#[test]
fn an_absent_optional_variable_is_none() {
unsafe { std::env::remove_var("TEST_ENV_OPTIONAL_ABSENT") };
assert_eq!(
parse_optional_env::<usize>("TEST_ENV_OPTIONAL_ABSENT").unwrap(),
None
);
}
#[test]
fn an_invalid_optional_variable_is_an_error_instead_of_the_default() {
unsafe { std::env::set_var("TEST_ENV_OPTIONAL_INVALID", "many") };
assert!(parse_optional_env::<usize>("TEST_ENV_OPTIONAL_INVALID").is_err());
}
} }
+2 -2
View File
@@ -11,7 +11,7 @@ const PROGRESS_CAPACITY: usize = 256;
const HISTORY_CAPACITY: usize = 256; const HISTORY_CAPACITY: usize = 256;
const MAX_EVENT_ARGUMENTS: usize = 200; const MAX_EVENT_ARGUMENTS: usize = 200;
/// Évènement de cycle de vie d'un job, diffusé globalement. /// Lifecycle event of a job, broadcast globally.
#[derive(Clone, Debug, Serialize)] #[derive(Clone, Debug, Serialize)]
#[serde(tag = "type", rename_all = "snake_case")] #[serde(tag = "type", rename_all = "snake_case")]
pub enum JobEvent { pub enum JobEvent {
@@ -212,7 +212,7 @@ impl Reporter {
}); });
} }
/// Signale la fin d'un appel d'outil. /// Reports the end of a tool call.
pub fn tool_result(&self, iteration: usize, name: &str, truncated: bool, output: &str) { pub fn tool_result(&self, iteration: usize, name: &str, truncated: bool, output: &str) {
self.channel.emit(Progress::ToolResult { self.channel.emit(Progress::ToolResult {
iteration, iteration,
1
+25 -23
View File
@@ -8,8 +8,8 @@ use tokio_util::io::StreamReader;
use tracing::{instrument, warn}; use tracing::{instrument, warn};
use crate::{ use crate::{
bot::{ReviewResult, ReviewSeverity, ReviewSide},
consts::MAX_DIFF_SIZE, consts::MAX_DIFF_SIZE,
domain::review::{ReviewResult, ReviewSeverity, ReviewSide},
errors::AppError, errors::AppError,
}; };
@@ -27,7 +27,7 @@ pub struct GiteaAPI {
} }
impl GiteaAPI { impl GiteaAPI {
pub fn new(base_url: &str, token: &str, timeout: u64) -> anyhow::Result<Self> { pub fn new(base_url: &str, token: &str, timeout: Duration) -> anyhow::Result<Self> {
let mut default_headers = reqwest::header::HeaderMap::new(); let mut default_headers = reqwest::header::HeaderMap::new();
default_headers.insert( default_headers.insert(
reqwest::header::HeaderName::from_static("authorization"), reqwest::header::HeaderName::from_static("authorization"),
@@ -38,7 +38,7 @@ impl GiteaAPI {
base_url: String::from(base_url), base_url: String::from(base_url),
token: String::from(token), token: String::from(token),
client: reqwest::Client::builder() client: reqwest::Client::builder()
.timeout(Duration::from_secs(timeout)) .timeout(timeout)
.default_headers(default_headers) .default_headers(default_headers)
.build()?, .build()?,
}) })
@@ -320,6 +320,21 @@ pub struct ReviewPayload {
pub comment: Comment, pub comment: Comment,
} }
impl ReviewPayload {
/// Checks that the comment is a new one that mentions the bot.
fn check_addressed_to(&self, bot_name: &str) -> Result<(), AppError> {
if self.action != "created" {
return Err(AppError::InvalidActionErr);
}
if !self.comment.body.starts_with(&format!("@{bot_name}")) {
return Err(AppError::UnauthorizedUserErr);
}
Ok(())
}
}
#[derive(Deserialize, Serialize, Debug, Clone)] #[derive(Deserialize, Serialize, Debug, Clone)]
pub struct PullRequest { pub struct PullRequest {
pub number: u64, pub number: u64,
@@ -357,28 +372,15 @@ pub struct PullRequestFile {
impl WebhookType { impl WebhookType {
pub fn from_event(event: &str, bot_name: &str, json: Value) -> Result<Self, AppError> { pub fn from_event(event: &str, bot_name: &str, json: Value) -> Result<Self, AppError> {
let wb = match event { match event {
"pull_request_comment" => Ok(WebhookType::Review(serde_json::from_value(json)?)), "pull_request_comment" => {
let payload: ReviewPayload = serde_json::from_value(json)?;
payload.check_addressed_to(bot_name)?;
Ok(WebhookType::Review(payload))
}
_ => Err(AppError::UnknownEventErr), _ => Err(AppError::UnknownEventErr),
}?;
let action = match &wb {
WebhookType::Review(review_payload) => &review_payload.action,
};
if action != "created" {
return Err(AppError::InvalidActionErr);
} }
let pr_body = match &wb {
WebhookType::Review(review_payload) => &review_payload.comment.body,
};
if !pr_body.starts_with(&format!("@{}", bot_name)) {
return Err(AppError::UnauthorizedUserErr);
}
Ok(wb)
} }
} }
+157 -79
View File
@@ -1,26 +1,30 @@
use std::{collections::VecDeque, time::Duration}; use std::sync::Arc;
use futures_util::stream::{self, Stream}; use toasty::{Db, stmt::Value};
use toasty::Db; use tokio::sync::Notify;
use tracing::error;
use crate::{ use crate::{
gitea::WebhookType, gitea::WebhookType,
schemas::{Job, JobStatus}, schemas::{Job, JobStatus},
}; };
/// Délai entre deux interrogations de la table quand la file est vide. /// The reviews to do, stored in PostgreSQL.
const POLL_INTERVAL: Duration = Duration::from_secs(1); ///
/// Steep runs as a single instance, so every job is pushed by this process: the
/// File des reviews à faire. /// worker is woken up by [`JobQueue::push`] instead of polling the table. The table
/// only makes the queue survive a restart.
#[derive(Clone)] #[derive(Clone)]
pub struct JobQueue { pub struct JobQueue {
db: Db, db: Db,
work: Arc<Notify>,
} }
impl JobQueue { impl JobQueue {
pub fn new(db: Db) -> Self { pub fn new(db: Db) -> Self {
Self { db } Self {
db,
work: Arc::new(Notify::new()),
}
} }
pub async fn push(&self, webhook: WebhookType) -> toasty::Result<Job> { pub async fn push(&self, webhook: WebhookType) -> toasty::Result<Job> {
@@ -40,54 +44,76 @@ impl JobQueue {
.exec(&mut db) .exec(&mut db)
.await?; .await?;
// `notify_one` keeps a permit when nobody is waiting yet, so a job pushed
// while the worker is busy is not missed.
self.work.notify_one();
Ok(job) Ok(job)
} }
pub fn pending(&self) -> impl Stream<Item = Job> { /// Whether a job is already queued for this event.
let db = self.db.clone(); pub async fn is_queued(&self, event_id: u64) -> toasty::Result<bool> {
stream::unfold(VecDeque::new(), move |mut buffer| {
let db = db.clone();
async move {
let mut db = db;
loop {
if let Some(job) = buffer.pop_front() {
return Some((job, buffer));
}
match Job::filter_by_status(JobStatus::Pending)
.exec(&mut db)
.await
{
Ok(jobs) if jobs.is_empty() => {
tokio::time::sleep(POLL_INTERVAL).await;
}
Ok(jobs) => buffer.extend(jobs),
Err(err) => {
error!(%err, "failed to poll pending jobs");
tokio::time::sleep(POLL_INTERVAL).await;
}
}
}
}
})
}
/// Marque un job comme en cours de traitement.
pub async fn mark_processing(&self, id: u64) -> toasty::Result<()> {
let mut db = self.db.clone(); let mut db = self.db.clone();
Job::update_by_id(id) Ok(Job::filter_by_event_id(event_id)
.status(JobStatus::Processing) .first()
.exec(&mut db)
qpismont marked this conversation as resolved
Review

[bug] La sélection des jobs Pending et leur passage à Processing sont deux opérations distinctes, sans réservation atomique. Avec plusieurs instances, chacune peut récupérer le même job et lancer la même review avant que l’autre ne le marque Processing. Il faut réclamer les jobs de façon atomique.

**[bug]** La sélection des jobs Pending et leur passage à Processing sont deux opérations distinctes, sans réservation atomique. Avec plusieurs instances, chacune peut récupérer le même job et lancer la même review avant que l’autre ne le marque Processing. Il faut réclamer les jobs de façon atomique.
.await?
.is_some())
}
/// Puts back in the queue the jobs a previous process left `processing`.
///
/// To call once at startup, before the worker starts: it is only correct because
/// no other instance can be working on those jobs.
qpismont marked this conversation as resolved Outdated
Outdated
Review

[bug] La lecture des jobs Pending et leur passage à Processing ne sont pas atomiques. Avec plusieurs instances du serveur (ou plusieurs consommateurs), elles peuvent toutes récupérer le même job avant l'appel à mark_processing, puis exécuter plusieurs fois la review et publier des commentaires dupliqués. Il faut effectuer une réservation conditionnelle en base (UPDATE ... WHERE status = 'pending' ... RETURNING) plutôt que seulement filtrer puis mettre à jour.

**[bug]** La lecture des jobs `Pending` et leur passage à `Processing` ne sont pas atomiques. Avec plusieurs instances du serveur (ou plusieurs consommateurs), elles peuvent toutes récupérer le même job avant l'appel à `mark_processing`, puis exécuter plusieurs fois la review et publier des commentaires dupliqués. Il faut effectuer une réservation conditionnelle en base (`UPDATE ... WHERE status = 'pending' ... RETURNING`) plutôt que seulement filtrer puis mettre à jour.
Outdated
Review

[performance] Cette requête récupère tous les jobs en attente sans limite, puis les accumule dans le VecDeque à la ligne suivante. Une file PostgreSQL qui reçoit plus de jobs qu'elle n'en traite peut donc faire croître la mémoire du processus sans borne pratique. Il faudrait paginer ou limiter le nombre de jobs réservés par poll.

**[performance]** Cette requête récupère tous les jobs en attente sans limite, puis les accumule dans le `VecDeque` à la ligne suivante. Une file PostgreSQL qui reçoit plus de jobs qu'elle n'en traite peut donc faire croître la mémoire du processus sans borne pratique. Il faudrait paginer ou limiter le nombre de jobs réservés par poll.
Review

[performance] Cette requête récupère tous les jobs en attente, puis les met en mémoire dans le buffer. Si la file grossit ou si le service redémarre avec un grand retard à rattraper, chaque worker peut charger un volume non borné de payloads JSON. Limitez la taille des lots et ne chargez que les jobs nécessaires à la capacité disponible.

**[performance]** Cette requête récupère tous les jobs en attente, puis les met en mémoire dans le buffer. Si la file grossit ou si le service redémarre avec un grand retard à rattraper, chaque worker peut charger un volume non borné de payloads JSON. Limitez la taille des lots et ne chargez que les jobs nécessaires à la capacité disponible.
Review

[bug] La lecture des jobs en attente n’en réserve aucun : deux instances peuvent récupérer le même job avant que l’une d’elles n’appelle mark_processing, puis lancer deux reviews et poster des résultats en double. Il faut réclamer les jobs de façon atomique, par exemple avec une mise à jour conditionnelle ou un verrouillage SKIP LOCKED.

**[bug]** La lecture des jobs en attente n’en réserve aucun : deux instances peuvent récupérer le même job avant que l’une d’elles n’appelle `mark_processing`, puis lancer deux reviews et poster des résultats en double. Il faut réclamer les jobs de façon atomique, par exemple avec une mise à jour conditionnelle ou un verrouillage `SKIP LOCKED`.
pub async fn requeue_interrupted(&self) -> toasty::Result<()> {
let mut db = self.db.clone();
Job::update_by_status(JobStatus::Processing)
.status(JobStatus::Pending)
.exec(&mut db) .exec(&mut db)
.await?; .await?;
Ok(()) Ok(())
} }
/// Retire un job de la file, une fois la review enregistrée. /// Takes the oldest pending job and marks it `processing`, or returns `None` when
/// the queue is empty.
///
/// The selection and the update are a single statement, so a job cannot be taken
/// twice.
pub async fn claim_next(&self) -> toasty::Result<Option<Job>> {
let mut db = self.db.clone();
let rows = toasty::sql::query(
"UPDATE jobs
SET status = 'processing'
WHERE id = (
SELECT id FROM jobs
WHERE status = 'pending'
ORDER BY id
LIMIT 1
)
RETURNING id",
)
.exec(&mut db)
.await?;
let Some(Value::Record(row)) = rows.into_iter().next() else {
return Ok(None);
};
let id = u64::try_from(row.as_slice()[0].clone())?;
Ok(Some(Job::get_by_id(&mut db, id).await?))
}
/// Waits until a job may have been pushed.
pub async fn wait_for_work(&self) {
self.work.notified().await;
}
/// Removes a job from the queue, once its review is recorded.
pub async fn remove(&self, id: u64) -> toasty::Result<()> { pub async fn remove(&self, id: u64) -> toasty::Result<()> {
let mut db = self.db.clone(); let mut db = self.db.clone();
@@ -108,11 +134,13 @@ impl JobQueue {
mod tests { mod tests {
use std::time::Duration; use std::time::Duration;
use futures_util::StreamExt; use tokio::sync::{Mutex, MutexGuard};
use super::*; use super::*;
use crate::gitea::{Comment, PullRequest, Repository, ReviewPayload}; use crate::gitea::{Comment, PullRequest, Repository, ReviewPayload};
const FIRST_EVENT: u64 = 999_000_200;
fn webhook(event_id: u64) -> WebhookType { fn webhook(event_id: u64) -> WebhookType {
WebhookType::Review(ReviewPayload { WebhookType::Review(ReviewPayload {
action: "created".to_string(), action: "created".to_string(),
@@ -131,41 +159,40 @@ mod tests {
}) })
} }
async fn next_job(queue: &JobQueue, event_id: u64) -> Job { /// The tests claim whichever job is next, so they must not overlap: one would
let mut pending = std::pin::pin!(queue.pending()); /// claim the jobs of another.
async fn exclusive_queue() -> (MutexGuard<'static, ()>, JobQueue, Db) {
static LOCK: Mutex<()> = Mutex::const_new(());
tokio::time::timeout(Duration::from_secs(10), async { let guard = LOCK.lock().await;
loop { let db = crate::db::test_connect().await;
if let Some(job) = pending.next().await
&& job.event_id == event_id toasty::sql::statement(
{ "DELETE FROM jobs WHERE event_id >= 999000200 AND event_id < 999000300",
break job; )
} .exec(&mut db.clone())
}
})
.await .await
.expect("timed out waiting for the job") .unwrap();
(guard, JobQueue::new(db.clone()), db)
} }
#[tokio::test] #[tokio::test]
#[ignore = "requires DATABASE_URL"] #[ignore = "requires DATABASE_URL"]
async fn a_pushed_job_is_pending_then_removed() { async fn a_pushed_job_is_claimed_then_removed() {
let db = crate::db::test_connect().await; let (_guard, queue, db) = exclusive_queue().await;
let queue = JobQueue::new(db.clone());
let event_id = 999_000_201; queue.push(webhook(FIRST_EVENT)).await.unwrap();
Job::delete_by_event_id(&mut db.clone(), event_id) assert!(queue.is_queued(FIRST_EVENT).await.unwrap());
.await
.ok();
queue.push(webhook(event_id)).await.unwrap(); let job = queue.claim_next().await.unwrap().expect("a job to claim");
assert_eq!(job.event_id, FIRST_EVENT);
assert_eq!(job.status, JobStatus::Processing);
let job = next_job(&queue, event_id).await;
queue.mark_processing(job.id).await.unwrap();
queue.remove(job.id).await.unwrap(); queue.remove(job.id).await.unwrap();
assert!( assert!(
Job::filter_by_event_id(event_id) Job::filter_by_event_id(FIRST_EVENT)
.first() .first()
.exec(&mut db.clone()) .exec(&mut db.clone())
.await .await
@@ -177,19 +204,70 @@ mod tests {
#[tokio::test] #[tokio::test]
#[ignore = "requires DATABASE_URL"] #[ignore = "requires DATABASE_URL"]
async fn a_duplicate_event_id_is_rejected() { async fn a_duplicate_event_id_is_rejected() {
let db = crate::db::test_connect().await; let (_guard, queue, _db) = exclusive_queue().await;
let queue = JobQueue::new(db.clone());
let event_id = 999_000_202; queue.push(webhook(FIRST_EVENT + 1)).await.unwrap();
Job::delete_by_event_id(&mut db.clone(), event_id) assert!(queue.push(webhook(FIRST_EVENT + 1)).await.is_err());
.await }
.ok();
queue.push(webhook(event_id)).await.unwrap(); #[tokio::test]
assert!(queue.push(webhook(event_id)).await.is_err()); #[ignore = "requires DATABASE_URL"]
async fn a_claimed_job_is_not_claimed_again() {
let (_guard, queue, _db) = exclusive_queue().await;
Job::delete_by_event_id(&mut db.clone(), event_id) queue.push(webhook(FIRST_EVENT + 2)).await.unwrap();
assert!(queue.claim_next().await.unwrap().is_some());
assert!(queue.claim_next().await.unwrap().is_none());
}
#[tokio::test]
#[ignore = "requires DATABASE_URL"]
async fn jobs_are_claimed_oldest_first() {
let (_guard, queue, _db) = exclusive_queue().await;
queue.push(webhook(FIRST_EVENT + 3)).await.unwrap();
queue.push(webhook(FIRST_EVENT + 4)).await.unwrap();
let first = queue.claim_next().await.unwrap().unwrap();
let second = queue.claim_next().await.unwrap().unwrap();
assert_eq!(first.event_id, FIRST_EVENT + 3);
assert_eq!(second.event_id, FIRST_EVENT + 4);
}
#[tokio::test]
#[ignore = "requires DATABASE_URL"]
async fn an_interrupted_job_is_requeued() {
let (_guard, queue, _db) = exclusive_queue().await;
queue.push(webhook(FIRST_EVENT + 5)).await.unwrap();
let job = queue.claim_next().await.unwrap().unwrap();
assert!(queue.claim_next().await.unwrap().is_none());
// The process died here: on restart, the job goes back to the queue.
queue.requeue_interrupted().await.unwrap();
let again = queue.claim_next().await.unwrap().expect("the job back");
assert_eq!(again.id, job.id);
}
#[tokio::test]
#[ignore = "requires DATABASE_URL"]
async fn waiting_for_work_returns_as_soon_as_a_job_is_pushed() {
let (_guard, queue, _db) = exclusive_queue().await;
let waiter = {
let queue = queue.clone();
tokio::spawn(async move { queue.wait_for_work().await })
};
tokio::time::sleep(Duration::from_millis(100)).await;
queue.push(webhook(FIRST_EVENT + 6)).await.unwrap();
tokio::time::timeout(Duration::from_secs(1), waiter)
.await .await
.expect("the worker was not woken up")
.unwrap(); .unwrap();
} }
} }
+36 -18
View File
@@ -1,8 +1,14 @@
use std::time::Duration; use std::time::Duration;
use crate::{ use crate::{
bot::Bot, dedup::Dedup, event_bus::EventBus, gitea::GiteaAPI, job_queue::JobQueue, bot::{Bot, BotDeps},
open_router::OpenRouterClient, sandbox::SandboxConfig, state::AppState, event_bus::EventBus,
gitea::GiteaAPI,
job_queue::JobQueue,
open_router::OpenRouterClient,
review_store::ReviewStore,
sandbox::SandboxConfig,
state::AppState,
}; };
use dotenvy::dotenv; use dotenvy::dotenv;
@@ -16,7 +22,7 @@ mod bot;
mod bot_actions; mod bot_actions;
mod consts; mod consts;
mod db; mod db;
mod dedup; mod domain;
mod env; mod env;
mod errors; mod errors;
mod event_bus; mod event_bus;
@@ -24,21 +30,29 @@ mod gitea;
mod job_queue; mod job_queue;
mod metrics; mod metrics;
mod open_router; mod open_router;
mod review_store;
mod sandbox; mod sandbox;
mod schemas; mod schemas;
mod state; mod state;
mod text; mod text;
/// Délai laissé aux reviews en cours et aux serveurs pour s'arrêter proprement. /// Time left to the running reviews and the servers to stop cleanly.
/// ///
/// Sans lui, une review prise dans une sandbox récalcitrante garderait le processus /// Without it, a review stuck in an unresponsive sandbox would keep the process
/// en vie jusqu'à ce que le superviseur le tue. /// alive until the supervisor kills it.
const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(60); const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(60);
fn main() -> anyhow::Result<()> { fn main() -> anyhow::Result<()> {
dotenv().ok(); dotenv().ok();
// Sous-commande de migration : `steep-server migration <generate|apply|...>`. // TLS goes through rustls, which has no default crypto provider here: the
// workspace compiles `ring` only, and it is installed once, for every client
// (reqwest, sentry).
rustls::crypto::ring::default_provider()
.install_default()
.map_err(|_| anyhow::anyhow!("a rustls crypto provider is already installed"))?;
// Migration subcommand: `steep-server migration <generate|apply|...>`.
if std::env::args().nth(1).as_deref() == Some("migration") { if std::env::args().nth(1).as_deref() == Some("migration") {
return tokio::runtime::Runtime::new()?.block_on(run_migrations()); return tokio::runtime::Runtime::new()?.block_on(run_migrations());
} }
@@ -46,7 +60,7 @@ fn main() -> anyhow::Result<()> {
tracing_subscriber::registry() tracing_subscriber::registry()
.with(fmt::layer()) .with(fmt::layer())
.with( .with(
EnvFilter::try_from_default_env() // lit RUST_LOG depuis l'env EnvFilter::try_from_default_env() // reads RUST_LOG from the environment
.unwrap_or_else(|_| EnvFilter::new("info")), .unwrap_or_else(|_| EnvFilter::new("info")),
) )
.init(); .init();
@@ -124,25 +138,29 @@ async fn run() -> anyhow::Result<()> {
); );
} }
let db = db::connect(&config.database_url).await?; let db = db::open(&config.database_url).await?;
let dedup = Dedup::new(db.clone()); let reviews = ReviewStore::new(db.clone());
let queue = JobQueue::new(db); let queue = JobQueue::new(db);
queue.requeue_interrupted().await?;
let event_bus = EventBus::new(); let event_bus = EventBus::new();
let bot = Bot::new( let bot = Bot::new(
gitea_api, BotDeps {
open_router_client, gitea: gitea_api,
open_router: open_router_client,
sandbox,
reviews: reviews.clone(),
queue: queue.clone(),
event_bus: event_bus.clone(),
},
config.bot_max_concurrent, config.bot_max_concurrent,
config.open_router_model.clone(),
sandbox,
dedup.clone(),
queue.clone(),
event_bus.clone(),
); );
let app_state = AppState { let app_state = AppState {
queue, queue,
dedup, reviews,
event_bus, event_bus,
bot_name: gitea_user.login.clone(), bot_name: gitea_user.login.clone(),
config, config,
+7 -4
View File
@@ -254,16 +254,19 @@ pub struct OpenRouterClient {
} }
impl OpenRouterClient { impl OpenRouterClient {
pub fn new(token: &str, model: &str, timeout: u64) -> anyhow::Result<Self> { pub fn new(token: &str, model: &str, timeout: Duration) -> anyhow::Result<Self> {
Ok(Self { Ok(Self {
client: reqwest::Client::builder() client: reqwest::Client::builder().timeout(timeout).build()?,
.timeout(Duration::from_secs(timeout))
.build()?,
api_key: String::from(token), api_key: String::from(token),
model: String::from(model), model: String::from(model),
}) })
} }
/// The model every completion is requested from.
pub fn model(&self) -> &str {
&self.model
}
/// Sends a conversation with tool definitions and returns either a final /// Sends a conversation with tool definitions and returns either a final
/// message or the tool calls requested by the model. /// message or the tool calls requested by the model.
/// ///
@@ -1,7 +1,8 @@
use toasty::Db; use toasty::Db;
use crate::schemas::{Job, Review}; use crate::schemas::Review;
/// What is kept of a processed review.
pub struct ReviewRecord { pub struct ReviewRecord {
pub event_id: u64, pub event_id: u64,
pub repo: String, pub repo: String,
@@ -14,35 +15,19 @@ pub struct ReviewRecord {
pub duration_ms: u64, pub duration_ms: u64,
} }
/// The history of the reviews Steep has processed.
///
/// It also is what keeps an event from being reviewed twice.
#[derive(Clone)] #[derive(Clone)]
pub struct Dedup { pub struct ReviewStore {
db: Db, db: Db,
} }
impl Dedup { impl ReviewStore {
pub fn new(db: Db) -> Self { pub fn new(db: Db) -> Self {
Self { db } Self { db }
} }
pub async fn is_known(&self, event_id: u64) -> toasty::Result<bool> {
let mut db = self.db.clone();
if Job::filter_by_event_id(event_id)
.first()
.exec(&mut db)
.await?
.is_some()
{
return Ok(true);
}
Ok(Review::filter_by_event_id(event_id)
.first()
.exec(&mut db)
.await?
.is_some())
}
pub async fn is_reviewed(&self, event_id: u64) -> toasty::Result<bool> { pub async fn is_reviewed(&self, event_id: u64) -> toasty::Result<bool> {
let mut db = self.db.clone(); let mut db = self.db.clone();
@@ -53,7 +38,8 @@ impl Dedup {
.is_some()) .is_some())
} }
pub async fn mark_reviewed(&self, record: ReviewRecord) -> toasty::Result<()> { /// Records a processed review, replacing the one of the same event if any.
pub async fn record(&self, record: ReviewRecord) -> toasty::Result<()> {
let mut db = self.db.clone(); let mut db = self.db.clone();
Review::upsert_by_event_id(record.event_id) Review::upsert_by_event_id(record.event_id)
@@ -71,7 +57,7 @@ impl Dedup {
Ok(()) Ok(())
} }
pub async fn list_reviews(&self) -> toasty::Result<Vec<Review>> { pub async fn list(&self) -> toasty::Result<Vec<Review>> {
let mut db = self.db.clone(); let mut db = self.db.clone();
let reviews = Review::all().exec(&mut db).await?; let reviews = Review::all().exec(&mut db).await?;
Review

[performance] list() charge toutes les reviews, y compris la colonne markdown, sans pagination ni tri, et /reviews les renvoie toutes d'un coup. La table ne fait que grossir et chaque ouverture ou rafraîchissement du dashboard relit tout l'historique. Il faut une limite (par exemple les N dernières, triées par completed_at DESC), ou omettre markdown dans la liste.

**[performance]** `list()` charge toutes les reviews, y compris la colonne `markdown`, sans pagination ni tri, et `/reviews` les renvoie toutes d'un coup. La table ne fait que grossir et chaque ouverture ou rafraîchissement du dashboard relit tout l'historique. Il faut une limite (par exemple les N dernières, triées par `completed_at DESC`), ou omettre `markdown` dans la liste.
@@ -82,24 +68,22 @@ impl Dedup {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::schemas::Review;
#[tokio::test] #[tokio::test]
#[ignore = "requires DATABASE_URL"] #[ignore = "requires DATABASE_URL"]
async fn a_recorded_review_is_known_and_reviewed() { async fn a_recorded_review_is_reviewed() {
let db = crate::db::test_connect().await; let db = crate::db::test_connect().await;
let dedup = Dedup::new(db.clone()); let store = ReviewStore::new(db.clone());
let event_id = 999_000_101; let event_id = 999_000_101;
Review::delete_by_event_id(&mut db.clone(), event_id) Review::delete_by_event_id(&mut db.clone(), event_id)
.await .await
.ok(); .ok();
assert!(!dedup.is_known(event_id).await.unwrap()); assert!(!store.is_reviewed(event_id).await.unwrap());
assert!(!dedup.is_reviewed(event_id).await.unwrap());
dedup store
.mark_reviewed(ReviewRecord { .record(ReviewRecord {
event_id, event_id,
repo: "owner/repo".to_string(), repo: "owner/repo".to_string(),
pr: 1, pr: 1,
@@ -113,8 +97,15 @@ mod tests {
.await .await
.unwrap(); .unwrap();
assert!(dedup.is_reviewed(event_id).await.unwrap()); assert!(store.is_reviewed(event_id).await.unwrap());
assert!(dedup.is_known(event_id).await.unwrap()); assert!(
store
.list()
.await
.unwrap()
.iter()
.any(|r| r.event_id == event_id)
);
Review::delete_by_event_id(&mut db.clone(), event_id) Review::delete_by_event_id(&mut db.clone(), event_id)
.await .await
+5 -5
View File
@@ -95,8 +95,8 @@ where
let mut last_rejection: Option<String> = None; let mut last_rejection: Option<String> = None;
for iteration in 1..=max_iterations { for iteration in 1..=max_iterations {
// Le dernier tour n'est plus un tour d'exploration : le modèle doit rendre // The last turn is no longer an exploration turn: the model must give its
// sa réponse, avec ce qu'il a déjà vu. // answer, with what it has already seen.
let last = iteration == max_iterations; let last = iteration == max_iterations;
if last { if last {
@@ -132,8 +132,8 @@ where
"Model answered" "Model answered"
); );
// Au dernier tour, les outils sont refusés : même si le fournisseur les // On the last turn, tools are refused: even if the provider returns them
// renvoie malgré tout, on ne les exécute pas et on tente la réponse. // anyway, we do not run them and try the answer instead.
if last || response.tool_calls.is_empty() { if last || response.tool_calls.is_empty() {
let answer = response.message.unwrap_or_default(); let answer = response.message.unwrap_or_default();
@@ -181,7 +181,7 @@ where
arguments = %excerpt(call.arguments_json(), MAX_LOGGED_ARGUMENTS), arguments = %excerpt(call.arguments_json(), MAX_LOGGED_ARGUMENTS),
output_bytes = content.len(), output_bytes = content.len(),
truncated, truncated,
// Les sauts de ligne casseraient la lisibilité d'une ligne de log. // Line breaks would hurt the readability of a log line.
output = %excerpt(&content, MAX_LOGGED_OUTPUT).replace('\n', " "), output = %excerpt(&content, MAX_LOGGED_OUTPUT).replace('\n', " "),
elapsed_ms = started.elapsed().as_millis() as u64, elapsed_ms = started.elapsed().as_millis() as u64,
"Tool call finished" "Tool call finished"
+18 -18
View File
@@ -1,5 +1,5 @@
//! Le [`Sandbox`] : clone d'une pull request exécuté dans un devcontainer //! The [`Sandbox`]: a pull request clone running inside an ephemeral
//! éphémère, et sa configuration. //! devcontainer, and its configuration.
use std::{ use std::{
path::{Path, PathBuf}, path::{Path, PathBuf},
@@ -19,7 +19,7 @@ const DEVCONTAINER_PATHS: [&str; 2] = [".devcontainer/devcontainer.json", ".devc
/// Sandbox-related runtime configuration. /// Sandbox-related runtime configuration.
#[derive(Clone)] #[derive(Clone)]
pub struct SandboxConfig { pub struct SandboxConfig {
/// Client du daemon de containers qui exécute la sandbox. /// Client of the container daemon that runs the sandbox.
pub runtime: ContainerRuntime, pub runtime: ContainerRuntime,
/// Maximum number of tool-calling iterations per agent run. /// Maximum number of tool-calling iterations per agent run.
pub max_iterations: usize, pub max_iterations: usize,
@@ -79,12 +79,12 @@ impl Sandbox {
Ok(sandbox) Ok(sandbox)
} }
/// Vérifie que le clone est bien visible dans le container. /// Checks that the clone is actually visible inside the container.
/// ///
/// Sans ce contrôle, un workspace vide — par exemple un daemon qui n'a pas pu /// Without this check, an empty workspace (for example a daemon that could not
/// recevoir le clone — fait échouer chaque outil ; le modèle enchaîne alors les /// receive the clone) makes every tool fail; the model then chains failed calls
/// appels ratés jusqu'au budget d'itérations, sans jamais pouvoir reviewer quoi /// until the iteration budget is spent, without ever being able to review
/// que ce soit. /// anything.
async fn check_workspace(&self) -> anyhow::Result<()> { async fn check_workspace(&self) -> anyhow::Result<()> {
let workspace_folder = self.workspace_folder(); let workspace_folder = self.workspace_folder();
let probe = self let probe = self
@@ -118,12 +118,12 @@ impl Sandbox {
Ok(()) Ok(())
} }
/// Rassemble de quoi expliquer un refus d'accès au workspace. /// Gathers enough information to explain a refused access to the workspace.
/// ///
/// Un `EACCES` a deux causes possibles, indistinguables dans le message de /// An `EACCES` has two possible causes, indistinguishable in the `ls` message:
/// `ls` : les permissions du clone, ou un confinement du noyau qui bloque /// the clone's permissions, or kernel confinement blocking the access. The
/// l'accès. L'identité de l'utilisateur d'exec et les permissions du point de /// identity of the exec user and the permissions of the mount point tell them
/// montage permettent de trancher. /// apart.
async fn diagnose_workspace(&self) -> String { async fn diagnose_workspace(&self) -> String {
let mut details = Vec::new(); let mut details = Vec::new();
@@ -215,12 +215,12 @@ async fn clone_pull_request(
Ok(()) Ok(())
} }
/// Rend le clone lisible par les utilisateurs du container sandbox. /// Makes the clone readable by the users of the sandbox container.
/// ///
/// Le container peut ne pas avoir les mêmes uid que Steep (podman rootless /// The container may not share Steep's uids (rootless podman maps uids through
/// mappe les uid à travers des plages subuid), et l'umask de l'opérateur peut être /// subuid ranges), and the operator's umask may be restrictive: without this, the
/// restrictif : sans cela, les outils de la sandbox échouent en `Permission /// sandbox tools fail with `Permission denied` on files that Steep itself just
/// denied` sur des fichiers que Steep vient de cloner lui-même. /// cloned.
async fn make_readable(repo_dir: &Path) -> anyhow::Result<()> { async fn make_readable(repo_dir: &Path) -> anyhow::Result<()> {
let output = tokio::process::Command::new("chmod") let output = tokio::process::Command::new("chmod")
.args(["-R", "a+rX"]) .args(["-R", "a+rX"])
+6 -6
View File
@@ -167,11 +167,11 @@ async fn read_file(sandbox: &Sandbox, args: &Value) -> anyhow::Result<String> {
Ok(number_lines(&into_stdout(output)?, first_line)) Ok(number_lines(&into_stdout(output)?, first_line))
} }
/// Préfixe chaque ligne par son numéro. /// Prefixes each line with its number.
/// ///
/// Le modèle doit citer une ligne précise pour ancrer son commentaire : sans /// The model must cite a precise line to anchor its comment: without numbers, it
/// numéros, il les compte lui-même et se décale de quelques lignes, ce qui place le /// counts them itself and drifts by a few lines, which puts the comment next to
/// commentaire à côté du code visé. /// the targeted code.
fn number_lines(content: &str, first_line: u64) -> String { fn number_lines(content: &str, first_line: u64) -> String {
content content
.lines() .lines()
@@ -291,8 +291,8 @@ mod tests {
#[test] #[test]
fn a_range_keeps_the_absolute_line_numbers() { fn a_range_keeps_the_absolute_line_numbers() {
// Un extrait lu à partir de la ligne 12 doit garder la numérotation du // An excerpt read from line 12 must keep the file's numbering: otherwise the
// fichier : sinon le modèle citerait des lignes décalées. // model would cite shifted lines.
assert_eq!(number_lines("x\ny", 12), "12:x\n13:y"); assert_eq!(number_lines("x\ny", 12), "12:x\n13:y");
} }
+6 -6
View File
@@ -13,22 +13,22 @@ pub struct Review {
pub repo: String, pub repo: String,
pub pr: u64, pub pr: u64,
/// Vrai si la review a abouti. /// True if the review succeeded.
pub ok: bool, pub ok: bool,
/// Modèle utilisé. /// Model used.
pub model: String, pub model: String,
/// Markdown posté sur la pull request (vide en cas d'échec). /// Markdown posted on the pull request (empty on failure).
pub markdown: String, pub markdown: String,
/// Coût OpenRouter en USD, si connu. /// OpenRouter cost in USD, if known.
pub cost: Option<f64>, pub cost: Option<f64>,
/// Nombre de commentaires produits. /// Number of comments produced.
pub comments: u64, pub comments: u64,
/// Durée du traitement en millisecondes. /// Processing duration in milliseconds.
pub duration_ms: u64, pub duration_ms: u64,
#[default(jiff::Timestamp::now())] #[default(jiff::Timestamp::now())]
+3 -3
View File
@@ -1,14 +1,14 @@
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use crate::{dedup::Dedup, env::EnvConfig, event_bus::EventBus, job_queue::JobQueue}; use crate::{env::EnvConfig, event_bus::EventBus, job_queue::JobQueue, review_store::ReviewStore};
#[derive(Clone)] #[derive(Clone)]
pub struct AppState { pub struct AppState {
pub queue: JobQueue, pub queue: JobQueue,
pub dedup: Dedup, pub reviews: ReviewStore,
pub event_bus: EventBus, pub event_bus: EventBus,
pub bot_name: String, pub bot_name: String,
pub config: EnvConfig, pub config: EnvConfig,
/// Jeton annulé au démarrage de l'arrêt gracieux : il sert à clore les flux SSE. /// Cancelled when the graceful shutdown starts: it closes the SSE streams.
pub shutdown: CancellationToken, pub shutdown: CancellationToken,
} }
+53
View File
@@ -0,0 +1,53 @@
# cargo-deny configuration: https://embarkstudios.github.io/cargo-deny/
# Run locally with `cargo deny check`.
[graph]
all-features = true
[advisories]
# A vulnerable, unmaintained or yanked dependency fails the build.
version = 2
yanked = "deny"
ignore = [
# Unmaintained, but only a thin wrapper around `rustls-pki-types`, and pulled by
# `toasty-driver-postgresql`: it can only go away with a toasty release.
{ id = "RUSTSEC-2025-0134", reason = "transitive dependency of toasty, no vulnerability" },
]
[licenses]
version = 2
# The workspace crates are not published, and have no license of their own.
private = { ignore = true }
allow = [
"Apache-2.0",
"Apache-2.0 WITH LLVM-exception",
"BSD-2-Clause",
"BSD-3-Clause",
"BSL-1.0",
"CC0-1.0",
"CDLA-Permissive-2.0",
"ISC",
"MIT",
"MIT-0",
"MPL-2.0",
"OpenSSL",
"Unicode-3.0",
"Zlib",
]
[bans]
multiple-versions = "warn"
wildcards = "deny"
# `path` dependencies between the (unpublished) workspace crates carry no version.
allow-wildcard-paths = true
# TLS goes through rustls: linking libssl would be a regression of the image
# (it would need `libssl` installed again).
deny = [
{ crate = "openssl-sys", reason = "TLS is rustls only, so that the image needs no libssl" },
{ crate = "native-tls", reason = "TLS is rustls only, so that the image needs no libssl" },
]
[sources]
unknown-registry = "deny"
unknown-git = "deny"
allow-registry = ["https://github.com/rust-lang/crates.io-index"]
+1 -1
View File
@@ -3,7 +3,7 @@
export const endpoints = { export const endpoints = {
jobs: "/jobs", jobs: "/jobs",
reviews: "/dedup", reviews: "/reviews",
lifecycle: "/events", lifecycle: "/events",
jobEvents: (id) => `/jobs/${id}/events`, jobEvents: (id) => `/jobs/${id}/events`,
}; };
+1 -1
View File
@@ -1,6 +1,6 @@
// <steep-dashboard> : assemble le tableau de bord. // <steep-dashboard> : assemble le tableau de bord.
// //
// Au montage, il charge l'état initial via REST (`/jobs`, `/dedup`), puis // Au montage, il charge l'état initial via REST (`/jobs`, `/reviews`), puis
// s'abonne au flux global `/events` pour tenir les listes à jour en direct. // s'abonne au flux global `/events` pour tenir les listes à jour en direct.
// La sélection d'un job ouvre son flux fin via <steep-job-detail>. // La sélection d'un job ouvre son flux fin via <steep-job-detail>.
2