1 parent
7493de6952
commit
3078ba5705
45 files changed
+2129
-1870
No files matched your search
@@ -11,7 +11,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||||||
curl \
|
curl \
|
||||||
git \
|
git \
|
||||||
build-essential \
|
build-essential \
|
||||||
libssl-dev \
|
|
||||||
cmake \
|
cmake \
|
||||||
pkg-config \
|
pkg-config \
|
||||||
clang \
|
clang \
|
||||||
|
|||||||
@@ -23,6 +23,12 @@ steps:
|
|||||||
- rustup component add clippy
|
- rustup component add clippy
|
||||||
- cargo clippy --workspace --all-targets --all-features -- -D warnings
|
- cargo clippy --workspace --all-targets --all-features -- -D warnings
|
||||||
|
|
||||||
|
- name: deny
|
||||||
|
image: rust:1.99
|
||||||
|
commands:
|
||||||
|
- cargo install cargo-deny --locked --version 0.18.9
|
||||||
|
- cargo deny check
|
||||||
|
|
||||||
- name: wait-for-postgres
|
- name: wait-for-postgres
|
||||||
image: postgres:18
|
image: postgres:18
|
||||||
commands:
|
commands:
|
||||||
|
|||||||
Generated
+15
-93
@@ -1392,21 +1392,6 @@ version = "0.2.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
|
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "foreign-types"
|
|
||||||
version = "0.3.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
|
|
||||||
dependencies = [
|
|
||||||
"foreign-types-shared",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "foreign-types-shared"
|
|
||||||
version = "0.1.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "form_urlencoded"
|
name = "form_urlencoded"
|
||||||
version = "1.2.2"
|
version = "1.2.2"
|
||||||
@@ -1811,18 +1796,17 @@ dependencies = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hyper-tls"
|
name = "hyper-rustls"
|
||||||
version = "0.6.0"
|
version = "0.27.10"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
|
checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes",
|
"http 1.5.0",
|
||||||
"http-body-util",
|
|
||||||
"hyper",
|
"hyper",
|
||||||
"hyper-util",
|
"hyper-util",
|
||||||
"native-tls",
|
"rustls",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-native-tls",
|
"tokio-rustls",
|
||||||
"tower-service",
|
"tower-service",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -2579,23 +2563,6 @@ dependencies = [
|
|||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "native-tls"
|
|
||||||
version = "0.2.18"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2"
|
|
||||||
dependencies = [
|
|
||||||
"libc",
|
|
||||||
"log",
|
|
||||||
"openssl",
|
|
||||||
"openssl-probe",
|
|
||||||
"openssl-sys",
|
|
||||||
"schannel",
|
|
||||||
"security-framework",
|
|
||||||
"security-framework-sys",
|
|
||||||
"tempfile",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "nix"
|
name = "nix"
|
||||||
version = "0.31.3"
|
version = "0.31.3"
|
||||||
@@ -2856,49 +2823,12 @@ version = "0.3.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
|
checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "openssl"
|
|
||||||
version = "0.10.81"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags 2.13.1",
|
|
||||||
"cfg-if",
|
|
||||||
"foreign-types",
|
|
||||||
"libc",
|
|
||||||
"openssl-macros",
|
|
||||||
"openssl-sys",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "openssl-macros"
|
|
||||||
version = "0.1.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"syn 2.0.119",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "openssl-probe"
|
name = "openssl-probe"
|
||||||
version = "0.2.1"
|
version = "0.2.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
|
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "openssl-sys"
|
|
||||||
version = "0.9.117"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695"
|
|
||||||
dependencies = [
|
|
||||||
"cc",
|
|
||||||
"libc",
|
|
||||||
"pkg-config",
|
|
||||||
"vcpkg",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "os_info"
|
name = "os_info"
|
||||||
version = "3.15.0"
|
version = "3.15.0"
|
||||||
@@ -3444,19 +3374,20 @@ dependencies = [
|
|||||||
"http-body",
|
"http-body",
|
||||||
"http-body-util",
|
"http-body-util",
|
||||||
"hyper",
|
"hyper",
|
||||||
"hyper-tls",
|
"hyper-rustls",
|
||||||
"hyper-util",
|
"hyper-util",
|
||||||
"js-sys",
|
"js-sys",
|
||||||
"log",
|
"log",
|
||||||
"native-tls",
|
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
|
"rustls",
|
||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
|
"rustls-platform-verifier",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sync_wrapper",
|
"sync_wrapper",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-native-tls",
|
"tokio-rustls",
|
||||||
"tokio-util",
|
"tokio-util",
|
||||||
"tower",
|
"tower",
|
||||||
"tower-http 0.6.11",
|
"tower-http 0.6.11",
|
||||||
@@ -3604,6 +3535,7 @@ version = "0.23.45"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
|
checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"log",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"ring",
|
"ring",
|
||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
@@ -3759,8 +3691,8 @@ checksum = "76a88b65feb368d9dde5d531a2b59b25016e36fd6e4978432371a3ee77746ca2"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"cfg_aliases",
|
"cfg_aliases",
|
||||||
"httpdate",
|
"httpdate",
|
||||||
"native-tls",
|
|
||||||
"reqwest",
|
"reqwest",
|
||||||
|
"rustls",
|
||||||
"sentry-actix",
|
"sentry-actix",
|
||||||
"sentry-backtrace",
|
"sentry-backtrace",
|
||||||
"sentry-contexts",
|
"sentry-contexts",
|
||||||
@@ -4281,6 +4213,7 @@ dependencies = [
|
|||||||
"metrics-exporter-prometheus",
|
"metrics-exporter-prometheus",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"ring",
|
"ring",
|
||||||
|
"rustls",
|
||||||
"sentry",
|
"sentry",
|
||||||
"sentry-anyhow",
|
"sentry-anyhow",
|
||||||
"serde",
|
"serde",
|
||||||
@@ -4721,16 +4654,6 @@ dependencies = [
|
|||||||
"syn 3.0.4",
|
"syn 3.0.4",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "tokio-native-tls"
|
|
||||||
version = "0.3.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
|
|
||||||
dependencies = [
|
|
||||||
"native-tls",
|
|
||||||
"tokio",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tokio-postgres"
|
name = "tokio-postgres"
|
||||||
version = "0.7.18"
|
version = "0.7.18"
|
||||||
@@ -5318,14 +5241,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d"
|
checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64 0.23.1",
|
"base64 0.23.1",
|
||||||
"der 0.8.2",
|
|
||||||
"log",
|
"log",
|
||||||
"native-tls",
|
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
|
"rustls",
|
||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
"ureq-proto",
|
"ureq-proto",
|
||||||
"utf8-zero",
|
"utf8-zero",
|
||||||
"webpki-root-certs",
|
"webpki-roots",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|||||||
+22
-2
@@ -6,14 +6,31 @@ members = [
|
|||||||
resolver = "3"
|
resolver = "3"
|
||||||
|
|
||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
reqwest = { version = "0.13", default-features = false, features = ["json", "native-tls", "stream"] }
|
# TLS is pure Rust (rustls + ring): no libssl is needed at build or run time.
|
||||||
|
# `rustls-no-provider` leaves the crypto provider to the binary, which installs
|
||||||
|
# `ring` at startup (see `main`), so that a single provider is ever compiled in.
|
||||||
|
reqwest = { version = "0.13", default-features = false, features = ["json", "rustls-no-provider", "stream"] }
|
||||||
|
rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] }
|
||||||
tokio = { version = "1.53", features = ["full"] }
|
tokio = { version = "1.53", features = ["full"] }
|
||||||
tokio-stream = "0.1"
|
tokio-stream = "0.1"
|
||||||
tokio-util = "0.7"
|
tokio-util = "0.7"
|
||||||
futures-util = "0.3"
|
futures-util = "0.3"
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
serde = { version = "1.0", features = ["derive"] }
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
sentry = { version = "0.49", features = ["tower-axum-matched-path"] }
|
# Sentry's default `transport` feature pulls `native-tls`; the same set of features
|
||||||
|
# is spelled out here with rustls instead.
|
||||||
|
sentry = { version = "0.49", default-features = false, features = [
|
||||||
|
"backtrace",
|
||||||
|
"contexts",
|
||||||
|
"debug-images",
|
||||||
|
"logs",
|
||||||
|
"metrics",
|
||||||
|
"panic",
|
||||||
|
"release-health",
|
||||||
|
"reqwest",
|
||||||
|
"rustls-no-provider",
|
||||||
|
"tower-axum-matched-path",
|
||||||
|
] }
|
||||||
sentry-anyhow = { version = "0.49", features = ["backtrace"] }
|
sentry-anyhow = { version = "0.49", features = ["backtrace"] }
|
||||||
dotenvy = "0.15"
|
dotenvy = "0.15"
|
||||||
tower = "0.5"
|
tower = "0.5"
|
||||||
@@ -31,6 +48,9 @@ metrics-exporter-prometheus = { version = "0.18", default-features = false, feat
|
|||||||
toasty = { version = "0.11", features = ["postgresql", "serde", "jiff"] }
|
toasty = { version = "0.11", features = ["postgresql", "serde", "jiff"] }
|
||||||
toasty-cli = "0.11"
|
toasty-cli = "0.11"
|
||||||
jiff = { version = "0.2", features = ["serde"] }
|
jiff = { version = "0.2", features = ["serde"] }
|
||||||
|
tempfile = "3"
|
||||||
|
bollard = "0.21"
|
||||||
|
tar = "0.4"
|
||||||
|
|
||||||
[profile.release]
|
[profile.release]
|
||||||
debug = 1
|
debug = 1
|
||||||
@@ -14,9 +14,6 @@ FROM debian:trixie-slim
|
|||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
git \
|
git \
|
||||||
libssl3t64 \
|
|
||||||
libzstd1 \
|
|
||||||
zlib1g \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ Steep reads its configuration from environment variables (a `.env` file is suppo
|
|||||||
| `METRICS_BIND_ADDR` | *(optional)* Bind address for the Prometheus metrics endpoint (e.g. `0.0.0.0:9100`). If unset, the metrics exporter is disabled. |
|
| `METRICS_BIND_ADDR` | *(optional)* Bind address for the Prometheus metrics endpoint (e.g. `0.0.0.0:9100`). If unset, the metrics exporter is disabled. |
|
||||||
| `SENTRY_DSN` | *(optional)* Sentry DSN for error tracking |
|
| `SENTRY_DSN` | *(optional)* Sentry DSN for error tracking |
|
||||||
| `RUST_LOG` | *(optional)* Log level, defaults to `info` |
|
| `RUST_LOG` | *(optional)* Log level, defaults to `info` |
|
||||||
| `SANDBOX_MAX_ITERATIONS` | *(optional)* Maximum number of tool-calling iterations per sandboxed review. Defaults to `8` |
|
| `SANDBOX_MAX_ITERATIONS` | *(optional)* Maximum number of tool-calling iterations per sandboxed review. Defaults to `8`; a value that is set but is not a number stops the startup |
|
||||||
| `DOCKER_HOST` | *(optional)* Container daemon socket Steep drives, e.g. `unix:///run/user/1000/podman/podman.sock` for a rootless podman. Defaults to `unix:///var/run/docker.sock` |
|
| `DOCKER_HOST` | *(optional)* Container daemon socket Steep drives, e.g. `unix:///run/user/1000/podman/podman.sock` for a rootless podman. Defaults to `unix:///var/run/docker.sock` |
|
||||||
|
|
||||||
## Sandboxed reviews
|
## Sandboxed reviews
|
||||||
@@ -139,6 +139,24 @@ cargo run -p steep-server
|
|||||||
|
|
||||||
Copy `.env.example` to `.env` and fill in your values before running.
|
Copy `.env.example` to `.env` and fill in your values before running.
|
||||||
|
|
||||||
|
## Job queue
|
||||||
|
|
||||||
|
Steep is meant to run as a **single instance**. Webhooks are stored in the `jobs` table
|
||||||
|
of PostgreSQL, and the worker is woken up as soon as a job is pushed (no polling).
|
||||||
|
A job still `processing` at startup was interrupted by the previous process: it is put
|
||||||
|
back in the queue and reviewed again. Do not run two instances on the same database,
|
||||||
|
they would both take over each other's running jobs.
|
||||||
|
|
||||||
|
## Dependencies
|
||||||
|
|
||||||
|
TLS is handled by [rustls](https://github.com/rustls/rustls) (with `ring`), so neither
|
||||||
|
the build nor the image needs OpenSSL. The `Containerfile` runtime image only
|
||||||
|
installs `ca-certificates` and `git`.
|
||||||
|
|
||||||
|
`cargo deny check` audits the dependencies (security advisories, licenses, banned
|
||||||
|
crates, sources) with the policy of `deny.toml`. It runs in the CI, and it also fails
|
||||||
|
if `openssl-sys` or `native-tls` comes back into the dependency tree.
|
||||||
|
|
||||||
## Metrics
|
## Metrics
|
||||||
|
|
||||||
Steep optionally exposes a Prometheus metrics endpoint, useful for scraping with an [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) or any Prometheus-compatible scraper.
|
Steep optionally exposes a Prometheus metrics endpoint, useful for scraping with an [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) or any Prometheus-compatible scraper.
|
||||||
|
|||||||
@@ -2,12 +2,13 @@
|
|||||||
name = "devcontainer-rs"
|
name = "devcontainer-rs"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
publish = false
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
bollard = "0.21"
|
bollard = { workspace = true }
|
||||||
bytes = { workspace = true }
|
bytes = { workspace = true }
|
||||||
futures-util = { workspace = true }
|
futures-util = { workspace = true }
|
||||||
tar = "0.4"
|
tar = { workspace = true }
|
||||||
tokio = { workspace = true }
|
tokio = { workspace = true }
|
||||||
tokio-stream = { workspace = true }
|
tokio-stream = { workspace = true }
|
||||||
tracing = { workspace = true }
|
tracing = { workspace = true }
|
||||||
@@ -17,4 +18,4 @@ anyhow = { workspace = true }
|
|||||||
thiserror = { workspace = true }
|
thiserror = { workspace = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
tempfile = "3"
|
tempfile = { workspace = true }
|
||||||
@@ -1,25 +1,24 @@
|
|||||||
//! Constantes de la crate : endpoint du daemon, timeouts et découpage du
|
//! Crate constants: daemon endpoint, timeouts and build context chunking.
|
||||||
//! contexte de build.
|
|
||||||
|
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
/// Endpoint affiché dans les logs quand `DOCKER_HOST` n'est pas défini.
|
/// Endpoint shown in the logs when `DOCKER_HOST` is not set.
|
||||||
pub(crate) const DEFAULT_ENDPOINT: &str = "unix:///var/run/docker.sock";
|
pub(crate) const DEFAULT_ENDPOINT: &str = "unix:///var/run/docker.sock";
|
||||||
|
|
||||||
/// Timeout appliqué aux opérations de build/run/stop/remove.
|
/// Timeout applied to build/run/stop/remove operations.
|
||||||
pub(crate) const DEFAULT_COMMAND_TIMEOUT: Duration = Duration::from_secs(600);
|
pub(crate) const DEFAULT_COMMAND_TIMEOUT: Duration = Duration::from_secs(600);
|
||||||
|
|
||||||
/// Timeout appliqué aux commandes exécutées dans un container en cours d'exécution.
|
/// Timeout applied to commands run inside a running container.
|
||||||
pub(crate) const DEFAULT_EXEC_TIMEOUT: Duration = Duration::from_secs(60);
|
pub(crate) const DEFAULT_EXEC_TIMEOUT: Duration = Duration::from_secs(60);
|
||||||
|
|
||||||
/// Taille des morceaux du contexte de build envoyés au daemon.
|
/// Size of the build context chunks sent to the daemon.
|
||||||
pub(crate) const CONTEXT_CHUNK_SIZE: usize = 64 * 1024;
|
pub(crate) const CONTEXT_CHUNK_SIZE: usize = 64 * 1024;
|
||||||
|
|
||||||
/// Nombre de morceaux pouvant attendre dans le canal : c'est ce qui borne la
|
/// Number of chunks that can wait in the channel: this is what bounds the
|
||||||
/// mémoire occupée par le contexte, quelle que soit sa taille.
|
/// memory held by the context, regardless of its size.
|
||||||
pub(crate) const CONTEXT_CHUNKS: usize = 4;
|
pub(crate) const CONTEXT_CHUNKS: usize = 4;
|
||||||
|
|
||||||
/// Lectures successives du code de sortie d'un exec, et attente entre elles.
|
/// Successive reads of an exec's exit code, and the delay between them.
|
||||||
pub(crate) const EXIT_CODE_ATTEMPTS: usize = 10;
|
pub(crate) const EXIT_CODE_ATTEMPTS: usize = 10;
|
||||||
pub(crate) const EXIT_CODE_DELAY: Duration = Duration::from_millis(20);
|
pub(crate) const EXIT_CODE_DELAY: Duration = Duration::from_millis(20);
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
//! Un devcontainer en cours d'exécution : inspection et exécution de commandes.
|
//! A running devcontainer: inspection and command execution.
|
||||||
|
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
@@ -7,7 +7,7 @@ use crate::{
|
|||||||
runtime::ContainerRuntime,
|
runtime::ContainerRuntime,
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Un devcontainer en cours d'exécution.
|
/// A running devcontainer.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct Container {
|
pub struct Container {
|
||||||
runtime: ContainerRuntime,
|
runtime: ContainerRuntime,
|
||||||
@@ -19,8 +19,8 @@ pub struct Container {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl Container {
|
impl Container {
|
||||||
/// Assemble un container démarré, avec son réseau et, le cas échéant, l'image
|
/// Assembles a started container, with its network and, if any, the image
|
||||||
/// construite pour lui et à nettoyer.
|
/// built for it that must be cleaned up.
|
||||||
pub(crate) fn new(
|
pub(crate) fn new(
|
||||||
runtime: ContainerRuntime,
|
runtime: ContainerRuntime,
|
||||||
name: String,
|
name: String,
|
||||||
@@ -47,10 +47,10 @@ impl Container {
|
|||||||
&self.workspace_folder
|
&self.workspace_folder
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Exécute une commande dans le container et renvoie sa sortie.
|
/// Runs a command in the container and returns its output.
|
||||||
///
|
///
|
||||||
/// La commande est transmise sous forme de vecteur d'arguments (argv, sans shell),
|
/// The command is passed as an argument vector (argv, no shell), so no
|
||||||
/// donc aucun échappement ni interpolation n'est effectué.
|
/// escaping or interpolation is performed.
|
||||||
pub async fn exec(&self, cmd: &[&str]) -> Result<ExecOutput, ContainerError> {
|
pub async fn exec(&self, cmd: &[&str]) -> Result<ExecOutput, ContainerError> {
|
||||||
self.exec_with_timeout(cmd, DEFAULT_EXEC_TIMEOUT).await
|
self.exec_with_timeout(cmd, DEFAULT_EXEC_TIMEOUT).await
|
||||||
}
|
}
|
||||||
@@ -65,19 +65,19 @@ impl Container {
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Exécute un script shell dans le container via `sh -c`.
|
/// Runs a shell script in the container via `sh -c`.
|
||||||
pub async fn exec_shell(&self, script: &str) -> Result<ExecOutput, ContainerError> {
|
pub async fn exec_shell(&self, script: &str) -> Result<ExecOutput, ContainerError> {
|
||||||
self.exec(&["sh", "-c", script]).await
|
self.exec(&["sh", "-c", script]).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Arrête le container.
|
/// Stops the container.
|
||||||
pub async fn stop(&self) -> Result<(), ContainerError> {
|
pub async fn stop(&self) -> Result<(), ContainerError> {
|
||||||
self.runtime.stop_container(&self.name).await
|
self.runtime.stop_container(&self.name).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Supprime le container, son réseau et son image.
|
/// Removes the container, its network and its image.
|
||||||
///
|
///
|
||||||
/// La suppression du réseau et de l'image est best-effort : ils peuvent déjà être absents.
|
/// Removing the network and the image is best-effort: they may already be gone.
|
||||||
pub async fn remove(&self) -> Result<(), ContainerError> {
|
pub async fn remove(&self) -> Result<(), ContainerError> {
|
||||||
self.runtime.remove_container(&self.name).await?;
|
self.runtime.remove_container(&self.name).await?;
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
//! Empaquetage en flux du contexte de build envoyé au daemon.
|
//! Streaming packaging of the build context sent to the daemon.
|
||||||
|
|
||||||
use std::{
|
use std::{
|
||||||
io::{BufWriter, Write},
|
io::{BufWriter, Write},
|
||||||
@@ -12,13 +12,13 @@ use tokio_stream::wrappers::ReceiverStream;
|
|||||||
|
|
||||||
use crate::consts::{CONTEXT_CHUNK_SIZE, CONTEXT_CHUNKS};
|
use crate::consts::{CONTEXT_CHUNK_SIZE, CONTEXT_CHUNKS};
|
||||||
|
|
||||||
/// Empaquette le contexte de build dans un tar, **en flux**.
|
/// Packs the build context into a tar, **as a stream**.
|
||||||
///
|
///
|
||||||
/// Le CLI `docker build` empaquette le contexte puis l'envoie ; le faire d'un coup
|
/// The `docker build` CLI packs the context and then sends it; doing that in one go
|
||||||
/// chargerait tout le dossier en mémoire, ce qui devient intenable dès que le
|
/// would load the whole folder into memory, which becomes untenable as soon as the
|
||||||
/// `.devcontainer` embarque des binaires. Ici l'archive est écrite par une tâche
|
/// `.devcontainer` ships binaries. Here the archive is written by a blocking task
|
||||||
/// bloquante et poussée morceau par morceau, avec la contre-pression du canal :
|
/// and pushed chunk by chunk, with the channel's backpressure: if the daemon reads
|
||||||
/// si le daemon lit lentement, l'empaquetage ralentit.
|
/// slowly, packing slows down.
|
||||||
pub(crate) fn tar_directory_stream(
|
pub(crate) fn tar_directory_stream(
|
||||||
dir: &Path,
|
dir: &Path,
|
||||||
) -> impl Stream<Item = Result<Bytes, std::io::Error>> + Send + 'static {
|
) -> impl Stream<Item = Result<Bytes, std::io::Error>> + Send + 'static {
|
||||||
@@ -33,13 +33,13 @@ pub(crate) fn tar_directory_stream(
|
|||||||
let result = builder
|
let result = builder
|
||||||
.append_dir_all(".", &dir)
|
.append_dir_all(".", &dir)
|
||||||
.and_then(|()| builder.finish())
|
.and_then(|()| builder.finish())
|
||||||
// `finish` écrit les blocs de fin d'archive ; il reste à vider le tampon
|
// `finish` writes the end-of-archive blocks; the buffer still has to be
|
||||||
// pour que le daemon reçoive tout, y compris une archive vide.
|
// flushed so the daemon receives everything, even an empty archive.
|
||||||
.and_then(|()| builder.get_mut().flush());
|
.and_then(|()| builder.get_mut().flush());
|
||||||
|
|
||||||
if let Err(error) = result {
|
if let Err(error) = result {
|
||||||
// L'échec est remonté au daemon par le flux : sinon il ne verrait qu'un
|
// The failure is reported to the daemon through the stream: otherwise it
|
||||||
// tar tronqué et signalerait une erreur incompréhensible.
|
// would only see a truncated tar and report a confusing error.
|
||||||
let _ = failures.blocking_send(Err(error));
|
let _ = failures.blocking_send(Err(error));
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -47,9 +47,9 @@ pub(crate) fn tar_directory_stream(
|
|||||||
ReceiverStream::new(receiver)
|
ReceiverStream::new(receiver)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Écrit les morceaux du tar dans un canal, en attendant qu'il se vide.
|
/// Writes the tar chunks into a channel, waiting for it to drain.
|
||||||
///
|
///
|
||||||
/// Utilisé depuis une tâche bloquante, seul contexte où `blocking_send` est autorisé.
|
/// Used from a blocking task, the only context where `blocking_send` is allowed.
|
||||||
struct ChannelWriter {
|
struct ChannelWriter {
|
||||||
sender: mpsc::Sender<Result<Bytes, std::io::Error>>,
|
sender: mpsc::Sender<Result<Bytes, std::io::Error>>,
|
||||||
}
|
}
|
||||||
@@ -80,7 +80,7 @@ mod tests {
|
|||||||
|
|
||||||
use futures_util::StreamExt;
|
use futures_util::StreamExt;
|
||||||
|
|
||||||
/// Rassemble le flux du contexte en un tar complet, pour l'inspecter.
|
/// Collects the context stream into a complete tar, for inspection.
|
||||||
async fn context_tar(dir: &Path) -> Vec<u8> {
|
async fn context_tar(dir: &Path) -> Vec<u8> {
|
||||||
let mut tar = Vec::new();
|
let mut tar = Vec::new();
|
||||||
let mut chunks = tar_directory_stream(dir);
|
let mut chunks = tar_directory_stream(dir);
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
//! Représentation analysée d'un `devcontainer.json` et son cycle de vie :
|
//! Parsed representation of a `devcontainer.json` and its lifecycle: naming,
|
||||||
//! nommage, build, démarrage et hooks.
|
//! build, startup and hooks.
|
||||||
|
|
||||||
use std::{
|
use std::{
|
||||||
collections::HashMap,
|
collections::HashMap,
|
||||||
@@ -29,29 +29,29 @@ pub struct DevContainer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl DevContainer {
|
impl DevContainer {
|
||||||
/// Dossier de workspace dans le container, par défaut `/workspace`.
|
/// Workspace folder inside the container, `/workspace` by default.
|
||||||
pub fn workspace_folder(&self) -> String {
|
pub fn workspace_folder(&self) -> String {
|
||||||
self.workspace_folder
|
self.workspace_folder
|
||||||
.clone()
|
.clone()
|
||||||
.unwrap_or_else(|| "/workspace".to_string())
|
.unwrap_or_else(|| "/workspace".to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Tag de l'image construite localement à partir du Dockerfile.
|
/// Tag of the image built locally from the Dockerfile.
|
||||||
fn local_image_tag(&self, id: &str) -> String {
|
fn local_image_tag(&self, id: &str) -> String {
|
||||||
format!("devcontainer-rs/{}:{}", self.sanitized_name(), id)
|
format!("devcontainer-rs/{}:{}", self.sanitized_name(), id)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Nom du container de cette sandbox.
|
/// Name of this sandbox's container.
|
||||||
fn container_name(&self, id: &str) -> String {
|
fn container_name(&self, id: &str) -> String {
|
||||||
format!("devcontainer-rs-{}-{}", self.sanitized_name(), id)
|
format!("devcontainer-rs-{}-{}", self.sanitized_name(), id)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Nom du devcontainer nettoyé pour servir de nom d'image/container.
|
/// Devcontainer name, sanitized to be usable as an image/container name.
|
||||||
fn sanitized_name(&self) -> String {
|
fn sanitized_name(&self) -> String {
|
||||||
sanitize(self.name.as_deref().unwrap_or("devcontainer"))
|
sanitize(self.name.as_deref().unwrap_or("devcontainer"))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Construit l'image devcontainer sous `image_tag`.
|
/// Builds the devcontainer image as `image_tag`.
|
||||||
async fn build(
|
async fn build(
|
||||||
&self,
|
&self,
|
||||||
runtime: &ContainerRuntime,
|
runtime: &ContainerRuntime,
|
||||||
@@ -239,7 +239,7 @@ impl Default for DevContainer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Nettoie une chaîne pour qu'elle puisse servir de nom d'image/container docker.
|
/// Sanitizes a string so it can be used as a docker image/container name.
|
||||||
fn sanitize(input: &str) -> String {
|
fn sanitize(input: &str) -> String {
|
||||||
let sanitized: String = input
|
let sanitized: String = input
|
||||||
.chars()
|
.chars()
|
||||||
@@ -260,7 +260,7 @@ fn sanitize(input: &str) -> String {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Identifiant unique à une exécution de sandbox, combinant l'identifiant de processus et un horodatage.
|
/// Identifier unique to a sandbox run, combining the process id and a timestamp.
|
||||||
fn unique_id() -> String {
|
fn unique_id() -> String {
|
||||||
let nanos = SystemTime::now()
|
let nanos = SystemTime::now()
|
||||||
.duration_since(UNIX_EPOCH)
|
.duration_since(UNIX_EPOCH)
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
//! Erreurs de la crate : échecs du daemon de containers et échecs d'analyse du
|
//! Crate errors: container daemon failures and `devcontainer.json` parsing failures.
|
||||||
//! `devcontainer.json`.
|
|
||||||
|
|
||||||
use std::{path::PathBuf, time::Duration};
|
use std::{path::PathBuf, time::Duration};
|
||||||
|
|
||||||
@@ -7,25 +6,25 @@ use bollard::errors::Error as BollardError;
|
|||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
pub enum ContainerError {
|
pub enum ContainerError {
|
||||||
/// Le daemon est injoignable, ou a refusé une requête.
|
/// The daemon is unreachable, or rejected a request.
|
||||||
#[error("container daemon request failed: {source}")]
|
#[error("container daemon request failed: {source}")]
|
||||||
Request {
|
Request {
|
||||||
#[from]
|
#[from]
|
||||||
source: BollardError,
|
source: BollardError,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// Une opération a dépassé son timeout.
|
/// An operation exceeded its timeout.
|
||||||
#[error("`{operation}` timed out after {timeout:?}")]
|
#[error("`{operation}` timed out after {timeout:?}")]
|
||||||
Timeout {
|
Timeout {
|
||||||
operation: String,
|
operation: String,
|
||||||
timeout: Duration,
|
timeout: Duration,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// La construction de l'image a échoué.
|
/// The image build failed.
|
||||||
#[error("image build failed: {message}")]
|
#[error("image build failed: {message}")]
|
||||||
Build { message: String },
|
Build { message: String },
|
||||||
|
|
||||||
/// Le daemon a répondu autre chose que ce qui était attendu.
|
/// The daemon replied with something other than what was expected.
|
||||||
#[error("{0}")]
|
#[error("{0}")]
|
||||||
Unexpected(String),
|
Unexpected(String),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
//! Résultat d'une commande exécutée dans un container.
|
//! Result of a command run in a container.
|
||||||
|
|
||||||
/// Résultat d'une commande exécutée dans un container.
|
/// Result of a command run in a container.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct ExecOutput {
|
pub struct ExecOutput {
|
||||||
/// Code de sortie, ou `-1` si le processus a été terminé par un signal.
|
/// Exit code, or `-1` if the process was terminated by a signal.
|
||||||
pub status: i32,
|
pub status: i32,
|
||||||
pub stdout: String,
|
pub stdout: String,
|
||||||
pub stderr: String,
|
pub stderr: String,
|
||||||
|
|||||||
@@ -1,28 +1,27 @@
|
|||||||
//! Primitives de cycle de vie de container pour un [`DevContainer`] analysé.
|
//! Container lifecycle primitives for a parsed [`DevContainer`].
|
||||||
//!
|
//!
|
||||||
//! Cette crate pilote l'API du daemon de containers pour préparer l'image
|
//! This crate drives the container daemon API to prepare the devcontainer image
|
||||||
//! devcontainer (construite depuis un Dockerfile, ou tirée d'un registre),
|
//! (built from a Dockerfile, or pulled from a registry), start a container and
|
||||||
//! démarrer un container et y copier le workspace, exécuter les
|
//! copy the workspace into it, run the `postCreateCommand` / `postStartCommand`
|
||||||
//! hooks `postCreateCommand` / `postStartCommand` et lancer des commandes à
|
//! hooks, and run commands inside the running container.
|
||||||
//! l'intérieur du container en cours d'exécution.
|
|
||||||
//!
|
//!
|
||||||
//! Le daemon est joint par son **socket** (celui de l'hôte, monté dans le
|
//! The daemon is reached through its **socket** (the host's, mounted into the
|
||||||
//! container de Steep) : [`ContainerRuntime::connect`] suit `DOCKER_HOST` comme
|
//! Steep container): [`ContainerRuntime::connect`] follows `DOCKER_HOST` like the
|
||||||
//! le fait le CLI `docker`, et retombe sur le socket local. Aucun binaire de
|
//! `docker` CLI does, and falls back to the local socket. No runtime binary is
|
||||||
//! runtime n'est donc requis dans l'image, et `podman` fonctionne de la même
|
//! therefore required in the image, and `podman` works the same way as long as it
|
||||||
//! façon dès lors qu'il expose son socket compatible Docker.
|
//! exposes its Docker-compatible socket.
|
||||||
//!
|
//!
|
||||||
//! # Isolation
|
//! # Isolation
|
||||||
//!
|
//!
|
||||||
//! Chaque sandbox dispose de son propre tag d'image, de son propre container et de
|
//! Each sandbox has its own image tag, its own container and its own network. The
|
||||||
//! son propre réseau. Le container démarre attaché à ce réseau afin que les hooks
|
//! container starts attached to that network so that the `postCreateCommand` /
|
||||||
//! `postCreateCommand` / `postStartCommand` puissent récupérer des dépendances
|
//! `postStartCommand` hooks can fetch dependencies (e.g. `npm install`); once the
|
||||||
//! (par ex. `npm install`) ; une fois les hooks exécutés, le container est déconnecté
|
//! hooks have run, the container is disconnected from the network for the rest of
|
||||||
//! du réseau pour le reste de sa durée de vie. Chaque commande est bornée par un timeout.
|
//! its lifetime. Every command is bounded by a timeout.
|
||||||
//!
|
//!
|
||||||
//! Les `runArgs` du `devcontainer.json` sont ignorés : ils viennent du dépôt, donc
|
//! The `devcontainer.json` `runArgs` are ignored: they come from the repository,
|
||||||
//! d'une pull request non fiable, et pourraient rattacher le container à un autre
|
//! hence from an untrusted pull request, and could attach the container to another
|
||||||
//! réseau ou lui donner des privilèges qui annuleraient cette isolation.
|
//! network or grant it privileges that would defeat this isolation.
|
||||||
|
|
||||||
mod consts;
|
mod consts;
|
||||||
mod container;
|
mod container;
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
//! Normalisation lexicale de chemins, sans accès au système de fichiers.
|
//! Lexical path normalization, without accessing the filesystem.
|
||||||
|
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
/// Normalise lexicalement un chemin, en résolvant `.` et `..` sans toucher au
|
/// Lexically normalizes a path, resolving `.` and `..` without touching the
|
||||||
/// système de fichiers. Renvoie `None` si le chemin sort de sa racine.
|
/// filesystem. Returns `None` if the path escapes its root.
|
||||||
pub fn normalize(path: &Path) -> Option<PathBuf> {
|
pub fn normalize(path: &Path) -> Option<PathBuf> {
|
||||||
use std::path::Component;
|
use std::path::Component;
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
//! Client de l'API du daemon de containers et opérations de cycle de vie.
|
//! Container daemon API client and lifecycle operations.
|
||||||
|
|
||||||
use std::{collections::HashMap, path::Path, time::Duration};
|
use std::{collections::HashMap, path::Path, time::Duration};
|
||||||
|
|
||||||
@@ -24,7 +24,7 @@ use crate::{
|
|||||||
exec::ExecOutput,
|
exec::ExecOutput,
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Un client de l'API du daemon de containers.
|
/// A container daemon API client.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct ContainerRuntime {
|
pub struct ContainerRuntime {
|
||||||
docker: Docker,
|
docker: Docker,
|
||||||
@@ -33,7 +33,7 @@ pub struct ContainerRuntime {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ContainerRuntime {
|
impl ContainerRuntime {
|
||||||
/// Se connecte au daemon désigné par `DOCKER_HOST`, ou au socket local par défaut.
|
/// Connects to the daemon designated by `DOCKER_HOST`, or to the default local socket.
|
||||||
pub fn connect() -> Result<Self, ContainerError> {
|
pub fn connect() -> Result<Self, ContainerError> {
|
||||||
let endpoint =
|
let endpoint =
|
||||||
std::env::var("DOCKER_HOST").unwrap_or_else(|_| String::from(DEFAULT_ENDPOINT));
|
std::env::var("DOCKER_HOST").unwrap_or_else(|_| String::from(DEFAULT_ENDPOINT));
|
||||||
@@ -45,13 +45,13 @@ impl ContainerRuntime {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Remplace le timeout appliqué aux opérations de build/run/stop/remove.
|
/// Replaces the timeout applied to build/run/stop/remove operations.
|
||||||
pub fn with_timeout(mut self, timeout: Duration) -> Self {
|
pub fn with_timeout(mut self, timeout: Duration) -> Self {
|
||||||
self.timeout = timeout;
|
self.timeout = timeout;
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Endpoint du daemon, pour les logs.
|
/// Daemon endpoint, for logging.
|
||||||
pub fn endpoint(&self) -> &str {
|
pub fn endpoint(&self) -> &str {
|
||||||
&self.endpoint
|
&self.endpoint
|
||||||
}
|
}
|
||||||
@@ -60,12 +60,12 @@ impl ContainerRuntime {
|
|||||||
self.timeout
|
self.timeout
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Vérifie que le daemon est joignable et répond.
|
/// Checks that the daemon is reachable and responding.
|
||||||
pub async fn available(&self) -> bool {
|
pub async fn available(&self) -> bool {
|
||||||
self.docker.ping().await.is_ok()
|
self.docker.ping().await.is_ok()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Exécute une requête du daemon en appliquant le timeout des commandes.
|
/// Runs a daemon request, applying the command timeout.
|
||||||
async fn request<T>(
|
async fn request<T>(
|
||||||
&self,
|
&self,
|
||||||
operation: &str,
|
operation: &str,
|
||||||
@@ -81,7 +81,7 @@ impl ContainerRuntime {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Construit l'image `tag` depuis le contexte `context_dir`.
|
/// Builds the image `tag` from the `context_dir` context.
|
||||||
pub(crate) async fn build_image(
|
pub(crate) async fn build_image(
|
||||||
&self,
|
&self,
|
||||||
context_dir: &Path,
|
context_dir: &Path,
|
||||||
@@ -102,8 +102,8 @@ impl ContainerRuntime {
|
|||||||
.docker
|
.docker
|
||||||
.build_image(options, None, Some(body_try_stream(context)));
|
.build_image(options, None, Some(body_try_stream(context)));
|
||||||
|
|
||||||
// Le flux porte la progression du build : la dernière erreur signalée par le
|
// The stream carries the build progress: the last error reported by the
|
||||||
// daemon fait échouer l'opération.
|
// daemon makes the operation fail.
|
||||||
let consume = async {
|
let consume = async {
|
||||||
let mut failure = None;
|
let mut failure = None;
|
||||||
|
|
||||||
@@ -162,7 +162,7 @@ impl ContainerRuntime {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Crée un container, sans le démarrer.
|
/// Creates a container without starting it.
|
||||||
pub(crate) async fn create_container(
|
pub(crate) async fn create_container(
|
||||||
&self,
|
&self,
|
||||||
name: &str,
|
name: &str,
|
||||||
@@ -217,7 +217,7 @@ impl ContainerRuntime {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Exécute une commande dans un container et renvoie sa sortie.
|
/// Runs a command in a container and returns its output.
|
||||||
pub(crate) async fn exec(
|
pub(crate) async fn exec(
|
||||||
&self,
|
&self,
|
||||||
container: &str,
|
container: &str,
|
||||||
@@ -254,8 +254,8 @@ impl ContainerRuntime {
|
|||||||
let mut stdout = String::new();
|
let mut stdout = String::new();
|
||||||
let mut stderr = String::new();
|
let mut stderr = String::new();
|
||||||
|
|
||||||
// Le timeout couvre l'exécution de la commande elle-même, pas seulement sa
|
// The timeout covers the execution of the command itself, not just its
|
||||||
// mise en place.
|
// setup.
|
||||||
let collect = async {
|
let collect = async {
|
||||||
while let Some(message) = output.next().await {
|
while let Some(message) = output.next().await {
|
||||||
match message? {
|
match message? {
|
||||||
@@ -276,10 +276,10 @@ impl ContainerRuntime {
|
|||||||
Ok(Ok(())) => {}
|
Ok(Ok(())) => {}
|
||||||
Ok(Err(source)) => return Err(ContainerError::Request { source }),
|
Ok(Err(source)) => return Err(ContainerError::Request { source }),
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
// Le processus continue de tourner dans le container : sans arrêt, il
|
// The process keeps running in the container: left alone, it would
|
||||||
// consommerait CPU et mémoire et pourrait encore modifier le workspace
|
// consume CPU and memory and could still modify the workspace for the
|
||||||
// pendant toute la durée de vie de la sandbox. L'API n'offre pas de
|
// whole lifetime of the sandbox. The API offers no way to kill an exec,
|
||||||
// moyen de tuer un exec, on arrête donc le container qui le porte.
|
// so we stop the container that hosts it.
|
||||||
let _ = self.stop_container(container).await;
|
let _ = self.stop_container(container).await;
|
||||||
|
|
||||||
return Err(ContainerError::Timeout {
|
return Err(ContainerError::Timeout {
|
||||||
@@ -289,10 +289,10 @@ impl ContainerRuntime {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Le code de sortie n'est pas forcément publié au moment où le flux se ferme :
|
// The exit code is not necessarily published by the time the stream closes:
|
||||||
// on laisse au daemon le temps de le renseigner, sans bloquer indéfiniment.
|
// we give the daemon time to fill it in, without blocking indefinitely.
|
||||||
// Sans cela, une commande réussie peut être rapportée en échec (code -1) et
|
// Without this, a successful command can be reported as failed (code -1) and
|
||||||
// l'outil renvoie une erreur au modèle à la place du contenu.
|
// the tool returns an error to the model instead of the content.
|
||||||
let mut inspected = self
|
let mut inspected = self
|
||||||
.request("inspect exec", self.docker.inspect_exec(&created.id))
|
.request("inspect exec", self.docker.inspect_exec(&created.id))
|
||||||
.await?;
|
.await?;
|
||||||
@@ -331,7 +331,7 @@ impl ContainerRuntime {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Supprime un container, ses volumes anonymes et le processus qui y tourne.
|
/// Removes a container, its anonymous volumes and the process running in it.
|
||||||
pub(crate) async fn remove_container(&self, name: &str) -> Result<(), ContainerError> {
|
pub(crate) async fn remove_container(&self, name: &str) -> Result<(), ContainerError> {
|
||||||
let options = RemoveContainerOptions {
|
let options = RemoveContainerOptions {
|
||||||
force: true,
|
force: true,
|
||||||
@@ -363,7 +363,7 @@ impl ContainerRuntime {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Crée le réseau dédié d'une sandbox.
|
/// Creates a sandbox's dedicated network.
|
||||||
pub(crate) async fn create_network(&self, name: &str) -> Result<(), ContainerError> {
|
pub(crate) async fn create_network(&self, name: &str) -> Result<(), ContainerError> {
|
||||||
let request = NetworkCreateRequest {
|
let request = NetworkCreateRequest {
|
||||||
name: String::from(name),
|
name: String::from(name),
|
||||||
@@ -376,7 +376,7 @@ impl ContainerRuntime {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Détache un container d'un réseau, ce qui coupe sa connectivité.
|
/// Detaches a container from a network, which cuts off its connectivity.
|
||||||
pub(crate) async fn disconnect_network(
|
pub(crate) async fn disconnect_network(
|
||||||
&self,
|
&self,
|
||||||
network: &str,
|
network: &str,
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
//! Schéma du `devcontainer.json`, analyse du fichier et résolution des
|
//! `devcontainer.json` schema, file parsing and resolution of `${localEnv:…}`
|
||||||
//! références `${localEnv:…}`.
|
//! references.
|
||||||
|
|
||||||
use std::{
|
use std::{
|
||||||
collections::HashMap,
|
collections::HashMap,
|
||||||
@@ -23,7 +23,7 @@ struct DevContainerSchema {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
name: Option<String>,
|
name: Option<String>,
|
||||||
|
|
||||||
/// Image de base tirée du registre, alternative à `build`.
|
/// Base image pulled from the registry, alternative to `build`.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
image: Option<String>,
|
image: Option<String>,
|
||||||
|
|
||||||
@@ -106,16 +106,16 @@ impl TryFrom<(DevContainerSchema, PathBuf)> for DevContainer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Résout les références `${localEnv:VAR}` et `${localEnv:VAR:default}` d'un
|
/// Resolves the `${localEnv:VAR}` and `${localEnv:VAR:default}` references of a
|
||||||
/// `devcontainer.json`.
|
/// `devcontainer.json`.
|
||||||
///
|
///
|
||||||
/// La spécification veut que `${localEnv:VAR}` soit lu dans l'environnement du
|
/// The specification says `${localEnv:VAR}` is read from the client's environment;
|
||||||
/// client ; ici le `devcontainer.json` vient d'une **pull request**, donc de code
|
/// here the `devcontainer.json` comes from a **pull request**, hence from untrusted
|
||||||
/// non fiable : lire l'environnement du processus lui permettrait de récupérer
|
/// code: reading the process environment would let it grab `GITEA_TOKEN`,
|
||||||
/// `GITEA_TOKEN`, `OPEN_ROUTER_API_KEY` ou n'importe quel autre secret de Steep et
|
/// `OPEN_ROUTER_API_KEY` or any other Steep secret and exfiltrate it through a
|
||||||
/// de l'exfiltrer depuis un `build.args`, un `containerEnv`, son Dockerfile ou un
|
/// `build.args`, a `containerEnv`, its Dockerfile or a hook. The environment is
|
||||||
/// hook. L'environnement n'est donc **jamais** consulté : seule la valeur par
|
/// therefore **never** consulted: only the default value is used, and a reference
|
||||||
/// défaut est utilisée, et une référence sans défaut devient une chaîne vide.
|
/// without a default becomes an empty string.
|
||||||
fn substitute_local_env(input: &str) -> String {
|
fn substitute_local_env(input: &str) -> String {
|
||||||
const PREFIX: &str = "${localEnv:";
|
const PREFIX: &str = "${localEnv:";
|
||||||
|
|
||||||
@@ -129,7 +129,7 @@ fn substitute_local_env(input: &str) -> String {
|
|||||||
match after.find('}') {
|
match after.find('}') {
|
||||||
Some(end) => {
|
Some(end) => {
|
||||||
let inner = &after[..end];
|
let inner = &after[..end];
|
||||||
// La clé est lue pour délimiter la référence, pas pour la résoudre.
|
// The key is read to delimit the reference, not to resolve it.
|
||||||
let (_key, default) = match inner.split_once(':') {
|
let (_key, default) = match inner.split_once(':') {
|
||||||
Some((key, default)) => (key, Some(default)),
|
Some((key, default)) => (key, Some(default)),
|
||||||
None => (inner, None),
|
None => (inner, None),
|
||||||
@@ -239,8 +239,8 @@ mod tests {
|
|||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
let devcontainer_path = dir.path().join("devcontainer.json");
|
let devcontainer_path = dir.path().join("devcontainer.json");
|
||||||
|
|
||||||
// `runArgs` vient d'une pull request non fiable : il doit être accepté par
|
// `runArgs` comes from an untrusted pull request: it must be accepted by
|
||||||
// l'analyse mais jamais repris dans le devcontainer.
|
// parsing but never carried over into the devcontainer.
|
||||||
fs::write(
|
fs::write(
|
||||||
&devcontainer_path,
|
&devcontainer_path,
|
||||||
r#"{
|
r#"{
|
||||||
@@ -271,8 +271,8 @@ mod tests {
|
|||||||
fn local_env_references_are_never_read_from_the_environment() {
|
fn local_env_references_are_never_read_from_the_environment() {
|
||||||
unsafe { std::env::set_var("DEVCONTAINER_TEST_SECRET", "s3cret") };
|
unsafe { std::env::set_var("DEVCONTAINER_TEST_SECRET", "s3cret") };
|
||||||
|
|
||||||
// Le `devcontainer.json` vient d'une pull request : résoudre la référence
|
// The `devcontainer.json` comes from a pull request: resolving the reference
|
||||||
// depuis l'environnement de Steep y exposerait ses jetons.
|
// from Steep's environment would expose its tokens to it.
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
substitute_local_env("${localEnv:DEVCONTAINER_TEST_SECRET}"),
|
substitute_local_env("${localEnv:DEVCONTAINER_TEST_SECRET}"),
|
||||||
""
|
""
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
name = "steep-server"
|
name = "steep-server"
|
||||||
version = "1.3.0"
|
version = "1.3.0"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
publish = false
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
reqwest = { workspace = true }
|
reqwest = { workspace = true }
|
||||||
@@ -26,7 +27,8 @@ hex = { workspace = true }
|
|||||||
metrics = { workspace = true }
|
metrics = { workspace = true }
|
||||||
metrics-exporter-prometheus = { workspace = true }
|
metrics-exporter-prometheus = { workspace = true }
|
||||||
devcontainer-rs = { path = "../devcontainer-rs" }
|
devcontainer-rs = { path = "../devcontainer-rs" }
|
||||||
tempfile = "3"
|
tempfile = { workspace = true }
|
||||||
|
rustls = { workspace = true }
|
||||||
toasty = { workspace = true }
|
toasty = { workspace = true }
|
||||||
toasty-cli = { workspace = true }
|
toasty-cli = { workspace = true }
|
||||||
jiff = { workspace = true }
|
jiff = { workspace = true }
|
||||||
@@ -28,8 +28,8 @@ pub async fn start(app_state: AppState) -> anyhow::Result<()> {
|
|||||||
let http_port = app_state.config.http_port;
|
let http_port = app_state.config.http_port;
|
||||||
let shutdown = app_state.shutdown.clone();
|
let shutdown = app_state.shutdown.clone();
|
||||||
|
|
||||||
// Le dashboard est servi depuis un dossier statique ; les routes API
|
// The dashboard is served from a static directory; the API routes below take
|
||||||
// ci-dessous restent prioritaires (le dossier n'est que le fallback).
|
// precedence (the directory is only the fallback).
|
||||||
let front_dir = std::env::var("FRONT_DIR").unwrap_or_else(|_| String::from("front"));
|
let front_dir = std::env::var("FRONT_DIR").unwrap_or_else(|_| String::from("front"));
|
||||||
if !std::path::Path::new(&front_dir).is_dir() {
|
if !std::path::Path::new(&front_dir).is_dir() {
|
||||||
warn!(dir = %front_dir, "front directory not found, the dashboard will 404");
|
warn!(dir = %front_dir, "front directory not found, the dashboard will 404");
|
||||||
@@ -38,7 +38,7 @@ pub async fn start(app_state: AppState) -> anyhow::Result<()> {
|
|||||||
|
|
||||||
let app = Router::new()
|
let app = Router::new()
|
||||||
.route("/webhook", post(webhook))
|
.route("/webhook", post(webhook))
|
||||||
.route("/dedup", get(dedup_list))
|
.route("/reviews", get(reviews_list))
|
||||||
.route("/jobs", get(jobs_list))
|
.route("/jobs", get(jobs_list))
|
||||||
.route("/events", get(lifecycle_events))
|
.route("/events", get(lifecycle_events))
|
||||||
.route("/jobs/{id}/events", get(job_events))
|
.route("/jobs/{id}/events", get(job_events))
|
||||||
@@ -63,8 +63,8 @@ pub async fn start(app_state: AppState) -> anyhow::Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn dedup_list(State(app_state): State<AppState>) -> Result<impl IntoResponse, AppError> {
|
async fn reviews_list(State(app_state): State<AppState>) -> Result<impl IntoResponse, AppError> {
|
||||||
let reviews = app_state.dedup.list_reviews().await?;
|
let reviews = app_state.reviews.list().await?;
|
||||||
|
|
||||||
Ok(Json(json!({
|
Ok(Json(json!({
|
||||||
"reviews": reviews
|
"reviews": reviews
|
||||||
@@ -81,10 +81,10 @@ async fn jobs_list(State(app_state): State<AppState>) -> Result<impl IntoRespons
|
|||||||
})))
|
})))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Flux SSE des évènements de cycle de vie global (liste des jobs).
|
/// SSE stream of the global lifecycle events (the job list).
|
||||||
///
|
///
|
||||||
/// Le flux est clos dès que l'arrêt gracieux est lancé, sans quoi une connexion
|
/// The stream is closed as soon as the graceful shutdown starts: a live SSE
|
||||||
/// SSE vivante empêcherait le serveur de s'arrêter.
|
/// connection would otherwise keep the server from stopping.
|
||||||
async fn lifecycle_events(
|
async fn lifecycle_events(
|
||||||
State(app_state): State<AppState>,
|
State(app_state): State<AppState>,
|
||||||
) -> Sse<impl Stream<Item = Result<Event, axum::Error>>> {
|
) -> Sse<impl Stream<Item = Result<Event, axum::Error>>> {
|
||||||
@@ -143,7 +143,7 @@ async fn webhook(
|
|||||||
metrics::webhook_received(event_type);
|
metrics::webhook_received(event_type);
|
||||||
|
|
||||||
let event_id = wb.event_id();
|
let event_id = wb.event_id();
|
||||||
match app_state.dedup.is_known(event_id).await {
|
match is_known(&app_state, event_id).await {
|
||||||
Ok(true) => {
|
Ok(true) => {
|
||||||
metrics::webhook_duplicate(event_type);
|
metrics::webhook_duplicate(event_type);
|
||||||
return Err(AppError::AlreadyProcessedErr);
|
return Err(AppError::AlreadyProcessedErr);
|
||||||
@@ -172,6 +172,12 @@ async fn webhook(
|
|||||||
Ok((StatusCode::CREATED, "Task started"))
|
Ok((StatusCode::CREATED, "Task started"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether an event is already queued, or already reviewed.
|
||||||
|
async fn is_known(app_state: &AppState, event_id: u64) -> toasty::Result<bool> {
|
||||||
|
Ok(app_state.queue.is_queued(event_id).await?
|
||||||
|
|| app_state.reviews.is_reviewed(event_id).await?)
|
||||||
|
}
|
||||||
|
|
||||||
pub struct WebhookExtract(pub WebhookType);
|
pub struct WebhookExtract(pub WebhookType);
|
||||||
|
|
||||||
impl<S> FromRequest<S> for WebhookExtract
|
impl<S> FromRequest<S> for WebhookExtract
|
||||||
|
|||||||
+85
-198
@@ -1,200 +1,95 @@
|
|||||||
use std::sync::Arc;
|
use std::{sync::Arc, time::Duration};
|
||||||
|
|
||||||
use futures_util::StreamExt;
|
use tokio::{sync::Semaphore, task::JoinSet};
|
||||||
use serde::{Deserialize, Deserializer};
|
|
||||||
use tokio::sync::Semaphore;
|
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
use tracing::{error, info, instrument};
|
use tracing::{error, info, instrument};
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
bot_actions::review::ReviewOutcome,
|
bot_actions::review::{self, ReviewContext, ReviewOutcome},
|
||||||
dedup::{Dedup, ReviewRecord},
|
|
||||||
event_bus::{EventBus, Reporter},
|
event_bus::{EventBus, Reporter},
|
||||||
gitea::{GiteaAPI, WebhookType},
|
gitea::{GiteaAPI, WebhookType},
|
||||||
job_queue::JobQueue,
|
job_queue::JobQueue,
|
||||||
metrics,
|
metrics,
|
||||||
open_router::OpenRouterClient,
|
open_router::OpenRouterClient,
|
||||||
sandbox::SandboxConfig,
|
review_store::{ReviewRecord, ReviewStore},
|
||||||
|
sandbox::{SandboxConfig, tools},
|
||||||
schemas::Job,
|
schemas::Job,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[derive(Deserialize, Debug)]
|
/// Delay before claiming again after the queue could not be reached.
|
||||||
pub struct ReviewResult {
|
const CLAIM_RETRY_DELAY: Duration = Duration::from_secs(1);
|
||||||
pub reviews: Vec<ReviewItem>,
|
|
||||||
pub comment: String,
|
|
||||||
pub cost: Option<f64>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Deserialize, Debug)]
|
/// Everything the bot works with: the external services and the stores.
|
||||||
pub struct ReviewItem {
|
#[derive(Clone)]
|
||||||
pub filename: String,
|
pub struct BotDeps {
|
||||||
pub line: Option<u64>,
|
pub gitea: GiteaAPI,
|
||||||
#[serde(default, deserialize_with = "deserialize_side")]
|
pub open_router: OpenRouterClient,
|
||||||
pub side: Option<ReviewSide>,
|
pub sandbox: SandboxConfig,
|
||||||
#[serde(default, deserialize_with = "deserialize_severity")]
|
pub reviews: ReviewStore,
|
||||||
pub severity: Option<ReviewSeverity>,
|
pub queue: JobQueue,
|
||||||
pub message: String,
|
pub event_bus: EventBus,
|
||||||
}
|
|
||||||
|
|
||||||
/// Which version of the file a review comment is anchored on.
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
||||||
pub enum ReviewSide {
|
|
||||||
/// The line was added by the pull request: `line` is a line number of the
|
|
||||||
/// new version of the file.
|
|
||||||
Added,
|
|
||||||
/// The line was removed by the pull request: `line` is a line number of the
|
|
||||||
/// old version of the file.
|
|
||||||
Removed,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ReviewSide {
|
|
||||||
/// Reads the side the model asked for, tolerating casing and synonyms.
|
|
||||||
fn parse(raw: &str) -> Option<Self> {
|
|
||||||
match raw.trim().to_ascii_lowercase().as_str() {
|
|
||||||
"added" | "add" | "new" | "right" => Some(Self::Added),
|
|
||||||
"removed" | "remove" | "deleted" | "delete" | "old" | "left" => Some(Self::Removed),
|
|
||||||
_ => None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// What kind of problem a review reports.
|
|
||||||
///
|
|
||||||
/// The categories are the ones the prompt asks for; a review whose severity is
|
|
||||||
/// missing or unreadable falls back to [`ReviewSeverity::Maintainability`], the
|
|
||||||
/// least alarming bucket, instead of failing the whole review.
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
||||||
pub enum ReviewSeverity {
|
|
||||||
/// Wrong behaviour.
|
|
||||||
Bug,
|
|
||||||
/// A vulnerability.
|
|
||||||
Security,
|
|
||||||
/// A resource problem.
|
|
||||||
Performance,
|
|
||||||
/// Everything else: readability, structure, naming, tests.
|
|
||||||
Maintainability,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ReviewSeverity {
|
|
||||||
/// Reads the severity the model asked for, tolerating casing and synonyms.
|
|
||||||
fn parse(raw: &str) -> Option<Self> {
|
|
||||||
match raw.trim().to_ascii_lowercase().as_str() {
|
|
||||||
"bug" | "bugs" | "correctness" | "error" => Some(Self::Bug),
|
|
||||||
"security" | "vulnerability" => Some(Self::Security),
|
|
||||||
"performance" | "perf" => Some(Self::Performance),
|
|
||||||
"maintainability" | "quality" | "style" => Some(Self::Maintainability),
|
|
||||||
_ => None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Lower-case label used in the review markdown.
|
|
||||||
pub fn label(self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
Self::Bug => "bug",
|
|
||||||
Self::Security => "security",
|
|
||||||
Self::Performance => "performance",
|
|
||||||
Self::Maintainability => "maintainability",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Reads the side the model asked for. An unreadable value is ignored instead of
|
|
||||||
/// failing the whole review: the side is then resolved from the changed lines.
|
|
||||||
fn deserialize_side<'de, D>(deserializer: D) -> Result<Option<ReviewSide>, D::Error>
|
|
||||||
where
|
|
||||||
D: Deserializer<'de>,
|
|
||||||
{
|
|
||||||
let raw = Option::<serde_json::Value>::deserialize(deserializer)?;
|
|
||||||
|
|
||||||
Ok(raw
|
|
||||||
.as_ref()
|
|
||||||
.and_then(serde_json::Value::as_str)
|
|
||||||
.and_then(ReviewSide::parse))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Reads the severity the model asked for, ignoring an unreadable value: the
|
|
||||||
/// review is then reported as [`ReviewSeverity::Maintainability`].
|
|
||||||
fn deserialize_severity<'de, D>(deserializer: D) -> Result<Option<ReviewSeverity>, D::Error>
|
|
||||||
where
|
|
||||||
D: Deserializer<'de>,
|
|
||||||
{
|
|
||||||
let raw = Option::<serde_json::Value>::deserialize(deserializer)?;
|
|
||||||
|
|
||||||
Ok(raw
|
|
||||||
.as_ref()
|
|
||||||
.and_then(serde_json::Value::as_str)
|
|
||||||
.and_then(ReviewSeverity::parse))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct Bot {
|
pub struct Bot {
|
||||||
gitea_api: GiteaAPI,
|
deps: BotDeps,
|
||||||
open_router_client: OpenRouterClient,
|
|
||||||
max_concurrent: usize,
|
max_concurrent: usize,
|
||||||
open_router_model: String,
|
|
||||||
sandbox: SandboxConfig,
|
|
||||||
dedup: Dedup,
|
|
||||||
queue: JobQueue,
|
|
||||||
event_bus: EventBus,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Bot {
|
impl Bot {
|
||||||
#[allow(clippy::too_many_arguments)]
|
pub fn new(deps: BotDeps, max_concurrent: usize) -> Self {
|
||||||
pub fn new(
|
|
||||||
gitea_api: GiteaAPI,
|
|
||||||
open_router_client: OpenRouterClient,
|
|
||||||
max_concurrent: usize,
|
|
||||||
open_router_model: String,
|
|
||||||
sandbox: SandboxConfig,
|
|
||||||
dedup: Dedup,
|
|
||||||
queue: JobQueue,
|
|
||||||
event_bus: EventBus,
|
|
||||||
) -> Self {
|
|
||||||
Self {
|
Self {
|
||||||
gitea_api,
|
deps,
|
||||||
open_router_client,
|
|
||||||
max_concurrent,
|
max_concurrent,
|
||||||
open_router_model,
|
|
||||||
sandbox,
|
|
||||||
dedup,
|
|
||||||
queue,
|
|
||||||
event_bus,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Consomme la file jusqu'à l'annulation de `shutdown`.
|
/// Consumes the queue until `shutdown` is cancelled.
|
||||||
///
|
///
|
||||||
/// Un sémaphore borne la concurrence ; les jobs en cours sont laissés se
|
/// A semaphore bounds the concurrency, and a job is only claimed once a slot is
|
||||||
/// terminer. Les jobs non terminés restent en base et seront repris au
|
/// free, so the jobs waiting for one stay `pending`. Running jobs are left to
|
||||||
/// prochain démarrage.
|
/// finish; the unfinished ones stay `processing` and are requeued at the next
|
||||||
|
/// startup.
|
||||||
pub async fn start(&self, shutdown: CancellationToken) -> anyhow::Result<()> {
|
pub async fn start(&self, shutdown: CancellationToken) -> anyhow::Result<()> {
|
||||||
info!("Bot started");
|
info!("Bot started");
|
||||||
|
|
||||||
let semaphore = Arc::new(Semaphore::new(self.max_concurrent));
|
let semaphore = Arc::new(Semaphore::new(self.max_concurrent));
|
||||||
let mut tasks = tokio::task::JoinSet::new();
|
let mut tasks = JoinSet::new();
|
||||||
let mut pending = std::pin::pin!(self.queue.pending());
|
|
||||||
|
|
||||||
loop {
|
'consume: loop {
|
||||||
let job = tokio::select! {
|
let permit = tokio::select! {
|
||||||
biased;
|
biased;
|
||||||
_ = shutdown.cancelled() => break,
|
() = shutdown.cancelled() => break,
|
||||||
job = pending.next() => match job {
|
permit = semaphore.clone().acquire_owned() => permit?,
|
||||||
Some(job) => job,
|
|
||||||
None => break,
|
|
||||||
},
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let job = loop {
|
||||||
while let Some(res) = tasks.try_join_next() {
|
while let Some(res) = tasks.try_join_next() {
|
||||||
if let Err(err) = res {
|
if let Err(err) = res {
|
||||||
error!("Task panicked: {err}");
|
error!("Task panicked: {err}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let permit = semaphore.clone().acquire_owned().await?;
|
match self.deps.queue.claim_next().await {
|
||||||
|
Ok(Some(job)) => break job,
|
||||||
if let Err(err) = self.queue.mark_processing(job.id).await {
|
Ok(None) => {
|
||||||
error!(%err, "failed to mark job as processing");
|
tokio::select! {
|
||||||
|
biased;
|
||||||
|
() = shutdown.cancelled() => break 'consume,
|
||||||
|
() = self.deps.queue.wait_for_work() => {}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
error!(%err, "failed to claim a job");
|
||||||
|
|
||||||
|
tokio::select! {
|
||||||
|
biased;
|
||||||
|
() = shutdown.cancelled() => break 'consume,
|
||||||
|
() = tokio::time::sleep(CLAIM_RETRY_DELAY) => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
let bot = self.clone();
|
let bot = self.clone();
|
||||||
metrics::increment_task_active();
|
metrics::increment_task_active();
|
||||||
@@ -212,30 +107,35 @@ impl Bot {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Runs a claimed job and records its outcome.
|
||||||
async fn process(&self, job: Job) {
|
async fn process(&self, job: Job) {
|
||||||
|
self.run_job(&job).await;
|
||||||
|
|
||||||
|
if let Err(err) = self.deps.queue.remove(job.id).await {
|
||||||
|
error!(%err, "failed to remove job");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_job(&self, job: &Job) {
|
||||||
|
let deps = &self.deps;
|
||||||
let event_type = job.payload.event_type_str();
|
let event_type = job.payload.event_type_str();
|
||||||
|
|
||||||
match self.dedup.is_reviewed(job.event_id).await {
|
match deps.reviews.is_reviewed(job.event_id).await {
|
||||||
Ok(true) => {
|
Ok(true) => {
|
||||||
metrics::task_skipped(event_type);
|
metrics::task_skipped(event_type);
|
||||||
info!("Review already done, skipping");
|
info!("Review already done, skipping");
|
||||||
self.event_bus.skip(job.id, &job.repo, job.pr);
|
deps.event_bus.skip(job.id, &job.repo, job.pr);
|
||||||
|
|
||||||
if let Err(err) = self.queue.remove(job.id).await {
|
|
||||||
error!(%err, "failed to remove job");
|
|
||||||
}
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Ok(false) => {}
|
Ok(false) => {}
|
||||||
Err(err) => error!(%err, "Dedup lookup failed, reviewing anyway"),
|
Err(err) => error!(%err, "Review lookup failed, reviewing anyway"),
|
||||||
}
|
}
|
||||||
|
|
||||||
let reporter = self.event_bus.start(job.id, job.repo.clone(), job.pr);
|
let reporter = deps.event_bus.start(job.id, job.repo.clone(), job.pr);
|
||||||
|
|
||||||
let payload = job.payload.0.clone();
|
|
||||||
let started = std::time::Instant::now();
|
let started = std::time::Instant::now();
|
||||||
let result = self.exec(payload, &reporter).await;
|
let result = self.exec(&job.payload, &reporter).await;
|
||||||
let duration_ms = started.elapsed().as_millis() as u64;
|
let duration_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||||
|
|
||||||
let (ok, outcome) = match result {
|
let (ok, outcome) = match result {
|
||||||
Ok(outcome) => {
|
Ok(outcome) => {
|
||||||
@@ -251,60 +151,47 @@ impl Bot {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let comments = outcome.comments as u64;
|
|
||||||
|
|
||||||
let record = ReviewRecord {
|
let record = ReviewRecord {
|
||||||
event_id: job.event_id,
|
event_id: job.event_id,
|
||||||
repo: job.repo.clone(),
|
repo: job.repo.clone(),
|
||||||
pr: job.pr,
|
pr: job.pr,
|
||||||
ok,
|
ok,
|
||||||
model: self.open_router_model.clone(),
|
model: deps.open_router.model().to_string(),
|
||||||
markdown: outcome.markdown,
|
markdown: outcome.markdown,
|
||||||
cost: outcome.cost,
|
cost: outcome.cost,
|
||||||
comments,
|
comments: outcome.comments,
|
||||||
duration_ms,
|
duration_ms,
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Err(err) = self.dedup.mark_reviewed(record).await {
|
if let Err(err) = deps.reviews.record(record).await {
|
||||||
error!(%err, "failed to record review");
|
error!(%err, "failed to record review");
|
||||||
}
|
}
|
||||||
|
|
||||||
self.event_bus
|
deps.event_bus
|
||||||
.finish(job.id, &job.repo, job.pr, ok, comments, duration_ms);
|
.finish(job.id, &job.repo, job.pr, ok, outcome.comments, duration_ms);
|
||||||
|
|
||||||
if let Err(err) = self.queue.remove(job.id).await {
|
|
||||||
error!(%err, "failed to remove job");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[instrument(skip(self, webhook, reporter), fields(repo, pr))]
|
/// Dispatches a webhook to the action it asks for.
|
||||||
pub async fn exec(
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(repo = %webhook.repository_full_name(), pr = webhook.pull_request_number())
|
||||||
|
)]
|
||||||
|
async fn exec(
|
||||||
&self,
|
&self,
|
||||||
webhook: WebhookType,
|
webhook: &WebhookType,
|
||||||
reporter: &Reporter,
|
reporter: &Reporter,
|
||||||
) -> anyhow::Result<ReviewOutcome> {
|
) -> anyhow::Result<ReviewOutcome> {
|
||||||
match &webhook {
|
let deps = &self.deps;
|
||||||
WebhookType::Review(p) => {
|
let ctx = ReviewContext {
|
||||||
tracing::Span::current().record("repo", &p.repository.full_name);
|
gitea: &deps.gitea,
|
||||||
tracing::Span::current().record("pr", p.pull_request.number);
|
open_router: &deps.open_router,
|
||||||
}
|
sandbox: &deps.sandbox,
|
||||||
|
reporter,
|
||||||
};
|
};
|
||||||
|
let tools = tools::for_webhook(webhook);
|
||||||
let tools = crate::sandbox::tools::for_webhook(&webhook);
|
|
||||||
|
|
||||||
match webhook {
|
match webhook {
|
||||||
WebhookType::Review(review_payload) => {
|
WebhookType::Review(payload) => review::exec_review(&ctx, tools, payload).await,
|
||||||
crate::bot_actions::review::exec_review(
|
|
||||||
&self.gitea_api,
|
|
||||||
&self.open_router_client,
|
|
||||||
&self.open_router_model,
|
|
||||||
&self.sandbox,
|
|
||||||
tools,
|
|
||||||
review_payload,
|
|
||||||
reporter,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,177 @@
|
|||||||
|
//! Anchoring of the model's reviews on the lines a pull request changes.
|
||||||
|
|
||||||
|
use tracing::warn;
|
||||||
|
|
||||||
|
use super::diff::ChangedLines;
|
||||||
|
use crate::domain::review::ReviewResult;
|
||||||
|
|
||||||
|
/// Resolves the side each review is anchored on and drops the reviews that do
|
||||||
|
/// not match a line the pull request changes.
|
||||||
|
///
|
||||||
|
/// The model is asked to pick a line and a side from the provided lists, but
|
||||||
|
/// nothing forces it to, and Gitea accepts any position: a wrong one places the
|
||||||
|
/// comment on an unrelated line of the file instead of failing. A review that
|
||||||
|
/// omits its side is resolved from the lists, and one that matches no changed
|
||||||
|
/// line is dropped.
|
||||||
|
pub(super) fn resolve_review_sides(review_result: &mut ReviewResult, changed_lines: &ChangedLines) {
|
||||||
|
let mut dropped = Vec::new();
|
||||||
|
|
||||||
|
review_result.reviews.retain_mut(|review| {
|
||||||
|
let side = review.line.and_then(|line| {
|
||||||
|
let file = changed_lines
|
||||||
|
.iter()
|
||||||
|
.find(|file| file.filename == review.filename)?;
|
||||||
|
|
||||||
|
let side = review.side.or_else(|| file.side_of(line))?;
|
||||||
|
|
||||||
|
file.lines(side).contains(&line).then_some(side)
|
||||||
|
});
|
||||||
|
|
||||||
|
match side {
|
||||||
|
Some(side) => {
|
||||||
|
review.side = Some(side);
|
||||||
|
true
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
dropped.push(match review.line {
|
||||||
|
Some(line) => format!("{}:{line}", review.filename),
|
||||||
|
None => format!("{}:no line", review.filename),
|
||||||
|
});
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if !dropped.is_empty() {
|
||||||
|
warn!(
|
||||||
|
dropped = dropped.len(),
|
||||||
|
reviews = %dropped.join(", "),
|
||||||
|
"Dropped reviews that are not anchored on a changed line"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::{
|
||||||
|
bot_actions::review::{diff::parse_changed_lines, fixtures::*},
|
||||||
|
domain::review::ReviewSide,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reviews_keep_their_changed_line_and_resolve_their_side() {
|
||||||
|
let changed_lines = parse_changed_lines(DIFF);
|
||||||
|
let mut review_result = review_result(vec![
|
||||||
|
review("src/foo.rs", Some(4), Some(ReviewSide::Added)),
|
||||||
|
review("src/foo.rs", Some(2), Some(ReviewSide::Removed)),
|
||||||
|
review("src/bar.rs", Some(13), None),
|
||||||
|
]);
|
||||||
|
|
||||||
|
resolve_review_sides(&mut review_result, &changed_lines);
|
||||||
|
|
||||||
|
let sides = review_result
|
||||||
|
.reviews
|
||||||
|
.iter()
|
||||||
|
.map(|review| review.side)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
sides,
|
||||||
|
vec![
|
||||||
|
Some(ReviewSide::Added),
|
||||||
|
Some(ReviewSide::Removed),
|
||||||
|
Some(ReviewSide::Added)
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_line_changed_on_both_sides_defaults_to_added() {
|
||||||
|
let changed_lines = parse_changed_lines(DIFF);
|
||||||
|
let mut review_result = review_result(vec![review("src/foo.rs", Some(2), None)]);
|
||||||
|
|
||||||
|
resolve_review_sides(&mut review_result, &changed_lines);
|
||||||
|
|
||||||
|
assert_eq!(review_result.reviews[0].side, Some(ReviewSide::Added));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reviews_outside_changed_lines_are_dropped() {
|
||||||
|
let changed_lines = parse_changed_lines(DIFF);
|
||||||
|
let mut review_result = review_result(vec![
|
||||||
|
// valid
|
||||||
|
review("src/foo.rs", Some(2), Some(ReviewSide::Added)),
|
||||||
|
// a line that exists but is not part of the change
|
||||||
|
review("src/foo.rs", Some(3), None),
|
||||||
|
// a line beyond the change
|
||||||
|
review("src/foo.rs", Some(999), None),
|
||||||
|
// a changed line, but on the wrong side
|
||||||
|
review("src/foo.rs", Some(4), Some(ReviewSide::Removed)),
|
||||||
|
// a line changed in another file
|
||||||
|
review("src/foo.rs", Some(11), None),
|
||||||
|
// an unknown file
|
||||||
|
review("src/baz.rs", Some(1), None),
|
||||||
|
// no line at all
|
||||||
|
review("src/bar.rs", None, None),
|
||||||
|
]);
|
||||||
|
|
||||||
|
resolve_review_sides(&mut review_result, &changed_lines);
|
||||||
|
|
||||||
|
assert_eq!(review_result.reviews.len(), 1);
|
||||||
|
assert_eq!(review_result.reviews[0].filename, "src/foo.rs");
|
||||||
|
assert_eq!(review_result.reviews[0].line, Some(2));
|
||||||
|
assert_eq!(review_result.reviews[0].side, Some(ReviewSide::Added));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_deleted_line_is_kept_as_a_removed_anchor() {
|
||||||
|
let changed_lines = parse_changed_lines(DELETION_ONLY);
|
||||||
|
let mut review_result = review_result(vec![
|
||||||
|
review("src/old.rs", Some(2), Some(ReviewSide::Removed)),
|
||||||
|
review("src/old.rs", Some(2), Some(ReviewSide::Added)),
|
||||||
|
]);
|
||||||
|
|
||||||
|
resolve_review_sides(&mut review_result, &changed_lines);
|
||||||
|
|
||||||
|
assert_eq!(review_result.reviews.len(), 1);
|
||||||
|
assert_eq!(review_result.reviews[0].side, Some(ReviewSide::Removed));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn odd_sides_from_the_model_are_tolerated() {
|
||||||
|
let changed_lines = parse_changed_lines(DIFF);
|
||||||
|
let mut review_result: ReviewResult = serde_json::from_str(
|
||||||
|
r#"{
|
||||||
|
"reviews": [
|
||||||
|
{ "filename": "src/foo.rs", "line": 4, "side": "Added", "message": "a" },
|
||||||
|
{ "filename": "src/foo.rs", "line": 2, "side": "new", "message": "b" },
|
||||||
|
{ "filename": "src/foo.rs", "line": 2, "side": "REMOVED", "message": "c" },
|
||||||
|
{ "filename": "src/foo.rs", "line": 2, "side": "banana", "message": "d" },
|
||||||
|
{ "filename": "src/foo.rs", "line": 5, "message": "e" }
|
||||||
|
],
|
||||||
|
"comment": ""
|
||||||
|
}"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
resolve_review_sides(&mut review_result, &changed_lines);
|
||||||
|
|
||||||
|
let sides = review_result
|
||||||
|
.reviews
|
||||||
|
.iter()
|
||||||
|
.map(|review| review.side)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
sides,
|
||||||
|
vec![
|
||||||
|
Some(ReviewSide::Added),
|
||||||
|
Some(ReviewSide::Added),
|
||||||
|
Some(ReviewSide::Removed),
|
||||||
|
Some(ReviewSide::Added),
|
||||||
|
Some(ReviewSide::Added)
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,389 @@
|
|||||||
|
//! Parsing of a pull request unified diff into the lines it changes.
|
||||||
|
|
||||||
|
use tracing::warn;
|
||||||
|
|
||||||
|
use crate::{domain::review::ReviewSide, gitea::PullRequestFile};
|
||||||
|
|
||||||
|
/// The lines a pull request changed, per file.
|
||||||
|
///
|
||||||
|
/// Line numbers are the ones Gitea expects to place a review comment:
|
||||||
|
/// [`ReviewSide::Added`] numbers refer to the new version of the file (sent as
|
||||||
|
/// `new_position`), [`ReviewSide::Removed`] numbers to the old version (sent as
|
||||||
|
/// `old_position`).
|
||||||
|
pub(super) type ChangedLines = Vec<ChangedFile>;
|
||||||
|
|
||||||
|
pub(super) struct ChangedFile {
|
||||||
|
pub(super) filename: String,
|
||||||
|
pub(super) added: Vec<u64>,
|
||||||
|
pub(super) removed: Vec<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ChangedFile {
|
||||||
|
fn new(filename: &str) -> Self {
|
||||||
|
Self {
|
||||||
|
filename: String::from(filename),
|
||||||
|
added: Vec::new(),
|
||||||
|
removed: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Lines that can be commented on for the given side.
|
||||||
|
pub(super) fn lines(&self, side: ReviewSide) -> &[u64] {
|
||||||
|
match side {
|
||||||
|
ReviewSide::Added => &self.added,
|
||||||
|
ReviewSide::Removed => &self.removed,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Side a line belongs to, used when the model did not state one.
|
||||||
|
pub(super) fn side_of(&self, line: u64) -> Option<ReviewSide> {
|
||||||
|
if self.added.contains(&line) {
|
||||||
|
Some(ReviewSide::Added)
|
||||||
|
} else if self.removed.contains(&line) {
|
||||||
|
Some(ReviewSide::Removed)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Lists the lines changed by the diff, per file, on both sides.
|
||||||
|
///
|
||||||
|
/// Only the line numbers are kept: the model reads the code itself through the
|
||||||
|
/// sandbox tools.
|
||||||
|
pub(super) fn parse_changed_lines(git_diff: &str) -> ChangedLines {
|
||||||
|
let mut files = Vec::new();
|
||||||
|
let mut current_file: Option<String> = None;
|
||||||
|
let mut in_hunk = false;
|
||||||
|
let mut old_line: u64 = 0;
|
||||||
|
let mut new_line: u64 = 0;
|
||||||
|
|
||||||
|
for line in git_diff.lines() {
|
||||||
|
if line.starts_with("diff --git ") {
|
||||||
|
current_file = None;
|
||||||
|
in_hunk = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// `--- a/x` and `+++ b/x` only appear before the first hunk of a file.
|
||||||
|
// Inside a hunk, a line may legitimately start with them: removing
|
||||||
|
// `--x` gives `---x`, adding `++i;` gives `+++i;`.
|
||||||
|
if !in_hunk && (line.starts_with("--- ") || line.starts_with("+++ ")) {
|
||||||
|
// A header is never a content line: `+++ /dev/null` on a deleted
|
||||||
|
// file keeps the path of the other side in place.
|
||||||
|
if let Some(path) = header_file_path(line) {
|
||||||
|
current_file = Some(path);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if line.starts_with("@@") {
|
||||||
|
if let Some((old_start, new_start)) = parse_hunk_starts(line) {
|
||||||
|
old_line = old_start;
|
||||||
|
new_line = new_start;
|
||||||
|
in_hunk = true;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some(filename) = current_file.as_deref() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
match line.as_bytes().first() {
|
||||||
|
Some(b' ') => {
|
||||||
|
old_line += 1;
|
||||||
|
new_line += 1;
|
||||||
|
}
|
||||||
|
Some(b'-') => {
|
||||||
|
changed_file(&mut files, filename).removed.push(old_line);
|
||||||
|
old_line += 1;
|
||||||
|
}
|
||||||
|
Some(b'+') => {
|
||||||
|
changed_file(&mut files, filename).added.push(new_line);
|
||||||
|
new_line += 1;
|
||||||
|
}
|
||||||
|
// `\ No newline at end of file`, and anything unexpected: a line
|
||||||
|
// that advances neither side.
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
files
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Path of the file on one side of the diff, from a `--- a/<path>` or
|
||||||
|
/// `+++ b/<path>` header line.
|
||||||
|
///
|
||||||
|
/// These lines are the only unambiguous source for the path: the `diff --git`
|
||||||
|
/// line is cut at the first space, and it names the old path of a renamed file.
|
||||||
|
/// `None` for `/dev/null`, which leaves the path of the other side in place.
|
||||||
|
fn header_file_path(line: &str) -> Option<String> {
|
||||||
|
let (prefix, raw) = match line.strip_prefix("--- ") {
|
||||||
|
Some(raw) => ("a/", raw),
|
||||||
|
None => ("b/", line.strip_prefix("+++ ")?),
|
||||||
|
};
|
||||||
|
|
||||||
|
let path = decode_git_path(raw);
|
||||||
|
|
||||||
|
Some(String::from(path.strip_prefix(prefix)?))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decodes a path as git writes it in a diff header: git wraps it in quotes and
|
||||||
|
/// escapes the bytes that need it (`\303\251` for `é`) when the path contains
|
||||||
|
/// non-printable or non-ASCII characters.
|
||||||
|
fn decode_git_path(raw: &str) -> String {
|
||||||
|
let Some(quoted) = raw.strip_prefix('"').and_then(|raw| raw.strip_suffix('"')) else {
|
||||||
|
return String::from(raw);
|
||||||
|
};
|
||||||
|
|
||||||
|
let bytes = quoted.as_bytes();
|
||||||
|
let mut decoded = Vec::with_capacity(bytes.len());
|
||||||
|
let mut index = 0;
|
||||||
|
|
||||||
|
while let Some(byte) = bytes.get(index) {
|
||||||
|
index += 1;
|
||||||
|
|
||||||
|
if *byte != b'\\' {
|
||||||
|
decoded.push(*byte);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
match bytes.get(index) {
|
||||||
|
// Octal escapes are the ones that matter for a path: git uses them
|
||||||
|
// for every non-ASCII byte.
|
||||||
|
Some(digit @ b'0'..=b'7') => {
|
||||||
|
let mut value = u32::from(digit - b'0');
|
||||||
|
let mut digits = 1;
|
||||||
|
|
||||||
|
while digits < 3 {
|
||||||
|
match bytes.get(index + digits) {
|
||||||
|
Some(next @ b'0'..=b'7') => {
|
||||||
|
value = value * 8 + u32::from(next - b'0');
|
||||||
|
digits += 1;
|
||||||
|
}
|
||||||
|
_ => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
decoded.push(u8::try_from(value).unwrap_or(b'?'));
|
||||||
|
index += digits;
|
||||||
|
}
|
||||||
|
Some(escaped) => {
|
||||||
|
decoded.push(match escaped {
|
||||||
|
b't' => b'\t',
|
||||||
|
b'n' => b'\n',
|
||||||
|
b'r' => b'\r',
|
||||||
|
other => *other,
|
||||||
|
});
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
None => decoded.push(b'\\'),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
String::from_utf8_lossy(&decoded).into_owned()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Replaces the paths parsed from the diff with the exact paths reported by the
|
||||||
|
/// API, which are the ones the model sees in the sandbox.
|
||||||
|
pub(super) fn resolve_filenames(changed_lines: &mut ChangedLines, files: &[PullRequestFile]) {
|
||||||
|
for changed in changed_lines.iter_mut() {
|
||||||
|
let parsed = changed.filename.as_str();
|
||||||
|
|
||||||
|
let Some(file) = files.iter().find(|file| file.filename == parsed) else {
|
||||||
|
if !files.is_empty() {
|
||||||
|
warn!(path = %parsed, "Changed file is not in the pull request file list");
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
changed.filename = file.filename.clone();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Basenames of generated files that are never reviewed.
|
||||||
|
///
|
||||||
|
/// Their diffs are machine-written dependency churn: they run to thousands of
|
||||||
|
/// lines, flood the prompt with line numbers and drown the code the model should
|
||||||
|
/// look at. A lockfile is also never a place where a review comment belongs.
|
||||||
|
const IGNORED_FILE_NAMES: [&str; 16] = [
|
||||||
|
"Cargo.lock",
|
||||||
|
"package-lock.json",
|
||||||
|
"npm-shrinkwrap.json",
|
||||||
|
"yarn.lock",
|
||||||
|
"pnpm-lock.yaml",
|
||||||
|
"bun.lock",
|
||||||
|
"bun.lockb",
|
||||||
|
"composer.lock",
|
||||||
|
"Gemfile.lock",
|
||||||
|
"poetry.lock",
|
||||||
|
"uv.lock",
|
||||||
|
"Pipfile.lock",
|
||||||
|
"go.sum",
|
||||||
|
"packages.lock.json",
|
||||||
|
"flake.lock",
|
||||||
|
"pubspec.lock",
|
||||||
|
];
|
||||||
|
|
||||||
|
/// Whether a changed path is a generated file that is never reviewed.
|
||||||
|
///
|
||||||
|
/// The comparison is on the basename, so a lockfile nested in a workspace (for
|
||||||
|
/// example `crates/foo/Cargo.lock`) is matched too.
|
||||||
|
pub(super) fn is_ignored(filename: &str) -> bool {
|
||||||
|
let basename = filename.rsplit('/').next().unwrap_or(filename);
|
||||||
|
|
||||||
|
IGNORED_FILE_NAMES.contains(&basename)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Drops the generated files from the changed lines.
|
||||||
|
///
|
||||||
|
/// Removing them here keeps the prompt free of their line numbers, and also makes
|
||||||
|
/// [`resolve_review_sides`] reject any review the model anchors on one of them.
|
||||||
|
pub(super) fn drop_generated_files(changed_lines: &mut ChangedLines) {
|
||||||
|
changed_lines.retain(|file| !is_ignored(&file.filename));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Borrows the entry of `files` for a file, creating it on its first change.
|
||||||
|
fn changed_file<'a>(files: &'a mut ChangedLines, filename: &str) -> &'a mut ChangedFile {
|
||||||
|
let index = match files.iter().position(|file| file.filename == filename) {
|
||||||
|
Some(index) => index,
|
||||||
|
None => {
|
||||||
|
files.push(ChangedFile::new(filename));
|
||||||
|
files.len() - 1
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
&mut files[index]
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extracts the old and new starting line numbers of a hunk header such as
|
||||||
|
/// `@@ -12,3 +12,5 @@`. The counts are optional and git may append a section
|
||||||
|
/// heading after the closing `@@`.
|
||||||
|
fn parse_hunk_starts(hunk_header: &str) -> Option<(u64, u64)> {
|
||||||
|
let body = hunk_header.strip_prefix("@@ ")?;
|
||||||
|
let body = body.split(" @@").next()?;
|
||||||
|
|
||||||
|
let (old, new) = body.split_once(" +")?;
|
||||||
|
let old = old.strip_prefix('-')?.split(',').next()?;
|
||||||
|
let new = new.split(',').next()?;
|
||||||
|
|
||||||
|
Some((old.parse().ok()?, new.parse().ok()?))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::bot_actions::review::{fixtures::*, prompt::format_changed_lines};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_deletion_only_pull_request_lists_removed_lines() {
|
||||||
|
let expected = "src/old.rs: removed 1, 2, 3";
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
format_changed_lines(&parse_changed_lines(DELETION_ONLY)),
|
||||||
|
expected
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn hunk_content_starting_with_plus_or_minus_is_counted() {
|
||||||
|
let expected = "src/tricky.js: added 2 / removed 2";
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
format_changed_lines(&parse_changed_lines(TRICKY_CONTENT)),
|
||||||
|
expected
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_renamed_file_uses_its_new_path() {
|
||||||
|
let expected = "src/new.rs: added 1 / removed 1";
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
format_changed_lines(&parse_changed_lines(RENAMED)),
|
||||||
|
expected
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_path_with_a_space_is_read_from_the_headers() {
|
||||||
|
let expected = "src/my file.rs: added 2";
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
format_changed_lines(&parse_changed_lines(PATH_WITH_SPACE)),
|
||||||
|
expected
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_quoted_path_is_decoded() {
|
||||||
|
let expected = "docs/café.md: added 2";
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
format_changed_lines(&parse_changed_lines(QUOTED_PATH)),
|
||||||
|
expected
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn filenames_are_resolved_against_the_api_list() {
|
||||||
|
let mut changed_lines = parse_changed_lines(DIFF);
|
||||||
|
let files = vec![
|
||||||
|
pull_request_file("src/bar.rs", None, "modified"),
|
||||||
|
pull_request_file("src/foo.rs", Some("src/renamed.rs"), "renamed"),
|
||||||
|
];
|
||||||
|
|
||||||
|
resolve_filenames(&mut changed_lines, &files);
|
||||||
|
|
||||||
|
assert_eq!(changed_lines[0].filename, "src/foo.rs");
|
||||||
|
assert_eq!(changed_lines[1].filename, "src/bar.rs");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_file_absent_from_the_api_list_keeps_the_diff_path() {
|
||||||
|
let mut changed_lines = parse_changed_lines(DIFF);
|
||||||
|
let files = vec![pull_request_file("src/bar.rs", None, "modified")];
|
||||||
|
|
||||||
|
resolve_filenames(&mut changed_lines, &files);
|
||||||
|
|
||||||
|
assert_eq!(changed_lines[0].filename, "src/foo.rs");
|
||||||
|
assert_eq!(changed_lines[1].filename, "src/bar.rs");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn generated_lockfiles_are_ignored() {
|
||||||
|
assert!(is_ignored("Cargo.lock"));
|
||||||
|
assert!(is_ignored("crates/steep-server/Cargo.lock"));
|
||||||
|
assert!(is_ignored("package-lock.json"));
|
||||||
|
assert!(is_ignored("web/yarn.lock"));
|
||||||
|
assert!(is_ignored("go.sum"));
|
||||||
|
|
||||||
|
assert!(!is_ignored("src/main.rs"));
|
||||||
|
assert!(!is_ignored("Cargo.toml"));
|
||||||
|
assert!(!is_ignored("docs/lockfiles.md"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn generated_files_are_dropped_from_the_changed_lines() {
|
||||||
|
const DIFF_WITH_LOCKFILE: &str = concat!(
|
||||||
|
"diff --git a/Cargo.lock b/Cargo.lock\n",
|
||||||
|
"--- a/Cargo.lock\n",
|
||||||
|
"+++ b/Cargo.lock\n",
|
||||||
|
"@@ -1,1 +1,2 @@\n",
|
||||||
|
" [[package]]\n",
|
||||||
|
"+name = \"x\"\n",
|
||||||
|
"diff --git a/src/foo.rs b/src/foo.rs\n",
|
||||||
|
"--- a/src/foo.rs\n",
|
||||||
|
"+++ b/src/foo.rs\n",
|
||||||
|
"@@ -1,1 +1,2 @@\n",
|
||||||
|
" fn a() {}\n",
|
||||||
|
"+fn b() {}\n",
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut changed_lines = parse_changed_lines(DIFF_WITH_LOCKFILE);
|
||||||
|
drop_generated_files(&mut changed_lines);
|
||||||
|
|
||||||
|
assert_eq!(format_changed_lines(&changed_lines), "src/foo.rs: added 2");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
//! Shared fixtures of the review tests.
|
||||||
|
|
||||||
|
use crate::{
|
||||||
|
domain::review::{ReviewItem, ReviewResult, ReviewSide},
|
||||||
|
gitea::PullRequestFile,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Additions, a removal and a line changed on both sides.
|
||||||
|
pub(super) const DIFF: &str = concat!(
|
||||||
|
"diff --git a/src/foo.rs b/src/foo.rs\n",
|
||||||
|
"--- a/src/foo.rs\n",
|
||||||
|
"+++ b/src/foo.rs\n",
|
||||||
|
"@@ -1,4 +1,6 @@\n",
|
||||||
|
" fn main() {\n",
|
||||||
|
"+ let x = 1;\n",
|
||||||
|
"- let removed = 0;\n",
|
||||||
|
" println!(\"hello\");\n",
|
||||||
|
"+ let y = 2;\n",
|
||||||
|
"+ let z = 3;\n",
|
||||||
|
" }\n",
|
||||||
|
"diff --git a/src/bar.rs b/src/bar.rs\n",
|
||||||
|
"--- a/src/bar.rs\n",
|
||||||
|
"+++ b/src/bar.rs\n",
|
||||||
|
"@@ -10,4 +10,6 @@\n",
|
||||||
|
" old context\n",
|
||||||
|
"+ let a = 10;\n",
|
||||||
|
" more context\n",
|
||||||
|
"+ let b = 20;\n",
|
||||||
|
);
|
||||||
|
|
||||||
|
/// A pull request that only deletes a file.
|
||||||
|
pub(super) const DELETION_ONLY: &str = concat!(
|
||||||
|
"diff --git a/src/old.rs b/src/old.rs\n",
|
||||||
|
"deleted file mode 100644\n",
|
||||||
|
"--- a/src/old.rs\n",
|
||||||
|
"+++ /dev/null\n",
|
||||||
|
"@@ -1,3 +0,0 @@\n",
|
||||||
|
"-fn a() {}\n",
|
||||||
|
"-fn b() {}\n",
|
||||||
|
"-fn c() {}\n",
|
||||||
|
);
|
||||||
|
|
||||||
|
/// A hunk whose content starts with `+++` / `---`, which must not be taken
|
||||||
|
/// for the file headers.
|
||||||
|
pub(super) const TRICKY_CONTENT: &str = concat!(
|
||||||
|
"diff --git a/src/tricky.js b/src/tricky.js\n",
|
||||||
|
"--- a/src/tricky.js\n",
|
||||||
|
"+++ b/src/tricky.js\n",
|
||||||
|
"@@ -1,4 +1,4 @@\n",
|
||||||
|
" let i = 0;\n",
|
||||||
|
"+++i;\n",
|
||||||
|
"---x;\n",
|
||||||
|
" console.log(i);\n",
|
||||||
|
);
|
||||||
|
|
||||||
|
/// A pull request that renames a file.
|
||||||
|
pub(super) const RENAMED: &str = concat!(
|
||||||
|
"diff --git a/src/old.rs b/src/new.rs\n",
|
||||||
|
"similarity index 50%\n",
|
||||||
|
"rename from src/old.rs\n",
|
||||||
|
"rename to src/new.rs\n",
|
||||||
|
"--- a/src/old.rs\n",
|
||||||
|
"+++ b/src/new.rs\n",
|
||||||
|
"@@ -1,1 +1,1 @@\n",
|
||||||
|
"-fn old() {}\n",
|
||||||
|
"+fn new() {}\n",
|
||||||
|
);
|
||||||
|
|
||||||
|
/// A file whose path contains a space, which the `diff --git` line cannot
|
||||||
|
/// express without ambiguity.
|
||||||
|
pub(super) const PATH_WITH_SPACE: &str = concat!(
|
||||||
|
"diff --git a/src/my file.rs b/src/my file.rs\n",
|
||||||
|
"--- a/src/my file.rs\n",
|
||||||
|
"+++ b/src/my file.rs\n",
|
||||||
|
"@@ -1,1 +1,2 @@\n",
|
||||||
|
" fn a() {}\n",
|
||||||
|
"+fn b() {}\n",
|
||||||
|
);
|
||||||
|
|
||||||
|
/// A file whose path git quotes and escapes (`caf\\303\\251.md` is
|
||||||
|
/// `caf\u{e9}.md`).
|
||||||
|
pub(super) const QUOTED_PATH: &str = concat!(
|
||||||
|
"diff --git \"a/docs/caf\\303\\251.md\" \"b/docs/caf\\303\\251.md\"\n",
|
||||||
|
"--- \"a/docs/caf\\303\\251.md\"\n",
|
||||||
|
"+++ \"b/docs/caf\\303\\251.md\"\n",
|
||||||
|
"@@ -1,1 +1,2 @@\n",
|
||||||
|
" intro\n",
|
||||||
|
"+ajout\n",
|
||||||
|
);
|
||||||
|
|
||||||
|
pub(super) fn review(filename: &str, line: Option<u64>, side: Option<ReviewSide>) -> ReviewItem {
|
||||||
|
ReviewItem {
|
||||||
|
filename: String::from(filename),
|
||||||
|
line,
|
||||||
|
side,
|
||||||
|
severity: None,
|
||||||
|
message: String::from("message"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn review_result(reviews: Vec<ReviewItem>) -> ReviewResult {
|
||||||
|
ReviewResult {
|
||||||
|
reviews,
|
||||||
|
comment: String::new(),
|
||||||
|
cost: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn pull_request_file(
|
||||||
|
filename: &str,
|
||||||
|
previous_filename: Option<&str>,
|
||||||
|
status: &str,
|
||||||
|
) -> PullRequestFile {
|
||||||
|
PullRequestFile {
|
||||||
|
filename: String::from(filename),
|
||||||
|
previous_filename: previous_filename.map(String::from),
|
||||||
|
status: String::from(status),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,210 @@
|
|||||||
|
//! The pull request review action.
|
||||||
|
//!
|
||||||
|
//! [`exec_review`] drives one review end to end. The work is split by concern:
|
||||||
|
//! - [`diff`] parses the pull request diff into the lines it changes,
|
||||||
|
//! - [`prompt`] renders those changes for the model,
|
||||||
|
//! - [`anchor`] checks that the model's reviews land on changed lines,
|
||||||
|
//! - [`render`] builds the markdown summary posted on the pull request.
|
||||||
|
|
||||||
|
mod anchor;
|
||||||
|
mod diff;
|
||||||
|
mod prompt;
|
||||||
|
mod render;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod fixtures;
|
||||||
|
|
||||||
|
use tracing::{info, instrument, warn};
|
||||||
|
|
||||||
|
use crate::{
|
||||||
|
consts::{BOT_PROCESS_MSG, REVIEW_PROMPT, SANDBOX_SYSTEM_PROMPT},
|
||||||
|
domain::review::ReviewResult,
|
||||||
|
event_bus::Reporter,
|
||||||
|
gitea::{GiteaAPI, ReviewPayload},
|
||||||
|
metrics,
|
||||||
|
open_router::{OpenRouterClient, Tool},
|
||||||
|
sandbox::{Sandbox, SandboxConfig, agent},
|
||||||
|
};
|
||||||
|
|
||||||
|
use self::{
|
||||||
|
anchor::resolve_review_sides,
|
||||||
|
diff::{drop_generated_files, parse_changed_lines, resolve_filenames},
|
||||||
|
prompt::format_changes,
|
||||||
|
render::review_result_to_markdown,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// The services a review needs, borrowed from the bot for the duration of one job.
|
||||||
|
pub struct ReviewContext<'a> {
|
||||||
|
pub gitea: &'a GiteaAPI,
|
||||||
|
pub open_router: &'a OpenRouterClient,
|
||||||
|
pub sandbox: &'a SandboxConfig,
|
||||||
|
pub reporter: &'a Reporter,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct ReviewOutcome {
|
||||||
|
pub markdown: String,
|
||||||
|
pub cost: Option<f64>,
|
||||||
|
pub comments: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ReviewOutcome {
|
||||||
|
pub fn empty() -> Self {
|
||||||
|
Self {
|
||||||
|
markdown: String::new(),
|
||||||
|
cost: None,
|
||||||
|
comments: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(repo = %payload.repository.full_name, pr = payload.pull_request.number)
|
||||||
|
)]
|
||||||
|
pub async fn exec_review(
|
||||||
|
ctx: &ReviewContext<'_>,
|
||||||
|
tools: Vec<Tool>,
|
||||||
|
payload: &ReviewPayload,
|
||||||
|
) -> anyhow::Result<ReviewOutcome> {
|
||||||
|
info!(action = %payload.action, "Starting review");
|
||||||
|
|
||||||
|
let full_name = &payload.repository.full_name;
|
||||||
|
let index = payload.pull_request.number;
|
||||||
|
|
||||||
|
let progress_comment = ctx
|
||||||
|
.gitea
|
||||||
|
.comment(
|
||||||
|
&BOT_PROCESS_MSG.replace("{model}", ctx.open_router.model()),
|
||||||
|
full_name,
|
||||||
|
index,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let review = review_pull_request(ctx, tools, payload).await;
|
||||||
|
|
||||||
|
// The progress comment is housekeeping: failing to update it must neither turn a
|
||||||
|
// posted review into a failure nor hide the error that made the review fail.
|
||||||
|
match review {
|
||||||
|
Ok((review_result, markdown)) => {
|
||||||
|
if let Err(err) = ctx
|
||||||
|
.gitea
|
||||||
|
.delete_comment(full_name, progress_comment.id)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
warn!(%err, "Failed to delete the progress comment");
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(ReviewOutcome {
|
||||||
|
markdown,
|
||||||
|
cost: review_result.cost,
|
||||||
|
comments: review_result.reviews.len() as u64,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
if let Err(edit_err) = ctx
|
||||||
|
.gitea
|
||||||
|
.edit_comment(
|
||||||
|
&format!("Error while reviewing: {err}"),
|
||||||
|
full_name,
|
||||||
|
progress_comment.id,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
warn!(err = %edit_err, "Failed to report the error on the progress comment");
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Runs the review and posts it, returning the result and the markdown that was
|
||||||
|
/// posted.
|
||||||
|
async fn review_pull_request(
|
||||||
|
ctx: &ReviewContext<'_>,
|
||||||
|
tools: Vec<Tool>,
|
||||||
|
payload: &ReviewPayload,
|
||||||
|
) -> anyhow::Result<(ReviewResult, String)> {
|
||||||
|
let full_name = &payload.repository.full_name;
|
||||||
|
let index = payload.pull_request.number;
|
||||||
|
|
||||||
|
let git_diff = ctx.gitea.pull_request_diff(full_name, index).await?;
|
||||||
|
|
||||||
|
// The file list only refines the paths and describes the changes: a failure is
|
||||||
|
// not fatal, the diff is the source of truth for the lines.
|
||||||
|
let files = match ctx.gitea.pull_request_files(full_name, index).await {
|
||||||
|
Ok(files) => files,
|
||||||
|
Err(err) => {
|
||||||
|
warn!(%err, "Failed to list the pull request files");
|
||||||
|
Vec::new()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut changed_lines = parse_changed_lines(&git_diff);
|
||||||
|
drop_generated_files(&mut changed_lines);
|
||||||
|
resolve_filenames(&mut changed_lines, &files);
|
||||||
|
|
||||||
|
let changes = format_changes(&files, &changed_lines);
|
||||||
|
|
||||||
|
let bot_request = REVIEW_PROMPT
|
||||||
|
.replace("{subject}", &payload.pull_request.title)
|
||||||
|
.replace("{comment}", &payload.comment.body)
|
||||||
|
.replace("{changes}", &changes);
|
||||||
|
|
||||||
|
let (mut review_result, cost) = run_sandboxed_review(ctx, tools, payload, &bot_request).await?;
|
||||||
|
|
||||||
|
resolve_review_sides(&mut review_result, &changed_lines);
|
||||||
|
|
||||||
|
review_result.cost = cost;
|
||||||
|
if let Some(cost) = review_result.cost {
|
||||||
|
metrics::openrouter_cost_usd(cost);
|
||||||
|
}
|
||||||
|
|
||||||
|
let markdown = review_result_to_markdown(&review_result);
|
||||||
|
|
||||||
|
ctx.gitea
|
||||||
|
.post_pull_request_review(&review_result, &markdown, full_name, index)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok((review_result, markdown))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_sandboxed_review(
|
||||||
|
ctx: &ReviewContext<'_>,
|
||||||
|
tools: Vec<Tool>,
|
||||||
|
payload: &ReviewPayload,
|
||||||
|
bot_request: &str,
|
||||||
|
) -> anyhow::Result<(ReviewResult, Option<f64>)> {
|
||||||
|
let sandbox = Sandbox::create(
|
||||||
|
&ctx.sandbox.runtime,
|
||||||
|
&payload.repository.clone_url,
|
||||||
|
ctx.gitea.token(),
|
||||||
|
payload.pull_request.number,
|
||||||
|
ctx.reporter,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let result = agent::run(
|
||||||
|
ctx.open_router,
|
||||||
|
&sandbox,
|
||||||
|
tools,
|
||||||
|
SANDBOX_SYSTEM_PROMPT,
|
||||||
|
bot_request,
|
||||||
|
ctx.sandbox.max_iterations,
|
||||||
|
ctx.reporter,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
if let Err(err) = sandbox.cleanup().await {
|
||||||
|
warn!(%err, "Failed to clean up sandbox container");
|
||||||
|
}
|
||||||
|
|
||||||
|
let result = result?;
|
||||||
|
info!(
|
||||||
|
iterations = result.iterations,
|
||||||
|
cost = ?result.cost,
|
||||||
|
"Sandboxed review finished"
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok((result.answer, result.cost))
|
||||||
|
}
|
||||||
@@ -0,0 +1,125 @@
|
|||||||
|
//! Rendering of a pull request's changes for the model.
|
||||||
|
|
||||||
|
use super::diff::{ChangedLines, is_ignored};
|
||||||
|
use crate::gitea::PullRequestFile;
|
||||||
|
|
||||||
|
/// Renders the changes for the model: the files the pull request touches, then
|
||||||
|
/// the lines to review per file.
|
||||||
|
pub(super) fn format_changes(files: &[PullRequestFile], changed_lines: &ChangedLines) -> String {
|
||||||
|
let mut sections = Vec::new();
|
||||||
|
|
||||||
|
let described = files
|
||||||
|
.iter()
|
||||||
|
.filter(|file| !is_ignored(&file.filename))
|
||||||
|
.map(describe_file)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ");
|
||||||
|
|
||||||
|
if !described.is_empty() {
|
||||||
|
sections.push(format!("Files changed by the pull request: {described}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
sections.push(format!(
|
||||||
|
"Lines to review, per file:\n{}",
|
||||||
|
format_changed_lines(changed_lines)
|
||||||
|
));
|
||||||
|
|
||||||
|
sections.join("\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Describes a changed file for the model, including how it changed.
|
||||||
|
fn describe_file(file: &PullRequestFile) -> String {
|
||||||
|
match &file.previous_filename {
|
||||||
|
Some(previous) => format!("{} ({} from {})", file.filename, file.status, previous),
|
||||||
|
None => format!("{} ({})", file.filename, file.status),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Renders the changed lines as `filename: added 1, 2 / removed 3`, one file per
|
||||||
|
/// line, keeping only the sides the file actually has.
|
||||||
|
pub(super) fn format_changed_lines(changed_lines: &ChangedLines) -> String {
|
||||||
|
changed_lines
|
||||||
|
.iter()
|
||||||
|
.map(|file| {
|
||||||
|
let mut sides = Vec::new();
|
||||||
|
|
||||||
|
for (label, lines) in [("added", &file.added), ("removed", &file.removed)] {
|
||||||
|
if !lines.is_empty() {
|
||||||
|
sides.push(format!("{label} {}", format_line_numbers(lines)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
format!("{}: {}", file.filename, sides.join(" / "))
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn format_line_numbers(lines: &[u64]) -> String {
|
||||||
|
lines
|
||||||
|
.iter()
|
||||||
|
.map(u64::to_string)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::bot_actions::review::{diff::parse_changed_lines, fixtures::*};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn changed_lines_are_listed_per_file_and_side() {
|
||||||
|
let expected = concat!(
|
||||||
|
"src/foo.rs: added 2, 4, 5 / removed 2\n",
|
||||||
|
"src/bar.rs: added 11, 13"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(format_changed_lines(&parse_changed_lines(DIFF)), expected);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn changes_describe_the_files_then_the_lines() {
|
||||||
|
let changed_lines = parse_changed_lines(DELETION_ONLY);
|
||||||
|
let files = vec![pull_request_file("src/old.rs", None, "deleted")];
|
||||||
|
|
||||||
|
let expected = concat!(
|
||||||
|
"Files changed by the pull request: src/old.rs (deleted)\n",
|
||||||
|
"\n",
|
||||||
|
"Lines to review, per file:\n",
|
||||||
|
"src/old.rs: removed 1, 2, 3"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(format_changes(&files, &changed_lines), expected);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn changes_without_the_api_list_only_hold_the_lines() {
|
||||||
|
let changed_lines = parse_changed_lines(DELETION_ONLY);
|
||||||
|
|
||||||
|
let expected = "Lines to review, per file:\nsrc/old.rs: removed 1, 2, 3";
|
||||||
|
|
||||||
|
assert_eq!(format_changes(&[], &changed_lines), expected);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_generated_file_is_not_described_to_the_model() {
|
||||||
|
let files = vec![
|
||||||
|
pull_request_file("Cargo.lock", None, "modified"),
|
||||||
|
pull_request_file("src/foo.rs", None, "modified"),
|
||||||
|
];
|
||||||
|
let changed_lines = parse_changed_lines(DIFF);
|
||||||
|
|
||||||
|
let changes = format_changes(&files, &changed_lines);
|
||||||
|
|
||||||
|
assert!(!changes.contains("Cargo.lock"));
|
||||||
|
assert!(changes.contains("src/foo.rs (modified)"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_renamed_file_is_described_with_its_previous_path() {
|
||||||
|
let file = pull_request_file("src/new.rs", Some("src/old.rs"), "renamed");
|
||||||
|
|
||||||
|
assert_eq!(describe_file(&file), "src/new.rs (renamed from src/old.rs)");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
//! Markdown summary of a review, as posted on the pull request.
|
||||||
|
|
||||||
|
use crate::domain::review::{ReviewItem, ReviewResult, ReviewSeverity};
|
||||||
|
|
||||||
|
pub(super) fn review_result_to_markdown(review_result: &ReviewResult) -> String {
|
||||||
|
if review_result.reviews.is_empty() {
|
||||||
|
return String::from("No issues found. ✅");
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut md = String::from("## Review Feedback\n\n");
|
||||||
|
|
||||||
|
md.push_str(&format!(
|
||||||
|
"### {} issues found.\n\n",
|
||||||
|
review_result.reviews.len()
|
||||||
|
));
|
||||||
|
|
||||||
|
let breakdown = severity_breakdown(&review_result.reviews);
|
||||||
|
if !breakdown.is_empty() {
|
||||||
|
md.push_str(&breakdown);
|
||||||
|
md.push('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
if !review_result.comment.is_empty() {
|
||||||
|
md.push_str("\n---\n\n");
|
||||||
|
md.push_str("### Summary\n\n");
|
||||||
|
md.push_str(&review_result.comment);
|
||||||
|
md.push('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(cost) = review_result.cost {
|
||||||
|
md.push_str("\n---\n\n");
|
||||||
|
md.push_str(&format!("### Cost: ${}", cost));
|
||||||
|
md.push('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
md
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Counts the reviews per severity, most severe first.
|
||||||
|
///
|
||||||
|
/// A review whose severity the model omitted or wrote unreadably is counted as
|
||||||
|
/// [`ReviewSeverity::Maintainability`], the same fallback used when posting it.
|
||||||
|
fn severity_breakdown(reviews: &[ReviewItem]) -> String {
|
||||||
|
[
|
||||||
|
ReviewSeverity::Bug,
|
||||||
|
ReviewSeverity::Security,
|
||||||
|
ReviewSeverity::Performance,
|
||||||
|
ReviewSeverity::Maintainability,
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|severity| {
|
||||||
|
let count = reviews
|
||||||
|
.iter()
|
||||||
|
.filter(|review| review.severity.unwrap_or(ReviewSeverity::Maintainability) == severity)
|
||||||
|
.count();
|
||||||
|
|
||||||
|
(count > 0).then(|| format!("- {} {}", count, severity.label()))
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::bot_actions::review::fixtures::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_breakdown_counts_severities_most_severe_first() {
|
||||||
|
let reviews = vec![
|
||||||
|
ReviewItem {
|
||||||
|
severity: Some(ReviewSeverity::Maintainability),
|
||||||
|
..review("a", Some(1), None)
|
||||||
|
},
|
||||||
|
ReviewItem {
|
||||||
|
severity: Some(ReviewSeverity::Bug),
|
||||||
|
..review("b", Some(2), None)
|
||||||
|
},
|
||||||
|
ReviewItem {
|
||||||
|
severity: Some(ReviewSeverity::Bug),
|
||||||
|
..review("c", Some(3), None)
|
||||||
|
},
|
||||||
|
// A severity the model left out counts as maintainability.
|
||||||
|
review("d", Some(4), None),
|
||||||
|
];
|
||||||
|
|
||||||
|
assert_eq!(severity_breakdown(&reviews), "- 2 bug\n- 2 maintainability");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_markdown_breaks_the_issues_down_by_severity() {
|
||||||
|
let mut result = review_result(vec![
|
||||||
|
ReviewItem {
|
||||||
|
severity: Some(ReviewSeverity::Bug),
|
||||||
|
..review("a", Some(1), None)
|
||||||
|
},
|
||||||
|
ReviewItem {
|
||||||
|
severity: Some(ReviewSeverity::Performance),
|
||||||
|
..review("b", Some(2), None)
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
result.comment = String::from("Le découpage en crates est propre.");
|
||||||
|
|
||||||
|
let markdown = review_result_to_markdown(&result);
|
||||||
|
|
||||||
|
assert!(markdown.contains("- 1 bug"));
|
||||||
|
assert!(markdown.contains("- 1 performance"));
|
||||||
|
assert!(markdown.contains("Le découpage en crates est propre."));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,7 +1,5 @@
|
|||||||
use toasty::Db;
|
use toasty::Db;
|
||||||
|
|
||||||
use crate::schemas::{Job, JobStatus};
|
|
||||||
|
|
||||||
pub async fn open(url: &str) -> toasty::Result<Db> {
|
pub async fn open(url: &str) -> toasty::Result<Db> {
|
||||||
Db::builder()
|
Db::builder()
|
||||||
.models(toasty::models!(crate::*))
|
.models(toasty::models!(crate::*))
|
||||||
@@ -9,35 +7,45 @@ pub async fn open(url: &str) -> toasty::Result<Db> {
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn connect(url: &str) -> toasty::Result<Db> {
|
/// A connection for a test, on the database `DATABASE_URL` points to.
|
||||||
let mut db = open(url).await?;
|
///
|
||||||
|
/// Every `#[tokio::test]` runs on its own runtime, and the connection tasks live on
|
||||||
Job::update_by_status(JobStatus::Processing)
|
/// the runtime that opened the pool: sharing one `Db` between tests would leave
|
||||||
.status(JobStatus::Pending)
|
/// them with a pool whose runtime is gone. Each test therefore opens its own, and
|
||||||
.exec(&mut db)
|
/// only the migrations are applied once.
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(db)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) async fn test_connect() -> Db {
|
pub(crate) async fn test_connect() -> Db {
|
||||||
static DB: tokio::sync::OnceCell<Db> = tokio::sync::OnceCell::const_new();
|
static MIGRATED: tokio::sync::OnceCell<()> = tokio::sync::OnceCell::const_new();
|
||||||
|
|
||||||
DB.get_or_init(|| async {
|
|
||||||
let url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set");
|
let url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set");
|
||||||
|
|
||||||
let migrations = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("toasty");
|
// The migrations run on a runtime of their own, which outlives the first test.
|
||||||
|
MIGRATED
|
||||||
|
.get_or_init(|| async {
|
||||||
|
let url = url.clone();
|
||||||
|
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
tokio::runtime::Runtime::new()
|
||||||
|
.expect("migration runtime")
|
||||||
|
.block_on(async {
|
||||||
|
let migrations =
|
||||||
|
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("toasty");
|
||||||
let config = toasty_cli::Config::new()
|
let config = toasty_cli::Config::new()
|
||||||
.migration(toasty_cli::MigrationConfig::new().path(migrations));
|
.migration(toasty_cli::MigrationConfig::new().path(migrations));
|
||||||
|
|
||||||
toasty_cli::ToastyCli::with_config(open(&url).await.expect("open pool"), config)
|
toasty_cli::ToastyCli::with_config(
|
||||||
|
open(&url).await.expect("open pool"),
|
||||||
|
config,
|
||||||
|
)
|
||||||
.parse_from(["toasty", "migration", "apply"])
|
.parse_from(["toasty", "migration", "apply"])
|
||||||
.await
|
.await
|
||||||
.expect("apply migrations");
|
.expect("apply migrations");
|
||||||
|
});
|
||||||
connect(&url).await.expect("connect to postgres")
|
|
||||||
})
|
})
|
||||||
.await
|
.join()
|
||||||
.clone()
|
.expect("migrations panicked");
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
|
||||||
|
open(&url).await.expect("connect to postgres")
|
||||||
}
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
pub mod review;
|
||||||
@@ -0,0 +1,247 @@
|
|||||||
|
//! Domain types of a review: what the model answers with, and how it is read.
|
||||||
|
|
||||||
|
use std::str::FromStr;
|
||||||
|
|
||||||
|
use serde::{Deserialize, Deserializer};
|
||||||
|
use tracing::warn;
|
||||||
|
|
||||||
|
use crate::text::excerpt;
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
pub struct ReviewResult {
|
||||||
|
pub reviews: Vec<ReviewItem>,
|
||||||
|
pub comment: String,
|
||||||
|
pub cost: Option<f64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
pub struct ReviewItem {
|
||||||
|
pub filename: String,
|
||||||
|
pub line: Option<u64>,
|
||||||
|
#[serde(default, deserialize_with = "deserialize_side")]
|
||||||
|
pub side: Option<ReviewSide>,
|
||||||
|
#[serde(default, deserialize_with = "deserialize_severity")]
|
||||||
|
pub severity: Option<ReviewSeverity>,
|
||||||
|
pub message: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Which version of the file a review comment is anchored on.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum ReviewSide {
|
||||||
|
/// The line was added by the pull request: `line` is a line number of the
|
||||||
|
/// new version of the file.
|
||||||
|
Added,
|
||||||
|
/// The line was removed by the pull request: `line` is a line number of the
|
||||||
|
/// old version of the file.
|
||||||
|
Removed,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ReviewSide {
|
||||||
|
/// Reads the side the model asked for, tolerating casing and synonyms.
|
||||||
|
fn parse(raw: &str) -> Option<Self> {
|
||||||
|
match raw.trim().to_ascii_lowercase().as_str() {
|
||||||
|
"added" | "add" | "new" | "right" => Some(Self::Added),
|
||||||
|
"removed" | "remove" | "deleted" | "delete" | "old" | "left" => Some(Self::Removed),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What kind of problem a review reports.
|
||||||
|
///
|
||||||
|
/// The categories are the ones the prompt asks for; a review whose severity is
|
||||||
|
/// missing or unreadable falls back to [`ReviewSeverity::Maintainability`], the
|
||||||
|
/// least alarming bucket, instead of failing the whole review.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum ReviewSeverity {
|
||||||
|
/// Wrong behaviour.
|
||||||
|
Bug,
|
||||||
|
/// A vulnerability.
|
||||||
|
Security,
|
||||||
|
/// A resource problem.
|
||||||
|
Performance,
|
||||||
|
/// Everything else: readability, structure, naming, tests.
|
||||||
|
Maintainability,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ReviewSeverity {
|
||||||
|
/// Reads the severity the model asked for, tolerating casing and synonyms.
|
||||||
|
fn parse(raw: &str) -> Option<Self> {
|
||||||
|
match raw.trim().to_ascii_lowercase().as_str() {
|
||||||
|
"bug" | "bugs" | "correctness" | "error" => Some(Self::Bug),
|
||||||
|
"security" | "vulnerability" => Some(Self::Security),
|
||||||
|
"performance" | "perf" => Some(Self::Performance),
|
||||||
|
"maintainability" | "quality" | "style" => Some(Self::Maintainability),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Lower-case label used in the review markdown.
|
||||||
|
pub fn label(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Bug => "bug",
|
||||||
|
Self::Security => "security",
|
||||||
|
Self::Performance => "performance",
|
||||||
|
Self::Maintainability => "maintainability",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reads the side the model asked for. An unreadable value is ignored instead of
|
||||||
|
/// failing the whole review: the side is then resolved from the changed lines.
|
||||||
|
fn deserialize_side<'de, D>(deserializer: D) -> Result<Option<ReviewSide>, D::Error>
|
||||||
|
where
|
||||||
|
D: Deserializer<'de>,
|
||||||
|
{
|
||||||
|
let raw = Option::<serde_json::Value>::deserialize(deserializer)?;
|
||||||
|
|
||||||
|
Ok(raw
|
||||||
|
.as_ref()
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.and_then(ReviewSide::parse))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reads the severity the model asked for, ignoring an unreadable value: the
|
||||||
|
/// review is then reported as [`ReviewSeverity::Maintainability`].
|
||||||
|
fn deserialize_severity<'de, D>(deserializer: D) -> Result<Option<ReviewSeverity>, D::Error>
|
||||||
|
where
|
||||||
|
D: Deserializer<'de>,
|
||||||
|
{
|
||||||
|
let raw = Option::<serde_json::Value>::deserialize(deserializer)?;
|
||||||
|
|
||||||
|
Ok(raw
|
||||||
|
.as_ref()
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.and_then(ReviewSeverity::parse))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Number of characters of a model answer kept in the logs when it cannot be
|
||||||
|
/// parsed.
|
||||||
|
const MAX_LOGGED_ANSWER: usize = 500;
|
||||||
|
|
||||||
|
impl FromStr for ReviewResult {
|
||||||
|
type Err = anyhow::Error;
|
||||||
|
|
||||||
|
/// Parses the review the model answered with.
|
||||||
|
///
|
||||||
|
/// This is the contract the agent loop enforces: a rejected answer is sent
|
||||||
|
/// back to the model, with the reason, so that it can correct itself.
|
||||||
|
///
|
||||||
|
/// The contract is a raw JSON object, but models sometimes wrap it in a
|
||||||
|
/// markdown code fence or surround it with a sentence: the object is then
|
||||||
|
/// extracted from the answer before failing, and the answer is logged so a
|
||||||
|
/// breach of the contract can be diagnosed.
|
||||||
|
fn from_str(message: &str) -> Result<Self, Self::Err> {
|
||||||
|
let error = match serde_json::from_str::<Self>(message) {
|
||||||
|
Ok(review_result) => return Ok(review_result),
|
||||||
|
Err(error) => error,
|
||||||
|
};
|
||||||
|
|
||||||
|
// A markdown code fence or a sentence around the object is tolerated, with
|
||||||
|
// a warning: the contract asks for a raw JSON object.
|
||||||
|
if let Some(json) = json_object(message)
|
||||||
|
&& let Ok(review_result) = serde_json::from_str::<Self>(json)
|
||||||
|
{
|
||||||
|
warn!(
|
||||||
|
"Model answer is not a raw JSON object, it was extracted from the surrounding text"
|
||||||
|
);
|
||||||
|
|
||||||
|
return Ok(review_result);
|
||||||
|
}
|
||||||
|
|
||||||
|
// The reason of the rejection is logged along with the answer: without it,
|
||||||
|
// a broken answer is impossible to diagnose.
|
||||||
|
warn!(
|
||||||
|
answer = %excerpt(message, MAX_LOGGED_ANSWER),
|
||||||
|
reason = %error,
|
||||||
|
"Model answer is not the expected JSON"
|
||||||
|
);
|
||||||
|
|
||||||
|
anyhow::bail!("the answer is not valid JSON: {error}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the outermost `{...}` of an answer, which ignores a markdown code
|
||||||
|
/// fence or any text around it.
|
||||||
|
fn json_object(message: &str) -> Option<&str> {
|
||||||
|
let start = message.find('{')?;
|
||||||
|
let end = message.rfind('}')?;
|
||||||
|
|
||||||
|
(start < end).then(|| &message[start..=end])
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_raw_answer_is_parsed() {
|
||||||
|
let answer = r#"{"reviews": [], "comment": "ok"}"#;
|
||||||
|
|
||||||
|
assert_eq!(answer.parse::<ReviewResult>().unwrap().comment, "ok");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_fenced_answer_is_extracted() {
|
||||||
|
let answer = concat!(
|
||||||
|
"Voici ma review :\n",
|
||||||
|
"```json\n",
|
||||||
|
"{\"reviews\": [], \"comment\": \"rien à signaler\"}\n",
|
||||||
|
"```\n"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
answer.parse::<ReviewResult>().unwrap().comment,
|
||||||
|
"rien à signaler"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_sentence_around_the_object_is_ignored() {
|
||||||
|
let answer = r#"Rien à signaler. {"reviews": [], "comment": "ok"} Bonne journée !"#;
|
||||||
|
|
||||||
|
assert_eq!(answer.parse::<ReviewResult>().unwrap().comment, "ok");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_answer_without_json_is_rejected() {
|
||||||
|
assert!("Je n'ai rien relevé.".parse::<ReviewResult>().is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_answer_that_is_an_object_but_not_a_review_is_rejected() {
|
||||||
|
assert!(r#"{"message": "LGTM"}"#.parse::<ReviewResult>().is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn severities_are_read_from_the_model() {
|
||||||
|
let result: ReviewResult = serde_json::from_str(
|
||||||
|
r#"{
|
||||||
|
"reviews": [
|
||||||
|
{ "filename": "a", "line": 1, "severity": "Bug", "message": "x" },
|
||||||
|
{ "filename": "b", "line": 2, "severity": "SECURITY", "message": "x" },
|
||||||
|
{ "filename": "c", "line": 3, "severity": "perf", "message": "x" },
|
||||||
|
{ "filename": "d", "line": 4, "severity": "banana", "message": "x" }
|
||||||
|
],
|
||||||
|
"comment": ""
|
||||||
|
}"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let severities = result
|
||||||
|
.reviews
|
||||||
|
.iter()
|
||||||
|
.map(|review| review.severity)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
severities,
|
||||||
|
vec![
|
||||||
|
Some(ReviewSeverity::Bug),
|
||||||
|
Some(ReviewSeverity::Security),
|
||||||
|
Some(ReviewSeverity::Performance),
|
||||||
|
None,
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,4 +1,9 @@
|
|||||||
use anyhow::anyhow;
|
use std::{str::FromStr, time::Duration};
|
||||||
|
|
||||||
|
use anyhow::Context;
|
||||||
|
|
||||||
|
/// Number of tool-calling iterations a sandboxed review may take, unless configured.
|
||||||
|
const DEFAULT_SANDBOX_MAX_ITERATIONS: usize = 8;
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct EnvConfig {
|
pub struct EnvConfig {
|
||||||
@@ -6,46 +11,31 @@ pub struct EnvConfig {
|
|||||||
pub webhook_secret: String,
|
pub webhook_secret: String,
|
||||||
pub open_router_api_key: String,
|
pub open_router_api_key: String,
|
||||||
pub open_router_model: String,
|
pub open_router_model: String,
|
||||||
pub open_router_timeout: u64,
|
pub open_router_timeout: Duration,
|
||||||
pub bot_max_concurrent: usize,
|
pub bot_max_concurrent: usize,
|
||||||
pub gitea_url: String,
|
pub gitea_url: String,
|
||||||
pub gitea_token: String,
|
pub gitea_token: String,
|
||||||
pub gitea_timeout: u64,
|
pub gitea_timeout: Duration,
|
||||||
pub database_url: String,
|
pub database_url: String,
|
||||||
pub metrics_bind_addr: Option<String>,
|
pub metrics_bind_addr: Option<String>,
|
||||||
pub sandbox_max_iterations: usize,
|
pub sandbox_max_iterations: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn load_config() -> anyhow::Result<EnvConfig> {
|
pub fn load_config() -> anyhow::Result<EnvConfig> {
|
||||||
let http_port = try_get_env("HTTP_PORT")?.parse()?;
|
|
||||||
let webhook_secret = try_get_env("WEBHOOK_SIG_HEADER_SECRET")?;
|
|
||||||
let open_router_api_key = try_get_env("OPEN_ROUTER_API_KEY")?;
|
|
||||||
let open_router_model = try_get_env("OPEN_ROUTER_MODEL")?;
|
|
||||||
let open_router_timeout = try_get_env("OPEN_ROUTER_TIMEOUT")?.parse()?;
|
|
||||||
let bot_max_concurrent = try_get_env("BOT_MAX_CONCURRENT")?.parse()?;
|
|
||||||
let gitea_url = try_get_env("GITEA_URL")?;
|
|
||||||
let gitea_token = try_get_env("GITEA_TOKEN")?;
|
|
||||||
let gitea_timeout = try_get_env("GITEA_TIMEOUT")?.parse()?;
|
|
||||||
let database_url = try_get_env("DATABASE_URL")?;
|
|
||||||
let metrics_bind_addr = std::env::var("METRICS_BIND_ADDR").ok();
|
|
||||||
let sandbox_max_iterations = std::env::var("SANDBOX_MAX_ITERATIONS")
|
|
||||||
.ok()
|
|
||||||
.and_then(|value| value.parse().ok())
|
|
||||||
.unwrap_or(8);
|
|
||||||
|
|
||||||
Ok(EnvConfig {
|
Ok(EnvConfig {
|
||||||
http_port,
|
http_port: parse_env("HTTP_PORT")?,
|
||||||
webhook_secret,
|
webhook_secret: try_get_env("WEBHOOK_SIG_HEADER_SECRET")?,
|
||||||
open_router_api_key,
|
open_router_api_key: try_get_env("OPEN_ROUTER_API_KEY")?,
|
||||||
open_router_model,
|
open_router_model: try_get_env("OPEN_ROUTER_MODEL")?,
|
||||||
open_router_timeout,
|
open_router_timeout: parse_seconds_env("OPEN_ROUTER_TIMEOUT")?,
|
||||||
bot_max_concurrent,
|
bot_max_concurrent: parse_env("BOT_MAX_CONCURRENT")?,
|
||||||
gitea_url,
|
gitea_url: try_get_env("GITEA_URL")?,
|
||||||
gitea_token,
|
gitea_token: try_get_env("GITEA_TOKEN")?,
|
||||||
gitea_timeout,
|
gitea_timeout: parse_seconds_env("GITEA_TIMEOUT")?,
|
||||||
database_url,
|
database_url: try_get_env("DATABASE_URL")?,
|
||||||
metrics_bind_addr,
|
metrics_bind_addr: std::env::var("METRICS_BIND_ADDR").ok(),
|
||||||
sandbox_max_iterations,
|
sandbox_max_iterations: parse_optional_env("SANDBOX_MAX_ITERATIONS")?
|
||||||
|
.unwrap_or(DEFAULT_SANDBOX_MAX_ITERATIONS),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -53,16 +43,57 @@ pub fn try_get_env(key: &str) -> anyhow::Result<String> {
|
|||||||
let env_value = std::env::var(key).map_err(|e| anyhow::anyhow!("{}: {}", key, e))?;
|
let env_value = std::env::var(key).map_err(|e| anyhow::anyhow!("{}: {}", key, e))?;
|
||||||
|
|
||||||
if env_value.trim().is_empty() {
|
if env_value.trim().is_empty() {
|
||||||
return Err(anyhow!(format!("env var {} is empty", key)));
|
anyhow::bail!("env var {key} is empty");
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(env_value)
|
Ok(env_value)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reads a required variable and parses it, naming the variable when it is invalid.
|
||||||
|
fn parse_env<T>(key: &str) -> anyhow::Result<T>
|
||||||
|
where
|
||||||
|
T: FromStr,
|
||||||
|
T::Err: std::error::Error + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
parse_value(key, &try_get_env(key)?)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reads an optional variable and parses it. A missing or empty variable is `None`;
|
||||||
|
/// one that is set but invalid is an error, like it is for the required ones.
|
||||||
|
fn parse_optional_env<T>(key: &str) -> anyhow::Result<Option<T>>
|
||||||
|
where
|
||||||
|
T: FromStr,
|
||||||
|
T::Err: std::error::Error + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
match std::env::var(key) {
|
||||||
|
Ok(value) if !value.trim().is_empty() => parse_value(key, &value).map(Some),
|
||||||
|
_ => Ok(None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reads a required variable holding a number of seconds.
|
||||||
|
fn parse_seconds_env(key: &str) -> anyhow::Result<Duration> {
|
||||||
|
parse_env::<u64>(key).map(Duration::from_secs)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_value<T>(key: &str, value: &str) -> anyhow::Result<T>
|
||||||
|
where
|
||||||
|
T: FromStr,
|
||||||
|
T::Err: std::error::Error + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
value
|
||||||
|
.trim()
|
||||||
|
.parse()
|
||||||
|
.with_context(|| format!("env var {key} is invalid: `{value}`"))
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
// Every test uses its own variable: the environment is shared by the tests,
|
||||||
|
// which run in parallel.
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_try_get_env_returns_value() {
|
fn test_try_get_env_returns_value() {
|
||||||
unsafe { std::env::set_var("TEST_ENV_PRESENT", "hello") };
|
unsafe { std::env::set_var("TEST_ENV_PRESENT", "hello") };
|
||||||
@@ -88,4 +119,38 @@ mod tests {
|
|||||||
let err = try_get_env("TEST_ENV_WHITESPACE").unwrap_err();
|
let err = try_get_env("TEST_ENV_WHITESPACE").unwrap_err();
|
||||||
assert!(err.to_string().contains("TEST_ENV_WHITESPACE"));
|
assert!(err.to_string().contains("TEST_ENV_WHITESPACE"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_timeout_is_read_as_seconds() {
|
||||||
|
unsafe { std::env::set_var("TEST_ENV_SECONDS", "30") };
|
||||||
|
assert_eq!(
|
||||||
|
parse_seconds_env("TEST_ENV_SECONDS").unwrap(),
|
||||||
|
Duration::from_secs(30)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_invalid_number_names_its_variable() {
|
||||||
|
unsafe { std::env::set_var("TEST_ENV_NOT_A_NUMBER", "soon") };
|
||||||
|
|
||||||
|
let err = parse_env::<u16>("TEST_ENV_NOT_A_NUMBER").unwrap_err();
|
||||||
|
|
||||||
|
assert!(format!("{err:#}").contains("TEST_ENV_NOT_A_NUMBER"));
|
||||||
|
assert!(format!("{err:#}").contains("soon"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_absent_optional_variable_is_none() {
|
||||||
|
unsafe { std::env::remove_var("TEST_ENV_OPTIONAL_ABSENT") };
|
||||||
|
assert_eq!(
|
||||||
|
parse_optional_env::<usize>("TEST_ENV_OPTIONAL_ABSENT").unwrap(),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_invalid_optional_variable_is_an_error_instead_of_the_default() {
|
||||||
|
unsafe { std::env::set_var("TEST_ENV_OPTIONAL_INVALID", "many") };
|
||||||
|
assert!(parse_optional_env::<usize>("TEST_ENV_OPTIONAL_INVALID").is_err());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -11,7 +11,7 @@ const PROGRESS_CAPACITY: usize = 256;
|
|||||||
const HISTORY_CAPACITY: usize = 256;
|
const HISTORY_CAPACITY: usize = 256;
|
||||||
const MAX_EVENT_ARGUMENTS: usize = 200;
|
const MAX_EVENT_ARGUMENTS: usize = 200;
|
||||||
|
|
||||||
/// Évènement de cycle de vie d'un job, diffusé globalement.
|
/// Lifecycle event of a job, broadcast globally.
|
||||||
#[derive(Clone, Debug, Serialize)]
|
#[derive(Clone, Debug, Serialize)]
|
||||||
#[serde(tag = "type", rename_all = "snake_case")]
|
#[serde(tag = "type", rename_all = "snake_case")]
|
||||||
pub enum JobEvent {
|
pub enum JobEvent {
|
||||||
@@ -212,7 +212,7 @@ impl Reporter {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Signale la fin d'un appel d'outil.
|
/// Reports the end of a tool call.
|
||||||
pub fn tool_result(&self, iteration: usize, name: &str, truncated: bool, output: &str) {
|
pub fn tool_result(&self, iteration: usize, name: &str, truncated: bool, output: &str) {
|
||||||
self.channel.emit(Progress::ToolResult {
|
self.channel.emit(Progress::ToolResult {
|
||||||
iteration,
|
iteration,
|
||||||
|
|||||||
@@ -8,8 +8,8 @@ use tokio_util::io::StreamReader;
|
|||||||
use tracing::{instrument, warn};
|
use tracing::{instrument, warn};
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
bot::{ReviewResult, ReviewSeverity, ReviewSide},
|
|
||||||
consts::MAX_DIFF_SIZE,
|
consts::MAX_DIFF_SIZE,
|
||||||
|
domain::review::{ReviewResult, ReviewSeverity, ReviewSide},
|
||||||
errors::AppError,
|
errors::AppError,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -27,7 +27,7 @@ pub struct GiteaAPI {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl GiteaAPI {
|
impl GiteaAPI {
|
||||||
pub fn new(base_url: &str, token: &str, timeout: u64) -> anyhow::Result<Self> {
|
pub fn new(base_url: &str, token: &str, timeout: Duration) -> anyhow::Result<Self> {
|
||||||
let mut default_headers = reqwest::header::HeaderMap::new();
|
let mut default_headers = reqwest::header::HeaderMap::new();
|
||||||
default_headers.insert(
|
default_headers.insert(
|
||||||
reqwest::header::HeaderName::from_static("authorization"),
|
reqwest::header::HeaderName::from_static("authorization"),
|
||||||
@@ -38,7 +38,7 @@ impl GiteaAPI {
|
|||||||
base_url: String::from(base_url),
|
base_url: String::from(base_url),
|
||||||
token: String::from(token),
|
token: String::from(token),
|
||||||
client: reqwest::Client::builder()
|
client: reqwest::Client::builder()
|
||||||
.timeout(Duration::from_secs(timeout))
|
.timeout(timeout)
|
||||||
.default_headers(default_headers)
|
.default_headers(default_headers)
|
||||||
.build()?,
|
.build()?,
|
||||||
})
|
})
|
||||||
@@ -320,6 +320,21 @@ pub struct ReviewPayload {
|
|||||||
pub comment: Comment,
|
pub comment: Comment,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl ReviewPayload {
|
||||||
|
/// Checks that the comment is a new one that mentions the bot.
|
||||||
|
fn check_addressed_to(&self, bot_name: &str) -> Result<(), AppError> {
|
||||||
|
if self.action != "created" {
|
||||||
|
return Err(AppError::InvalidActionErr);
|
||||||
|
}
|
||||||
|
|
||||||
|
if !self.comment.body.starts_with(&format!("@{bot_name}")) {
|
||||||
|
return Err(AppError::UnauthorizedUserErr);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Deserialize, Serialize, Debug, Clone)]
|
#[derive(Deserialize, Serialize, Debug, Clone)]
|
||||||
pub struct PullRequest {
|
pub struct PullRequest {
|
||||||
pub number: u64,
|
pub number: u64,
|
||||||
@@ -357,28 +372,15 @@ pub struct PullRequestFile {
|
|||||||
|
|
||||||
impl WebhookType {
|
impl WebhookType {
|
||||||
pub fn from_event(event: &str, bot_name: &str, json: Value) -> Result<Self, AppError> {
|
pub fn from_event(event: &str, bot_name: &str, json: Value) -> Result<Self, AppError> {
|
||||||
let wb = match event {
|
match event {
|
||||||
"pull_request_comment" => Ok(WebhookType::Review(serde_json::from_value(json)?)),
|
"pull_request_comment" => {
|
||||||
|
let payload: ReviewPayload = serde_json::from_value(json)?;
|
||||||
|
payload.check_addressed_to(bot_name)?;
|
||||||
|
|
||||||
|
Ok(WebhookType::Review(payload))
|
||||||
|
}
|
||||||
_ => Err(AppError::UnknownEventErr),
|
_ => Err(AppError::UnknownEventErr),
|
||||||
}?;
|
|
||||||
|
|
||||||
let action = match &wb {
|
|
||||||
WebhookType::Review(review_payload) => &review_payload.action,
|
|
||||||
};
|
|
||||||
|
|
||||||
if action != "created" {
|
|
||||||
return Err(AppError::InvalidActionErr);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let pr_body = match &wb {
|
|
||||||
WebhookType::Review(review_payload) => &review_payload.comment.body,
|
|
||||||
};
|
|
||||||
|
|
||||||
if !pr_body.starts_with(&format!("@{}", bot_name)) {
|
|
||||||
return Err(AppError::UnauthorizedUserErr);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(wb)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,26 +1,30 @@
|
|||||||
use std::{collections::VecDeque, time::Duration};
|
use std::sync::Arc;
|
||||||
|
|
||||||
use futures_util::stream::{self, Stream};
|
use toasty::{Db, stmt::Value};
|
||||||
use toasty::Db;
|
use tokio::sync::Notify;
|
||||||
use tracing::error;
|
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
gitea::WebhookType,
|
gitea::WebhookType,
|
||||||
schemas::{Job, JobStatus},
|
schemas::{Job, JobStatus},
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Délai entre deux interrogations de la table quand la file est vide.
|
/// The reviews to do, stored in PostgreSQL.
|
||||||
const POLL_INTERVAL: Duration = Duration::from_secs(1);
|
///
|
||||||
|
/// Steep runs as a single instance, so every job is pushed by this process: the
|
||||||
/// File des reviews à faire.
|
/// worker is woken up by [`JobQueue::push`] instead of polling the table. The table
|
||||||
|
/// only makes the queue survive a restart.
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct JobQueue {
|
pub struct JobQueue {
|
||||||
db: Db,
|
db: Db,
|
||||||
|
work: Arc<Notify>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl JobQueue {
|
impl JobQueue {
|
||||||
pub fn new(db: Db) -> Self {
|
pub fn new(db: Db) -> Self {
|
||||||
Self { db }
|
Self {
|
||||||
|
db,
|
||||||
|
work: Arc::new(Notify::new()),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn push(&self, webhook: WebhookType) -> toasty::Result<Job> {
|
pub async fn push(&self, webhook: WebhookType) -> toasty::Result<Job> {
|
||||||
@@ -40,54 +44,76 @@ impl JobQueue {
|
|||||||
.exec(&mut db)
|
.exec(&mut db)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
|
// `notify_one` keeps a permit when nobody is waiting yet, so a job pushed
|
||||||
|
// while the worker is busy is not missed.
|
||||||
|
self.work.notify_one();
|
||||||
|
|
||||||
Ok(job)
|
Ok(job)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn pending(&self) -> impl Stream<Item = Job> {
|
/// Whether a job is already queued for this event.
|
||||||
let db = self.db.clone();
|
pub async fn is_queued(&self, event_id: u64) -> toasty::Result<bool> {
|
||||||
|
|
||||||
stream::unfold(VecDeque::new(), move |mut buffer| {
|
|
||||||
let db = db.clone();
|
|
||||||
|
|
||||||
async move {
|
|
||||||
let mut db = db;
|
|
||||||
|
|
||||||
loop {
|
|
||||||
if let Some(job) = buffer.pop_front() {
|
|
||||||
return Some((job, buffer));
|
|
||||||
}
|
|
||||||
|
|
||||||
match Job::filter_by_status(JobStatus::Pending)
|
|
||||||
.exec(&mut db)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(jobs) if jobs.is_empty() => {
|
|
||||||
tokio::time::sleep(POLL_INTERVAL).await;
|
|
||||||
}
|
|
||||||
Ok(jobs) => buffer.extend(jobs),
|
|
||||||
Err(err) => {
|
|
||||||
error!(%err, "failed to poll pending jobs");
|
|
||||||
tokio::time::sleep(POLL_INTERVAL).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Marque un job comme en cours de traitement.
|
|
||||||
pub async fn mark_processing(&self, id: u64) -> toasty::Result<()> {
|
|
||||||
let mut db = self.db.clone();
|
let mut db = self.db.clone();
|
||||||
|
|
||||||
Job::update_by_id(id)
|
Ok(Job::filter_by_event_id(event_id)
|
||||||
.status(JobStatus::Processing)
|
.first()
|
||||||
|
.exec(&mut db)
|
||||||
|
.await?
|
||||||
|
.is_some())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Puts back in the queue the jobs a previous process left `processing`.
|
||||||
|
///
|
||||||
|
/// To call once at startup, before the worker starts: it is only correct because
|
||||||
|
/// no other instance can be working on those jobs.
|
||||||
|
pub async fn requeue_interrupted(&self) -> toasty::Result<()> {
|
||||||
|
let mut db = self.db.clone();
|
||||||
|
|
||||||
|
Job::update_by_status(JobStatus::Processing)
|
||||||
|
.status(JobStatus::Pending)
|
||||||
.exec(&mut db)
|
.exec(&mut db)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Retire un job de la file, une fois la review enregistrée.
|
/// Takes the oldest pending job and marks it `processing`, or returns `None` when
|
||||||
|
/// the queue is empty.
|
||||||
|
///
|
||||||
|
/// The selection and the update are a single statement, so a job cannot be taken
|
||||||
|
/// twice.
|
||||||
|
pub async fn claim_next(&self) -> toasty::Result<Option<Job>> {
|
||||||
|
let mut db = self.db.clone();
|
||||||
|
|
||||||
|
let rows = toasty::sql::query(
|
||||||
|
"UPDATE jobs
|
||||||
|
SET status = 'processing'
|
||||||
|
WHERE id = (
|
||||||
|
SELECT id FROM jobs
|
||||||
|
WHERE status = 'pending'
|
||||||
|
ORDER BY id
|
||||||
|
LIMIT 1
|
||||||
|
)
|
||||||
|
RETURNING id",
|
||||||
|
)
|
||||||
|
.exec(&mut db)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let Some(Value::Record(row)) = rows.into_iter().next() else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
|
||||||
|
let id = u64::try_from(row.as_slice()[0].clone())?;
|
||||||
|
|
||||||
|
Ok(Some(Job::get_by_id(&mut db, id).await?))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Waits until a job may have been pushed.
|
||||||
|
pub async fn wait_for_work(&self) {
|
||||||
|
self.work.notified().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Removes a job from the queue, once its review is recorded.
|
||||||
pub async fn remove(&self, id: u64) -> toasty::Result<()> {
|
pub async fn remove(&self, id: u64) -> toasty::Result<()> {
|
||||||
let mut db = self.db.clone();
|
let mut db = self.db.clone();
|
||||||
|
|
||||||
@@ -108,11 +134,13 @@ impl JobQueue {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use futures_util::StreamExt;
|
use tokio::sync::{Mutex, MutexGuard};
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::gitea::{Comment, PullRequest, Repository, ReviewPayload};
|
use crate::gitea::{Comment, PullRequest, Repository, ReviewPayload};
|
||||||
|
|
||||||
|
const FIRST_EVENT: u64 = 999_000_200;
|
||||||
|
|
||||||
fn webhook(event_id: u64) -> WebhookType {
|
fn webhook(event_id: u64) -> WebhookType {
|
||||||
WebhookType::Review(ReviewPayload {
|
WebhookType::Review(ReviewPayload {
|
||||||
action: "created".to_string(),
|
action: "created".to_string(),
|
||||||
@@ -131,41 +159,40 @@ mod tests {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn next_job(queue: &JobQueue, event_id: u64) -> Job {
|
/// The tests claim whichever job is next, so they must not overlap: one would
|
||||||
let mut pending = std::pin::pin!(queue.pending());
|
/// claim the jobs of another.
|
||||||
|
async fn exclusive_queue() -> (MutexGuard<'static, ()>, JobQueue, Db) {
|
||||||
|
static LOCK: Mutex<()> = Mutex::const_new(());
|
||||||
|
|
||||||
tokio::time::timeout(Duration::from_secs(10), async {
|
let guard = LOCK.lock().await;
|
||||||
loop {
|
let db = crate::db::test_connect().await;
|
||||||
if let Some(job) = pending.next().await
|
|
||||||
&& job.event_id == event_id
|
toasty::sql::statement(
|
||||||
{
|
"DELETE FROM jobs WHERE event_id >= 999000200 AND event_id < 999000300",
|
||||||
break job;
|
)
|
||||||
}
|
.exec(&mut db.clone())
|
||||||
}
|
|
||||||
})
|
|
||||||
.await
|
.await
|
||||||
.expect("timed out waiting for the job")
|
.unwrap();
|
||||||
|
|
||||||
|
(guard, JobQueue::new(db.clone()), db)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[ignore = "requires DATABASE_URL"]
|
#[ignore = "requires DATABASE_URL"]
|
||||||
async fn a_pushed_job_is_pending_then_removed() {
|
async fn a_pushed_job_is_claimed_then_removed() {
|
||||||
let db = crate::db::test_connect().await;
|
let (_guard, queue, db) = exclusive_queue().await;
|
||||||
let queue = JobQueue::new(db.clone());
|
|
||||||
|
|
||||||
let event_id = 999_000_201;
|
queue.push(webhook(FIRST_EVENT)).await.unwrap();
|
||||||
Job::delete_by_event_id(&mut db.clone(), event_id)
|
assert!(queue.is_queued(FIRST_EVENT).await.unwrap());
|
||||||
.await
|
|
||||||
.ok();
|
|
||||||
|
|
||||||
queue.push(webhook(event_id)).await.unwrap();
|
let job = queue.claim_next().await.unwrap().expect("a job to claim");
|
||||||
|
assert_eq!(job.event_id, FIRST_EVENT);
|
||||||
|
assert_eq!(job.status, JobStatus::Processing);
|
||||||
|
|
||||||
let job = next_job(&queue, event_id).await;
|
|
||||||
queue.mark_processing(job.id).await.unwrap();
|
|
||||||
queue.remove(job.id).await.unwrap();
|
queue.remove(job.id).await.unwrap();
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
Job::filter_by_event_id(event_id)
|
Job::filter_by_event_id(FIRST_EVENT)
|
||||||
.first()
|
.first()
|
||||||
.exec(&mut db.clone())
|
.exec(&mut db.clone())
|
||||||
.await
|
.await
|
||||||
@@ -177,19 +204,70 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[ignore = "requires DATABASE_URL"]
|
#[ignore = "requires DATABASE_URL"]
|
||||||
async fn a_duplicate_event_id_is_rejected() {
|
async fn a_duplicate_event_id_is_rejected() {
|
||||||
let db = crate::db::test_connect().await;
|
let (_guard, queue, _db) = exclusive_queue().await;
|
||||||
let queue = JobQueue::new(db.clone());
|
|
||||||
|
|
||||||
let event_id = 999_000_202;
|
queue.push(webhook(FIRST_EVENT + 1)).await.unwrap();
|
||||||
Job::delete_by_event_id(&mut db.clone(), event_id)
|
assert!(queue.push(webhook(FIRST_EVENT + 1)).await.is_err());
|
||||||
.await
|
}
|
||||||
.ok();
|
|
||||||
|
|
||||||
queue.push(webhook(event_id)).await.unwrap();
|
#[tokio::test]
|
||||||
assert!(queue.push(webhook(event_id)).await.is_err());
|
#[ignore = "requires DATABASE_URL"]
|
||||||
|
async fn a_claimed_job_is_not_claimed_again() {
|
||||||
|
let (_guard, queue, _db) = exclusive_queue().await;
|
||||||
|
|
||||||
Job::delete_by_event_id(&mut db.clone(), event_id)
|
queue.push(webhook(FIRST_EVENT + 2)).await.unwrap();
|
||||||
|
|
||||||
|
assert!(queue.claim_next().await.unwrap().is_some());
|
||||||
|
assert!(queue.claim_next().await.unwrap().is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore = "requires DATABASE_URL"]
|
||||||
|
async fn jobs_are_claimed_oldest_first() {
|
||||||
|
let (_guard, queue, _db) = exclusive_queue().await;
|
||||||
|
|
||||||
|
queue.push(webhook(FIRST_EVENT + 3)).await.unwrap();
|
||||||
|
queue.push(webhook(FIRST_EVENT + 4)).await.unwrap();
|
||||||
|
|
||||||
|
let first = queue.claim_next().await.unwrap().unwrap();
|
||||||
|
let second = queue.claim_next().await.unwrap().unwrap();
|
||||||
|
|
||||||
|
assert_eq!(first.event_id, FIRST_EVENT + 3);
|
||||||
|
assert_eq!(second.event_id, FIRST_EVENT + 4);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore = "requires DATABASE_URL"]
|
||||||
|
async fn an_interrupted_job_is_requeued() {
|
||||||
|
let (_guard, queue, _db) = exclusive_queue().await;
|
||||||
|
|
||||||
|
queue.push(webhook(FIRST_EVENT + 5)).await.unwrap();
|
||||||
|
let job = queue.claim_next().await.unwrap().unwrap();
|
||||||
|
assert!(queue.claim_next().await.unwrap().is_none());
|
||||||
|
|
||||||
|
// The process died here: on restart, the job goes back to the queue.
|
||||||
|
queue.requeue_interrupted().await.unwrap();
|
||||||
|
|
||||||
|
let again = queue.claim_next().await.unwrap().expect("the job back");
|
||||||
|
assert_eq!(again.id, job.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore = "requires DATABASE_URL"]
|
||||||
|
async fn waiting_for_work_returns_as_soon_as_a_job_is_pushed() {
|
||||||
|
let (_guard, queue, _db) = exclusive_queue().await;
|
||||||
|
|
||||||
|
let waiter = {
|
||||||
|
let queue = queue.clone();
|
||||||
|
tokio::spawn(async move { queue.wait_for_work().await })
|
||||||
|
};
|
||||||
|
|
||||||
|
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||||
|
queue.push(webhook(FIRST_EVENT + 6)).await.unwrap();
|
||||||
|
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), waiter)
|
||||||
.await
|
.await
|
||||||
|
.expect("the worker was not woken up")
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,8 +1,14 @@
|
|||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
bot::Bot, dedup::Dedup, event_bus::EventBus, gitea::GiteaAPI, job_queue::JobQueue,
|
bot::{Bot, BotDeps},
|
||||||
open_router::OpenRouterClient, sandbox::SandboxConfig, state::AppState,
|
event_bus::EventBus,
|
||||||
|
gitea::GiteaAPI,
|
||||||
|
job_queue::JobQueue,
|
||||||
|
open_router::OpenRouterClient,
|
||||||
|
review_store::ReviewStore,
|
||||||
|
sandbox::SandboxConfig,
|
||||||
|
state::AppState,
|
||||||
};
|
};
|
||||||
|
|
||||||
use dotenvy::dotenv;
|
use dotenvy::dotenv;
|
||||||
@@ -16,7 +22,7 @@ mod bot;
|
|||||||
mod bot_actions;
|
mod bot_actions;
|
||||||
mod consts;
|
mod consts;
|
||||||
mod db;
|
mod db;
|
||||||
mod dedup;
|
mod domain;
|
||||||
mod env;
|
mod env;
|
||||||
mod errors;
|
mod errors;
|
||||||
mod event_bus;
|
mod event_bus;
|
||||||
@@ -24,21 +30,29 @@ mod gitea;
|
|||||||
mod job_queue;
|
mod job_queue;
|
||||||
mod metrics;
|
mod metrics;
|
||||||
mod open_router;
|
mod open_router;
|
||||||
|
mod review_store;
|
||||||
mod sandbox;
|
mod sandbox;
|
||||||
mod schemas;
|
mod schemas;
|
||||||
mod state;
|
mod state;
|
||||||
mod text;
|
mod text;
|
||||||
|
|
||||||
/// Délai laissé aux reviews en cours et aux serveurs pour s'arrêter proprement.
|
/// Time left to the running reviews and the servers to stop cleanly.
|
||||||
///
|
///
|
||||||
/// Sans lui, une review prise dans une sandbox récalcitrante garderait le processus
|
/// Without it, a review stuck in an unresponsive sandbox would keep the process
|
||||||
/// en vie jusqu'à ce que le superviseur le tue.
|
/// alive until the supervisor kills it.
|
||||||
const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(60);
|
const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(60);
|
||||||
|
|
||||||
fn main() -> anyhow::Result<()> {
|
fn main() -> anyhow::Result<()> {
|
||||||
dotenv().ok();
|
dotenv().ok();
|
||||||
|
|
||||||
// Sous-commande de migration : `steep-server migration <generate|apply|...>`.
|
// TLS goes through rustls, which has no default crypto provider here: the
|
||||||
|
// workspace compiles `ring` only, and it is installed once, for every client
|
||||||
|
// (reqwest, sentry).
|
||||||
|
rustls::crypto::ring::default_provider()
|
||||||
|
.install_default()
|
||||||
|
.map_err(|_| anyhow::anyhow!("a rustls crypto provider is already installed"))?;
|
||||||
|
|
||||||
|
// Migration subcommand: `steep-server migration <generate|apply|...>`.
|
||||||
if std::env::args().nth(1).as_deref() == Some("migration") {
|
if std::env::args().nth(1).as_deref() == Some("migration") {
|
||||||
return tokio::runtime::Runtime::new()?.block_on(run_migrations());
|
return tokio::runtime::Runtime::new()?.block_on(run_migrations());
|
||||||
}
|
}
|
||||||
@@ -46,7 +60,7 @@ fn main() -> anyhow::Result<()> {
|
|||||||
tracing_subscriber::registry()
|
tracing_subscriber::registry()
|
||||||
.with(fmt::layer())
|
.with(fmt::layer())
|
||||||
.with(
|
.with(
|
||||||
EnvFilter::try_from_default_env() // lit RUST_LOG depuis l'env
|
EnvFilter::try_from_default_env() // reads RUST_LOG from the environment
|
||||||
.unwrap_or_else(|_| EnvFilter::new("info")),
|
.unwrap_or_else(|_| EnvFilter::new("info")),
|
||||||
)
|
)
|
||||||
.init();
|
.init();
|
||||||
@@ -124,25 +138,29 @@ async fn run() -> anyhow::Result<()> {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
let db = db::connect(&config.database_url).await?;
|
let db = db::open(&config.database_url).await?;
|
||||||
let dedup = Dedup::new(db.clone());
|
let reviews = ReviewStore::new(db.clone());
|
||||||
|
|
||||||
let queue = JobQueue::new(db);
|
let queue = JobQueue::new(db);
|
||||||
|
queue.requeue_interrupted().await?;
|
||||||
|
|
||||||
let event_bus = EventBus::new();
|
let event_bus = EventBus::new();
|
||||||
|
|
||||||
let bot = Bot::new(
|
let bot = Bot::new(
|
||||||
gitea_api,
|
BotDeps {
|
||||||
open_router_client,
|
gitea: gitea_api,
|
||||||
config.bot_max_concurrent,
|
open_router: open_router_client,
|
||||||
config.open_router_model.clone(),
|
|
||||||
sandbox,
|
sandbox,
|
||||||
dedup.clone(),
|
reviews: reviews.clone(),
|
||||||
queue.clone(),
|
queue: queue.clone(),
|
||||||
event_bus.clone(),
|
event_bus: event_bus.clone(),
|
||||||
|
},
|
||||||
|
config.bot_max_concurrent,
|
||||||
);
|
);
|
||||||
|
|
||||||
let app_state = AppState {
|
let app_state = AppState {
|
||||||
queue,
|
queue,
|
||||||
dedup,
|
reviews,
|
||||||
event_bus,
|
event_bus,
|
||||||
bot_name: gitea_user.login.clone(),
|
bot_name: gitea_user.login.clone(),
|
||||||
config,
|
config,
|
||||||
|
|||||||
@@ -254,16 +254,19 @@ pub struct OpenRouterClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl OpenRouterClient {
|
impl OpenRouterClient {
|
||||||
pub fn new(token: &str, model: &str, timeout: u64) -> anyhow::Result<Self> {
|
pub fn new(token: &str, model: &str, timeout: Duration) -> anyhow::Result<Self> {
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
client: reqwest::Client::builder()
|
client: reqwest::Client::builder().timeout(timeout).build()?,
|
||||||
.timeout(Duration::from_secs(timeout))
|
|
||||||
.build()?,
|
|
||||||
api_key: String::from(token),
|
api_key: String::from(token),
|
||||||
model: String::from(model),
|
model: String::from(model),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The model every completion is requested from.
|
||||||
|
pub fn model(&self) -> &str {
|
||||||
|
&self.model
|
||||||
|
}
|
||||||
|
|
||||||
/// Sends a conversation with tool definitions and returns either a final
|
/// Sends a conversation with tool definitions and returns either a final
|
||||||
/// message or the tool calls requested by the model.
|
/// message or the tool calls requested by the model.
|
||||||
///
|
///
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
use toasty::Db;
|
use toasty::Db;
|
||||||
|
|
||||||
use crate::schemas::{Job, Review};
|
use crate::schemas::Review;
|
||||||
|
|
||||||
|
/// What is kept of a processed review.
|
||||||
pub struct ReviewRecord {
|
pub struct ReviewRecord {
|
||||||
pub event_id: u64,
|
pub event_id: u64,
|
||||||
pub repo: String,
|
pub repo: String,
|
||||||
@@ -14,35 +15,19 @@ pub struct ReviewRecord {
|
|||||||
pub duration_ms: u64,
|
pub duration_ms: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The history of the reviews Steep has processed.
|
||||||
|
///
|
||||||
|
/// It also is what keeps an event from being reviewed twice.
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct Dedup {
|
pub struct ReviewStore {
|
||||||
db: Db,
|
db: Db,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Dedup {
|
impl ReviewStore {
|
||||||
pub fn new(db: Db) -> Self {
|
pub fn new(db: Db) -> Self {
|
||||||
Self { db }
|
Self { db }
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn is_known(&self, event_id: u64) -> toasty::Result<bool> {
|
|
||||||
let mut db = self.db.clone();
|
|
||||||
|
|
||||||
if Job::filter_by_event_id(event_id)
|
|
||||||
.first()
|
|
||||||
.exec(&mut db)
|
|
||||||
.await?
|
|
||||||
.is_some()
|
|
||||||
{
|
|
||||||
return Ok(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(Review::filter_by_event_id(event_id)
|
|
||||||
.first()
|
|
||||||
.exec(&mut db)
|
|
||||||
.await?
|
|
||||||
.is_some())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn is_reviewed(&self, event_id: u64) -> toasty::Result<bool> {
|
pub async fn is_reviewed(&self, event_id: u64) -> toasty::Result<bool> {
|
||||||
let mut db = self.db.clone();
|
let mut db = self.db.clone();
|
||||||
|
|
||||||
@@ -53,7 +38,8 @@ impl Dedup {
|
|||||||
.is_some())
|
.is_some())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn mark_reviewed(&self, record: ReviewRecord) -> toasty::Result<()> {
|
/// Records a processed review, replacing the one of the same event if any.
|
||||||
|
pub async fn record(&self, record: ReviewRecord) -> toasty::Result<()> {
|
||||||
let mut db = self.db.clone();
|
let mut db = self.db.clone();
|
||||||
|
|
||||||
Review::upsert_by_event_id(record.event_id)
|
Review::upsert_by_event_id(record.event_id)
|
||||||
@@ -71,7 +57,7 @@ impl Dedup {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn list_reviews(&self) -> toasty::Result<Vec<Review>> {
|
pub async fn list(&self) -> toasty::Result<Vec<Review>> {
|
||||||
let mut db = self.db.clone();
|
let mut db = self.db.clone();
|
||||||
let reviews = Review::all().exec(&mut db).await?;
|
let reviews = Review::all().exec(&mut db).await?;
|
||||||
|
|
||||||
@@ -82,24 +68,22 @@ impl Dedup {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::schemas::Review;
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[ignore = "requires DATABASE_URL"]
|
#[ignore = "requires DATABASE_URL"]
|
||||||
async fn a_recorded_review_is_known_and_reviewed() {
|
async fn a_recorded_review_is_reviewed() {
|
||||||
let db = crate::db::test_connect().await;
|
let db = crate::db::test_connect().await;
|
||||||
let dedup = Dedup::new(db.clone());
|
let store = ReviewStore::new(db.clone());
|
||||||
|
|
||||||
let event_id = 999_000_101;
|
let event_id = 999_000_101;
|
||||||
Review::delete_by_event_id(&mut db.clone(), event_id)
|
Review::delete_by_event_id(&mut db.clone(), event_id)
|
||||||
.await
|
.await
|
||||||
.ok();
|
.ok();
|
||||||
|
|
||||||
assert!(!dedup.is_known(event_id).await.unwrap());
|
assert!(!store.is_reviewed(event_id).await.unwrap());
|
||||||
assert!(!dedup.is_reviewed(event_id).await.unwrap());
|
|
||||||
|
|
||||||
dedup
|
store
|
||||||
.mark_reviewed(ReviewRecord {
|
.record(ReviewRecord {
|
||||||
event_id,
|
event_id,
|
||||||
repo: "owner/repo".to_string(),
|
repo: "owner/repo".to_string(),
|
||||||
pr: 1,
|
pr: 1,
|
||||||
@@ -113,8 +97,15 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
assert!(dedup.is_reviewed(event_id).await.unwrap());
|
assert!(store.is_reviewed(event_id).await.unwrap());
|
||||||
assert!(dedup.is_known(event_id).await.unwrap());
|
assert!(
|
||||||
|
store
|
||||||
|
.list()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.any(|r| r.event_id == event_id)
|
||||||
|
);
|
||||||
|
|
||||||
Review::delete_by_event_id(&mut db.clone(), event_id)
|
Review::delete_by_event_id(&mut db.clone(), event_id)
|
||||||
.await
|
.await
|
||||||
@@ -95,8 +95,8 @@ where
|
|||||||
let mut last_rejection: Option<String> = None;
|
let mut last_rejection: Option<String> = None;
|
||||||
|
|
||||||
for iteration in 1..=max_iterations {
|
for iteration in 1..=max_iterations {
|
||||||
// Le dernier tour n'est plus un tour d'exploration : le modèle doit rendre
|
// The last turn is no longer an exploration turn: the model must give its
|
||||||
// sa réponse, avec ce qu'il a déjà vu.
|
// answer, with what it has already seen.
|
||||||
let last = iteration == max_iterations;
|
let last = iteration == max_iterations;
|
||||||
|
|
||||||
if last {
|
if last {
|
||||||
@@ -132,8 +132,8 @@ where
|
|||||||
"Model answered"
|
"Model answered"
|
||||||
);
|
);
|
||||||
|
|
||||||
// Au dernier tour, les outils sont refusés : même si le fournisseur les
|
// On the last turn, tools are refused: even if the provider returns them
|
||||||
// renvoie malgré tout, on ne les exécute pas et on tente la réponse.
|
// anyway, we do not run them and try the answer instead.
|
||||||
if last || response.tool_calls.is_empty() {
|
if last || response.tool_calls.is_empty() {
|
||||||
let answer = response.message.unwrap_or_default();
|
let answer = response.message.unwrap_or_default();
|
||||||
|
|
||||||
@@ -181,7 +181,7 @@ where
|
|||||||
arguments = %excerpt(call.arguments_json(), MAX_LOGGED_ARGUMENTS),
|
arguments = %excerpt(call.arguments_json(), MAX_LOGGED_ARGUMENTS),
|
||||||
output_bytes = content.len(),
|
output_bytes = content.len(),
|
||||||
truncated,
|
truncated,
|
||||||
// Les sauts de ligne casseraient la lisibilité d'une ligne de log.
|
// Line breaks would hurt the readability of a log line.
|
||||||
output = %excerpt(&content, MAX_LOGGED_OUTPUT).replace('\n', " "),
|
output = %excerpt(&content, MAX_LOGGED_OUTPUT).replace('\n', " "),
|
||||||
elapsed_ms = started.elapsed().as_millis() as u64,
|
elapsed_ms = started.elapsed().as_millis() as u64,
|
||||||
"Tool call finished"
|
"Tool call finished"
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
//! Le [`Sandbox`] : clone d'une pull request exécuté dans un devcontainer
|
//! The [`Sandbox`]: a pull request clone running inside an ephemeral
|
||||||
//! éphémère, et sa configuration.
|
//! devcontainer, and its configuration.
|
||||||
|
|
||||||
use std::{
|
use std::{
|
||||||
path::{Path, PathBuf},
|
path::{Path, PathBuf},
|
||||||
@@ -19,7 +19,7 @@ const DEVCONTAINER_PATHS: [&str; 2] = [".devcontainer/devcontainer.json", ".devc
|
|||||||
/// Sandbox-related runtime configuration.
|
/// Sandbox-related runtime configuration.
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct SandboxConfig {
|
pub struct SandboxConfig {
|
||||||
/// Client du daemon de containers qui exécute la sandbox.
|
/// Client of the container daemon that runs the sandbox.
|
||||||
pub runtime: ContainerRuntime,
|
pub runtime: ContainerRuntime,
|
||||||
/// Maximum number of tool-calling iterations per agent run.
|
/// Maximum number of tool-calling iterations per agent run.
|
||||||
pub max_iterations: usize,
|
pub max_iterations: usize,
|
||||||
@@ -79,12 +79,12 @@ impl Sandbox {
|
|||||||
Ok(sandbox)
|
Ok(sandbox)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Vérifie que le clone est bien visible dans le container.
|
/// Checks that the clone is actually visible inside the container.
|
||||||
///
|
///
|
||||||
/// Sans ce contrôle, un workspace vide — par exemple un daemon qui n'a pas pu
|
/// Without this check, an empty workspace (for example a daemon that could not
|
||||||
/// recevoir le clone — fait échouer chaque outil ; le modèle enchaîne alors les
|
/// receive the clone) makes every tool fail; the model then chains failed calls
|
||||||
/// appels ratés jusqu'au budget d'itérations, sans jamais pouvoir reviewer quoi
|
/// until the iteration budget is spent, without ever being able to review
|
||||||
/// que ce soit.
|
/// anything.
|
||||||
async fn check_workspace(&self) -> anyhow::Result<()> {
|
async fn check_workspace(&self) -> anyhow::Result<()> {
|
||||||
let workspace_folder = self.workspace_folder();
|
let workspace_folder = self.workspace_folder();
|
||||||
let probe = self
|
let probe = self
|
||||||
@@ -118,12 +118,12 @@ impl Sandbox {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Rassemble de quoi expliquer un refus d'accès au workspace.
|
/// Gathers enough information to explain a refused access to the workspace.
|
||||||
///
|
///
|
||||||
/// Un `EACCES` a deux causes possibles, indistinguables dans le message de
|
/// An `EACCES` has two possible causes, indistinguishable in the `ls` message:
|
||||||
/// `ls` : les permissions du clone, ou un confinement du noyau qui bloque
|
/// the clone's permissions, or kernel confinement blocking the access. The
|
||||||
/// l'accès. L'identité de l'utilisateur d'exec et les permissions du point de
|
/// identity of the exec user and the permissions of the mount point tell them
|
||||||
/// montage permettent de trancher.
|
/// apart.
|
||||||
async fn diagnose_workspace(&self) -> String {
|
async fn diagnose_workspace(&self) -> String {
|
||||||
let mut details = Vec::new();
|
let mut details = Vec::new();
|
||||||
|
|
||||||
@@ -215,12 +215,12 @@ async fn clone_pull_request(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Rend le clone lisible par les utilisateurs du container sandbox.
|
/// Makes the clone readable by the users of the sandbox container.
|
||||||
///
|
///
|
||||||
/// Le container peut ne pas avoir les mêmes uid que Steep (podman rootless
|
/// The container may not share Steep's uids (rootless podman maps uids through
|
||||||
/// mappe les uid à travers des plages subuid), et l'umask de l'opérateur peut être
|
/// subuid ranges), and the operator's umask may be restrictive: without this, the
|
||||||
/// restrictif : sans cela, les outils de la sandbox échouent en `Permission
|
/// sandbox tools fail with `Permission denied` on files that Steep itself just
|
||||||
/// denied` sur des fichiers que Steep vient de cloner lui-même.
|
/// cloned.
|
||||||
async fn make_readable(repo_dir: &Path) -> anyhow::Result<()> {
|
async fn make_readable(repo_dir: &Path) -> anyhow::Result<()> {
|
||||||
let output = tokio::process::Command::new("chmod")
|
let output = tokio::process::Command::new("chmod")
|
||||||
.args(["-R", "a+rX"])
|
.args(["-R", "a+rX"])
|
||||||
|
|||||||
@@ -167,11 +167,11 @@ async fn read_file(sandbox: &Sandbox, args: &Value) -> anyhow::Result<String> {
|
|||||||
Ok(number_lines(&into_stdout(output)?, first_line))
|
Ok(number_lines(&into_stdout(output)?, first_line))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Préfixe chaque ligne par son numéro.
|
/// Prefixes each line with its number.
|
||||||
///
|
///
|
||||||
/// Le modèle doit citer une ligne précise pour ancrer son commentaire : sans
|
/// The model must cite a precise line to anchor its comment: without numbers, it
|
||||||
/// numéros, il les compte lui-même et se décale de quelques lignes, ce qui place le
|
/// counts them itself and drifts by a few lines, which puts the comment next to
|
||||||
/// commentaire à côté du code visé.
|
/// the targeted code.
|
||||||
fn number_lines(content: &str, first_line: u64) -> String {
|
fn number_lines(content: &str, first_line: u64) -> String {
|
||||||
content
|
content
|
||||||
.lines()
|
.lines()
|
||||||
@@ -291,8 +291,8 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_range_keeps_the_absolute_line_numbers() {
|
fn a_range_keeps_the_absolute_line_numbers() {
|
||||||
// Un extrait lu à partir de la ligne 12 doit garder la numérotation du
|
// An excerpt read from line 12 must keep the file's numbering: otherwise the
|
||||||
// fichier : sinon le modèle citerait des lignes décalées.
|
// model would cite shifted lines.
|
||||||
assert_eq!(number_lines("x\ny", 12), "12:x\n13:y");
|
assert_eq!(number_lines("x\ny", 12), "12:x\n13:y");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -13,22 +13,22 @@ pub struct Review {
|
|||||||
pub repo: String,
|
pub repo: String,
|
||||||
pub pr: u64,
|
pub pr: u64,
|
||||||
|
|
||||||
/// Vrai si la review a abouti.
|
/// True if the review succeeded.
|
||||||
pub ok: bool,
|
pub ok: bool,
|
||||||
|
|
||||||
/// Modèle utilisé.
|
/// Model used.
|
||||||
pub model: String,
|
pub model: String,
|
||||||
|
|
||||||
/// Markdown posté sur la pull request (vide en cas d'échec).
|
/// Markdown posted on the pull request (empty on failure).
|
||||||
pub markdown: String,
|
pub markdown: String,
|
||||||
|
|
||||||
/// Coût OpenRouter en USD, si connu.
|
/// OpenRouter cost in USD, if known.
|
||||||
pub cost: Option<f64>,
|
pub cost: Option<f64>,
|
||||||
|
|
||||||
/// Nombre de commentaires produits.
|
/// Number of comments produced.
|
||||||
pub comments: u64,
|
pub comments: u64,
|
||||||
|
|
||||||
/// Durée du traitement en millisecondes.
|
/// Processing duration in milliseconds.
|
||||||
pub duration_ms: u64,
|
pub duration_ms: u64,
|
||||||
|
|
||||||
#[default(jiff::Timestamp::now())]
|
#[default(jiff::Timestamp::now())]
|
||||||
|
|||||||
@@ -1,14 +1,14 @@
|
|||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
|
|
||||||
use crate::{dedup::Dedup, env::EnvConfig, event_bus::EventBus, job_queue::JobQueue};
|
use crate::{env::EnvConfig, event_bus::EventBus, job_queue::JobQueue, review_store::ReviewStore};
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct AppState {
|
pub struct AppState {
|
||||||
pub queue: JobQueue,
|
pub queue: JobQueue,
|
||||||
pub dedup: Dedup,
|
pub reviews: ReviewStore,
|
||||||
pub event_bus: EventBus,
|
pub event_bus: EventBus,
|
||||||
pub bot_name: String,
|
pub bot_name: String,
|
||||||
pub config: EnvConfig,
|
pub config: EnvConfig,
|
||||||
/// Jeton annulé au démarrage de l'arrêt gracieux : il sert à clore les flux SSE.
|
/// Cancelled when the graceful shutdown starts: it closes the SSE streams.
|
||||||
pub shutdown: CancellationToken,
|
pub shutdown: CancellationToken,
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
# cargo-deny configuration: https://embarkstudios.github.io/cargo-deny/
|
||||||
|
# Run locally with `cargo deny check`.
|
||||||
|
|
||||||
|
[graph]
|
||||||
|
all-features = true
|
||||||
|
|
||||||
|
[advisories]
|
||||||
|
# A vulnerable, unmaintained or yanked dependency fails the build.
|
||||||
|
version = 2
|
||||||
|
yanked = "deny"
|
||||||
|
ignore = [
|
||||||
|
# Unmaintained, but only a thin wrapper around `rustls-pki-types`, and pulled by
|
||||||
|
# `toasty-driver-postgresql`: it can only go away with a toasty release.
|
||||||
|
{ id = "RUSTSEC-2025-0134", reason = "transitive dependency of toasty, no vulnerability" },
|
||||||
|
]
|
||||||
|
|
||||||
|
[licenses]
|
||||||
|
version = 2
|
||||||
|
# The workspace crates are not published, and have no license of their own.
|
||||||
|
private = { ignore = true }
|
||||||
|
allow = [
|
||||||
|
"Apache-2.0",
|
||||||
|
"Apache-2.0 WITH LLVM-exception",
|
||||||
|
"BSD-2-Clause",
|
||||||
|
"BSD-3-Clause",
|
||||||
|
"BSL-1.0",
|
||||||
|
"CC0-1.0",
|
||||||
|
"CDLA-Permissive-2.0",
|
||||||
|
"ISC",
|
||||||
|
"MIT",
|
||||||
|
"MIT-0",
|
||||||
|
"MPL-2.0",
|
||||||
|
"OpenSSL",
|
||||||
|
"Unicode-3.0",
|
||||||
|
"Zlib",
|
||||||
|
]
|
||||||
|
|
||||||
|
[bans]
|
||||||
|
multiple-versions = "warn"
|
||||||
|
wildcards = "deny"
|
||||||
|
# `path` dependencies between the (unpublished) workspace crates carry no version.
|
||||||
|
allow-wildcard-paths = true
|
||||||
|
# TLS goes through rustls: linking libssl would be a regression of the image
|
||||||
|
# (it would need `libssl` installed again).
|
||||||
|
deny = [
|
||||||
|
{ crate = "openssl-sys", reason = "TLS is rustls only, so that the image needs no libssl" },
|
||||||
|
{ crate = "native-tls", reason = "TLS is rustls only, so that the image needs no libssl" },
|
||||||
|
]
|
||||||
|
|
||||||
|
[sources]
|
||||||
|
unknown-registry = "deny"
|
||||||
|
unknown-git = "deny"
|
||||||
|
allow-registry = ["https://github.com/rust-lang/crates.io-index"]
|
||||||
+1
-1
@@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
export const endpoints = {
|
export const endpoints = {
|
||||||
jobs: "/jobs",
|
jobs: "/jobs",
|
||||||
reviews: "/dedup",
|
reviews: "/reviews",
|
||||||
lifecycle: "/events",
|
lifecycle: "/events",
|
||||||
jobEvents: (id) => `/jobs/${id}/events`,
|
jobEvents: (id) => `/jobs/${id}/events`,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
// <steep-dashboard> : assemble le tableau de bord.
|
// <steep-dashboard> : assemble le tableau de bord.
|
||||||
//
|
//
|
||||||
// Au montage, il charge l'état initial via REST (`/jobs`, `/dedup`), puis
|
// Au montage, il charge l'état initial via REST (`/jobs`, `/reviews`), puis
|
||||||
// s'abonne au flux global `/events` pour tenir les listes à jour en direct.
|
// s'abonne au flux global `/events` pour tenir les listes à jour en direct.
|
||||||
// La sélection d'un job ouvre son flux fin via <steep-job-detail>.
|
// La sélection d'un job ouvre son flux fin via <steep-job-detail>.
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user