diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index fe1a8669b3..1e02b68cb6 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -143,12 +143,38 @@ jobs: push: true platforms: linux/amd64,linux/arm64 file: docker/Dockerfile.sandbox + target: sandbox context: . labels: ${{ steps.meta.outputs.labels }} tags: ${{ steps.meta.outputs.tags }} cache-from: type=gha,scope=build-sandbox cache-to: type=gha,scope=build-sandbox,mode=max + # Cube builds templates straight from the image and gates the build on + # GET :49983/health, so it needs the variant with envd baked in. + - name: Docker meta (Cube variant) + id: meta-cube + uses: docker/metadata-action@v6 + with: + images: ${{ secrets.DOCKERHUB_USERNAME }}/weknora-sandbox + flavor: | + suffix=-cube,onlatest=true + + # amd64 only: cubesandbox-base, the source of the envd binary, publishes + # no arm64 image. + - name: Build sandbox Cube Image + uses: docker/build-push-action@v7 + with: + push: true + platforms: linux/amd64 + file: docker/Dockerfile.sandbox + target: cube + context: . + labels: ${{ steps.meta-cube.outputs.labels }} + tags: ${{ steps.meta-cube.outputs.tags }} + cache-from: type=gha,scope=build-sandbox-cube + cache-to: type=gha,scope=build-sandbox-cube,mode=max + build-app: strategy: matrix: diff --git a/docker-compose.yml b/docker-compose.yml index 669ceb43b8..be48443759 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -370,6 +370,7 @@ services: build: context: . dockerfile: docker/Dockerfile.sandbox + target: sandbox profiles: - full command: ["true"] diff --git a/docker/Dockerfile.sandbox b/docker/Dockerfile.sandbox index 971a8b2a65..a64b8837fa 100644 --- a/docker/Dockerfile.sandbox +++ b/docker/Dockerfile.sandbox @@ -1,12 +1,21 @@ # WeKnora Sandbox Image # Pre-built environment for executing agent skill scripts in Docker sandbox # Multi-stage build, minimal dependencies +# +# Two targets ship from this file: +# runtime (default) - the plain environment. Used by the Docker backend and +# as the base image of E2B templates, whose builder +# injects its own envd. +# cube - the same environment plus Cube's envd daemon. Cube +# builds templates straight from the image and probes +# :49983/health, so an image without envd can only ever +# fail. See docs/sandbox-cluster.md. # Stage 1: Get Node.js binaries FROM node:20-slim AS node-base -# Stage 2: Final image -FROM python:3.11-slim +# Stage 2: Runtime image +FROM python:3.11-slim AS runtime # Copy Node.js from node image (avoids NodeSource install overhead) COPY --from=node-base /usr/local/bin/node /usr/local/bin/ @@ -22,9 +31,42 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # Note: Current preloaded skills only use Python stdlib # Add packages here when skills actually need them: -# Create non-root user (UID 1000) for sandbox execution -RUN groupadd -g 1000 sandbox && \ - useradd -u 1000 -g sandbox -m -s /bin/bash sandbox +# Create non-root user (UID 1000) for sandbox execution. The account is named +# "user" because that is the account name E2B templates are expected to expose +# and the one WeKnora names when running scripts and file operations against an +# E2B-protocol backend; the Docker backend addresses it numerically as +# 1000:1000, so both paths land on the same account. +RUN groupadd -g 1000 user && \ + useradd -u 1000 -g user -m -s /bin/bash user WORKDIR /workspace -USER sandbox +USER user + +# Stage 3: Cube template image +# +# envd is what Cube talks to for everything — the readiness probe that decides +# whether a template build succeeds, plus every later exec and file call. It +# runs as root and drops to the account named in each request, which is why +# this variant does not keep the runtime's USER user. +# +# This target is linux/amd64 only: cubesandbox-base publishes no arm64 image, +# which matches Cube itself (PVM is x86_64-only). +FROM ghcr.io/tencentcloud/cubesandbox-base:2026.16 AS cube-base + +FROM runtime AS cube + +USER root +COPY --from=cube-base /usr/bin/envd /usr/bin/envd +COPY --from=cube-base /usr/local/bin/cube-entrypoint.sh /usr/local/bin/cube-entrypoint.sh + +EXPOSE 49983 +# The entrypoint backgrounds envd and then waits on it, since this image +# defines no CMD of its own. +ENTRYPOINT ["/usr/local/bin/cube-entrypoint.sh"] + +# Stage 4: default target +# +# Docker builds the last stage when none is named, and that must stay the plain +# runtime: a bare `docker build -f docker/Dockerfile.sandbox .` is expected to +# produce the image the Docker backend runs, not the Cube variant. +FROM runtime AS sandbox diff --git a/docs/sandbox-cluster.md b/docs/sandbox-cluster.md index 519312207e..d8c75c8d8c 100644 --- a/docs/sandbox-cluster.md +++ b/docs/sandbox-cluster.md @@ -21,22 +21,91 @@ - Node.js 20、npm 与 npx； - jq 及基础 Shell 工具； - `/workspace` 工作目录； -- UID 1000 的非 root `sandbox` 用户。 +- UID 1000 的非 root `user` 账号（E2B 模板约定的账号名，WeKnora 以它执行脚本与文件操作）。 生产环境应使用与 WeKnora 相同的版本标签，不建议长期指向 `latest`。Skills 新增系统依赖时，应先更新标准镜像并重新注册模板，再切换集群的默认模板 ID。 +### 两个镜像变体 + +`docker/Dockerfile.sandbox` 产出两个 target，内容相同、入口不同： + +| 变体 | 标签 | 用途 | +| --- | --- | --- | +| `sandbox`（默认） | `wechatopenai/weknora-sandbox:<版本>` | Docker 后端直接执行脚本；同时作为 E2B 模板的基础镜像 | +| `cube` | `wechatopenai/weknora-sandbox:<版本>-cube` | CubeSandbox 模板 | + +区别在于 Cube 变体额外注入了 envd。Cube 直接把 OCI 镜像变成模板，并以 `GET :49983/health` 探活，这个端点只有 envd 提供；不带 envd 的镜像建模板必然以 `connection refused` 失败。E2B 不需要这个变体，因为它的构建流程会自行注入 envd；Docker 后端则完全不需要 envd。详见 [Cube 自带镜像接入](https://cubesandbox.com/zh/guide/tutorials/bring-your-own-image.html)。 + +Cube 变体只发布 linux/amd64——envd 的来源镜像 `cubesandbox-base` 没有 arm64，Cube 自身的 PVM 形态也只支持 x86_64。变体内 envd 以 root 运行，脚本仍按请求指定的账号执行，落在同一个 uid 1000 的 `user` 上。 + ## CubeSandbox -1. 按 [CubeSandbox Quick Start](https://github.com/TencentCloud/CubeSandbox/blob/master/docs/guide/quickstart.md) 完成控制面、计算节点、CubeProxy 和域名解析配置。生产环境还需要按官方文档完成鉴权、TLS、网络策略和多节点部署。 +### 先选部署形态 + +| 形态 | 适用 | 硬性前提 | +| --- | --- | --- | +| 裸金属 / 物理机 | 已有可用 KVM 的机器 | `/dev/kvm` 可读写；root 权限 | +| PVM | 云厂商屏蔽了嵌套虚拟化、`/dev/kvm` 不可用 | 仅 x86_64；需安装 PVM 宿主机内核并重启；ARM64 不支持 | +| Kubernetes（preview） | 已有集群、要多节点 | K8s 1.24+；计算节点打标签；计算节点非裸金属时需开启 PVM bootstrap | + +三种形态共同的前提： + +- glibc ≥ 2.31（官方二进制基于 Ubuntu 20.04 构建）； +- `/data/cubelet` 挂载 XFS，且开启 reflink（快照的 Copy-on-Write 依赖它）。Ubuntu/Debian 默认 ext4，需要单独准备分区或 loop 设备； +- 至少 50 GB 可用磁盘（要做多个模板则建议 200 GB 以上）、内存 ≥ 8 GB； +- 内核支持 eBPF 且 `/sys/fs/bpf` 已挂载为 bpffs（Cubelet 的网络运行时依赖）； +- 具备 `resolvectl` 或 NetworkManager（安装脚本据此配置 `cube.app` 的 DNS 解析）； +- Docker 可用（MySQL/Redis 以 Docker Compose 运行）。 + +安装脚本会把 CubeMaster、Cubelet、CubeShim 作为宿主机进程运行，因此**不能在没有 systemd 的容器化环境里部署**。CI 容器、无 init 系统的开发机请改用 K8s 形态或另找一台主机。 + +### 制作模板并对接 WeKnora + +1. 按 [CubeSandbox Quick Start](https://github.com/TencentCloud/CubeSandbox/blob/master/docs/zh/guide/quickstart.md) 完成控制面、计算节点、CubeProxy 与域名解析。生产环境还需按官方文档完成鉴权、TLS、网络策略与多节点部署。 2. 在 WeKnora 的空间设置中填写 CubeAPI、CubeProxy、sandbox domain 和可选 API Key。若这些端点位于 RFC1918/loopback 网络，显式打开“允许访问私网集群地址”。 -3. 点击“连接并继续”。WeKnora 先验证控制面地址与凭据，通过后才进入模板步骤并调用集群模板列表；如果不存在名为 `weknora` 的标准模板，会从 `wechatopenai/weknora-sandbox:latest` 发起一次构建。 +3. 点击“连接并继续”。WeKnora 先验证控制面地址与凭据，通过后才进入模板步骤并调用集群模板列表；如果集群里没有可用的 WeKnora 标准模板，会从 `wechatopenai/weknora-sandbox:latest-cube` 发起一次构建。已有模板按名称 `weknora` 或镜像识别，因此即使集群没有给模板登记别名也不会重复创建；已有模板构建失败时，走的是原地重建而不是新建一个。 4. 模板构建状态会自动刷新。状态变为 `READY` 后才可选择并进入运行配置；界面显示模板名称、状态和版本，配置内部才保存该集群自己的 `template_id`。 +模板镜像必须提供 uid 1000 的 `user` 账号：WeKnora 以该账号执行脚本与文件操作。写权限只保证在 `/workspace/output` 与 `/workspace/input` 下。 + 多实例 WeKnora 必须配置 Redis，以共享 session 到 sandbox 的绑定。只有单实例开发环境才应使用内存绑定。 -## E2B +### 验证 + +界面上的“连接验证”只覆盖控制面，“完整验证”会真实创建、执行并销毁一个沙箱。要覆盖 WeKnora 实际依赖的全部语义（会话内状态保持、shell_exec 复用同一沙箱、附件暂存、产物收集、执行超时），在能访问集群的机器上跑一致性测试： + +```bash +CUBE_API_URL=http://127.0.0.1:33000 \ +CUBE_PROXY_URL=http://127.0.0.1:80 \ +CUBE_TEMPLATE_ID=<模板 ID> \ +go test -tags=integration ./internal/sandbox -run Integration -count=1 -v +``` + +测试结束会归还自己创建的沙箱；跑完用 `cubemastercli` 确认没有残留实例。 + +### 排障 + +| 现象 | 排查方向 | +| --- | --- | +| 连接验证失败 | CubeAPI 地址是否是控制面端口（不是 Dashboard 端口）；私网地址是否已打开“允许访问私网集群地址” | +| 连接通过但执行报数据面错误 | CubeProxy 地址与 sandbox domain 是否与集群 `CUBE_API_SANDBOX_DOMAIN` 一致；Proxy 是否对 WeKnora 可达 | +| 模板长期停在构建中 | 镜像体积与网络；用 `cubemastercli tpl watch --job-id ` 看真实进度 | +| 模板构建失败 | 卡片上会显示集群返回的失败原因；点「刷新模板」会就地重建同一个模板，不会新建一个 | +| 失败原因是 `Get "http://:49983/health": connect: connection refused` | 模板镜像里没有 envd。确认建模板用的是 `-cube` 变体镜像；若已是该变体，再查 Cube 沙箱网段（默认 `192.168.0.0/18`）是否与物理内网冲突 | +| 沙箱能创建但脚本报权限错误 | 模板缺少 `user` 账号，或脚本写到了 `/workspace` 根目录 | +| 会话重连后状态丢失 | 多副本部署是否配置了 Redis 绑定存储；沙箱是否已被空闲 TTL 回收 | + +### 已知限制 + +- K8s 部署仍是 preview：计算节点资源紧张时 Pod 可能被误驱逐，计算面升级会重建 Big Pod 并中断存量沙箱。 +- 官方镜像的 Multi-Arch 覆盖尚不完整，ARM64 环境需自行构建模板镜像。 +- 暂不支持 GPU 直通。 + +其它 E2B 兼容后端（含容器隔离的 Kubernetes 实现）与协议层的统一方案见 [沙箱协议接入说明](./sandbox-protocol.md)。 + +## E2B 及其它 E2B 兼容实现 -E2B 官方托管服务和自建 E2B Infrastructure 都可接入。填写 API Key 后先执行“连接并继续”，流程与 Cube 相同：验证连接后列出账号可见模板，缺少 `weknora` 时通过 E2B Template API 从标准镜像启动后台构建。自建部署还需填写 API URL 和 sandbox domain；E2B 上游通过 Terraform 提供 AWS、GCP 等部署方式，具体以 [E2B self-hosting guide](https://github.com/e2b-dev/infra/blob/main/self-host.md) 和 [E2B Template 文档](https://e2b.dev/docs/template/quickstart) 为准。 +E2B 官方托管服务、自建 E2B Infrastructure，以及任意实现 E2B 协议的控制面（例如 Kubernetes 上以容器隔离的 Agent-Sandbox）都通过同一个 E2B 配置接入；自建集群通常还要填写 `proxy_url` 数据面网关，详见 [沙箱协议接入说明](./sandbox-protocol.md)。填写 API Key 后先执行“连接并继续”，流程与 Cube 相同：验证连接后列出账号可见模板，缺少 `weknora` 时通过 E2B Template API 从标准镜像启动后台构建。自建部署还需填写 API URL 和 sandbox domain；E2B 上游通过 Terraform 提供 AWS、GCP 等部署方式，具体以 [E2B self-hosting guide](https://github.com/e2b-dev/infra/blob/main/self-host.md) 和 [E2B Template 文档](https://e2b.dev/docs/template/quickstart) 为准。 ## 在设置页面完成接入 diff --git a/docs/sandbox-protocol.md b/docs/sandbox-protocol.md new file mode 100644 index 0000000000..8958e02257 --- /dev/null +++ b/docs/sandbox-protocol.md @@ -0,0 +1,103 @@ +# WeKnora 沙箱：以 E2B 协议为唯一接入契约 + +本文说明 WeKnora 为什么把 E2B 协议当作沙箱后端的唯一对接契约、现在还有哪些例外、以及可以直接拿来用的开源实现有哪些。面向部署方与要新接一种沙箱后端的开发者。 + +## 结论 + +- WeKnora 只维护一套远端沙箱协议客户端：E2B 协议（控制面 REST + 数据面 envd）。 +- 具体的隔离能力由社区实现提供，WeKnora 不自研沙箱运行时，也不自研“Docker 版控制面”。容器隔离、MicroVM 隔离、托管服务都有现成的 E2B 兼容实现，见下面的选型表。 +- 内置的 `docker` 与 `local` 后端是一次性（每次执行新建、执行完销毁）的本机后端，只适合开发调试。它们不参与会话级沙箱的能力集，也不会再扩展；生产环境请选 E2B 协议后端。 + +## 当前的后端形态 + +| 后端 | 协议 | 会话内状态 | shell_exec / 附件暂存 / 产物收集 | 定位 | +| --- | --- | --- | --- | --- | +| `e2b` | E2B 协议 | 持久（一个会话一个沙箱） | 支持 | 生产主路径，可指向任意 E2B 兼容控制面 | +| `cube` | E2B 兼容（走 Cube 官方 Go SDK） | 持久 | 支持 | CubeSandbox 专用适配器，见“为什么还留着 cube 适配器” | +| `docker` | 无（本机 `docker run --rm`） | 无，每次执行都是新容器 | 不支持 | 本机开发调试 | +| `local` | 无（本机进程） | 无 | 不支持 | 本机开发调试，隔离性最弱 | + +`docker` 与 `local` 的“一次性”不是遗漏而是它们的边界：把会话级持久沙箱做在单机 Docker 上，等于自己实现一遍控制面（生命周期、空闲回收、跨副本绑定、孤儿清理、多租户隔离），而这些正是 E2B 兼容实现已经提供的东西。因此这两个后端保持现状，能力矩阵在 `internal/sandbox/capabilities.go` 中显式表达，agent 侧据此决定是否注册 shell/文件类工具。 + +## 可直接使用的开源实现 + +| 实现 | 隔离方式 | 部署前提 | 适用场景 | +| --- | --- | --- | --- | +| [CubeSandbox](https://github.com/TencentCloud/CubeSandbox)（Apache-2.0） | KVM MicroVM，eBPF 网络隔离 | 裸金属/物理机需 `/dev/kvm`；普通云主机可用 PVM 内核；`/data/cubelet` 需 XFS（reflink）；K8s 部署为 preview | 需要内核级隔离、高密度、快照/回滚 | +| [Agent-Sandbox](https://github.com/agent-sandbox/agent-sandbox)（Apache-2.0） | Kubernetes Pod（容器），可叠加 gVisor/Kata runtimeClass | 一个 K8s 集群（1.26+），`kubectl apply -f install.yaml` | 已有 K8s、想要“容器版 E2B”、不想引入虚拟化依赖 | +| [e2b-dev/infra](https://github.com/e2b-dev/infra)（Apache-2.0） | Firecracker MicroVM | Nomad/Consul + 云厂商 Terraform（AWS/GCP） | 想自建与 E2B Cloud 完全一致的栈 | +| [E2B Cloud](https://e2b.dev) | 托管 MicroVM | 只需 API Key | 不想自己运维 | + +选型要点： + +- 只有容器可用（没有 KVM、也不想上 PVM 内核）时，走 Agent-Sandbox 这类 K8s 原生实现，而不是给 WeKnora 加一个 Docker 控制面。 +- 单机、有 KVM 或可装 PVM 内核，走 CubeSandbox。 +- 上述实现都通过同一个 `e2b` 配置接入，WeKnora 侧零改动。 + +不建议采用的方向：`e2bgateway`、`circlesac/sandbox`、`Cage` 这类项目虽然也宣称 E2B 兼容并支持 Docker 后端，但当前 star 数与维护强度都在个位数量级，作为生产依赖风险过高。 + +## 怎么接入一个 E2B 兼容控制面 + +在“设置 → 沙箱后端”中新建配置，选择 `E2B`，填写： + +| 字段 | 说明 | +| --- | --- | +| `api_key` | 控制面凭据。自建集群通常是它自己签发的 token | +| `api_url` | 控制面地址，例如 `http://agent-sandbox.internal/e2b/v1`。留空则用 E2B Cloud | +| `sandbox_domain` | 沙箱域名。数据面地址形如 `49983-.` | +| `proxy_url` | 数据面网关地址。见下 | +| `template_id` | 模板 / 镜像标识 | +| 允许访问私网集群地址 | 集群位于 RFC1918/loopback 时必须打开 | + +`proxy_url` 是自建集群的关键：E2B Cloud 通过公网 DNS 解析每个沙箱的域名并提供证书，自建集群通常把所有沙箱收敛到一个网关地址、按 Host 头路由。填了 `proxy_url` 之后，WeKnora 会把数据面请求直接拨到该网关，同时保留沙箱域名在 Host 头里；网关是 `http://` 时还会把数据面 scheme 一并降级——E2B SDK 把它写死成 https，这一步省掉了为泛域名申请证书的成本。控制面请求不受影响，仍走共享连接池（实现见 `internal/sandbox/gateway_transport.go`）。 + +配置保存前先执行“连接并继续”，上线前执行一次“完整验证”，后者会真实创建、执行并销毁一个沙箱。 + +## envd 协议的兼容性坑 + +数据面 envd 的契约和 `github.com/matiasinsaurralde/go-e2b` 的实现之间有两处偏差，WeKnora 在 `internal/sandbox/envd_compat_transport.go` 里统一补齐： + +- 认证：envd 要求 `Authorization: Basic base64(":")`，SDK 发的是 `X-User-ID` 头。E2B Cloud 对此宽容，其他实现直接返回 `unauthenticated: no user specified`。 +- 文件上传：envd 的 `POST /files` 只接受 `multipart/form-data`，SDK 发的是裸 `application/octet-stream`，会得到 500。 + +另外健康探针改用 `GET /v2/sandboxes`：旧的 `GET /sandboxes` 已不在客户端其他调用路径上，部分 E2B 兼容实现也只实现了 v2，用旧接口探活会把健康的后端判成不可用。文件操作现在也显式声明执行账号（`user`），与脚本运行账号保持一致，而不是依赖各实现的默认值。 + +模板镜像需要提供 `user` 账号（uid 1000），这是 E2B 模板的既定约定；WeKnora 以该账号执行脚本与文件操作。写权限只保证在 `/workspace/output`（产物目录，执行前由 WeKnora 创建并授权）与 `/workspace/input`（附件暂存）下，脚本不应假设 `/workspace` 根目录可写。 + +## 一致性测试 + +`internal/sandbox/e2b_compatible_integration_test.go` 是面向任意 E2B 兼容控制面的一致性测试，覆盖会话内状态保持、shell_exec 复用同一沙箱、附件暂存、产物收集、执行超时。接一种新后端时先跑它： + +```bash +E2B_INTEGRATION_API_URL=http://127.0.0.1:18080/e2b/v1 \ +E2B_INTEGRATION_API_KEY= \ +E2B_INTEGRATION_TEMPLATE=code-interpreter \ +E2B_INTEGRATION_SANDBOX_DOMAIN=localhost \ +E2B_INTEGRATION_PROXY_URL=http://127.0.0.1:18080 \ +go test -tags=e2b_integration ./internal/sandbox \ + -run '^TestE2BCompatibleControlPlaneConformance' -count=1 -v -timeout=15m +``` + +针对 E2B Cloud 时不要设置 `E2B_INTEGRATION_PROXY_URL`。该套件已在 Kubernetes 上的 Agent-Sandbox（容器后端）实测通过。 + +### 在本机复现一个容器版 E2B 后端 + +只需要 Docker，用 kind 起一个单节点集群即可，全程不涉及 KVM： + +```bash +kind create cluster --name e2b-poc +kubectl create namespace agent-sandbox +kubectl apply -n agent-sandbox -f https://raw.githubusercontent.com/agent-sandbox/agent-sandbox/main/install.yaml + +# 控制面需要一份模板配置；集群里没有 gVisor 时，先把模板的 runtimeClassName 去掉 +kubectl -n agent-sandbox create configmap agent-sandbox \ + --from-file=sandbox.yaml --from-file=templates.json + +kubectl -n agent-sandbox port-forward svc/agent-sandbox 18080:80 +``` + +之后把 `api_url` 指向 `http://127.0.0.1:18080/e2b/v1`、`proxy_url` 指向 `http://127.0.0.1:18080`、`sandbox_domain` 填 `localhost`，即可用上面的命令跑一致性测试。默认 token 在 install.yaml 中，生产部署务必替换。 + +## 为什么还留着 cube 适配器 + +CubeSandbox 兼容 E2B SDK，理论上可以只用 `e2b` 配置接入。目前仍保留独立适配器，原因是它使用 Cube 官方 Go SDK，模板构建、网络策略等控制面能力与 Cube 的 API 一一对应，而这些在通用 E2B 客户端里还没有等价物。合并的前置条件是：在真实 Cube 集群上跑通上面的一致性测试，并把模板构建、网络策略两块能力对齐到通用客户端。数据面路由已经不再是障碍——`proxy_url` 已经泛化成所有远端后端共用的能力。 diff --git a/frontend/src/api/system/index.ts b/frontend/src/api/system/index.ts index 404da8d9e9..617eaa6ba9 100644 --- a/frontend/src/api/system/index.ts +++ b/frontend/src/api/system/index.ts @@ -693,6 +693,7 @@ export interface SandboxCubeConfig { export interface SandboxE2BConfig { api_url?: string + proxy_url?: string sandbox_domain?: string api_key?: string template_id?: string @@ -737,6 +738,8 @@ export interface SandboxTemplate { created_at?: string updated_at?: string standard: boolean + /** The provider's own explanation for a failed build, when it reports one. */ + error?: string } export interface SandboxTemplateCatalog { diff --git a/frontend/src/components/SandboxConfigEditorDrawer.vue b/frontend/src/components/SandboxConfigEditorDrawer.vue index 1179b793ec..241107f970 100644 --- a/frontend/src/components/SandboxConfigEditorDrawer.vue +++ b/frontend/src/components/SandboxConfigEditorDrawer.vue @@ -152,6 +152,10 @@ + + +
@@ -229,6 +233,9 @@ {{ $t('settings.sandbox.templateUntaggedHint') }} + + {{ templateFailureReason(item) }} + {{ $t('settings.sandbox.templateBuildingHint') }} @@ -684,12 +691,25 @@ function isTemplateUntagged(item: SandboxTemplate): boolean { return item.status?.trim().toLowerCase() === 'untagged' } +function isTemplateFailed(item: SandboxTemplate): boolean { + const status = item.status?.trim().toLowerCase() + return ['failed', 'failure', 'error', 'cancelled', 'canceled'].includes(status || '') +} + +// A red "failed" badge on its own leaves no way to tell a registry credential +// problem from a node that ran out of disk, so the provider's own message is +// shown verbatim when it sends one. +function templateFailureReason(item: SandboxTemplate): string { + if (!isTemplateFailed(item)) return '' + const reason = item.error?.trim() + return reason ? t('settings.sandbox.templateFailedReason', { reason }) : '' +} + function templateStatusTheme(item: SandboxTemplate): 'success' | 'warning' | 'danger' | 'default' { if (isTemplateSelectable(item)) return 'success' if (isTemplateUntagged(item)) return 'danger' if (isTemplatePending(item)) return 'warning' - const status = item.status?.trim().toLowerCase() - if (['failed', 'failure', 'error', 'cancelled', 'canceled'].includes(status || '')) return 'danger' + if (isTemplateFailed(item)) return 'danger' return 'default' } @@ -697,10 +717,7 @@ function templateStatusLabel(item: SandboxTemplate): string { if (isTemplateSelectable(item)) return t('settings.sandbox.templateStatuses.ready') if (isTemplateUntagged(item)) return t('settings.sandbox.templateStatuses.untagged') if (isTemplatePending(item)) return t('settings.sandbox.templateStatuses.building') - const status = item.status?.trim().toLowerCase() - if (['failed', 'failure', 'error', 'cancelled', 'canceled'].includes(status || '')) { - return t('settings.sandbox.templateStatuses.failed') - } + if (isTemplateFailed(item)) return t('settings.sandbox.templateStatuses.failed') return t('settings.sandbox.templateStatuses.unknown') } diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index b8e3a5d5ba..36c035de56 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -1158,6 +1158,7 @@ export default { loadingTemplates: 'Loading templates from the cluster...', templateBuildingHint: 'The standard template is being built automatically. This list will refresh.', templateUntaggedHint: 'The builds finished but none carries the default tag, so sandbox creation cannot resolve this template. Delete it in E2B and refresh; WeKnora will rebuild it.', + templateFailedReason: 'Build failed: {reason}', noTemplates: 'No templates were returned by this cluster.', templateReadyHint: 'Template “{name}” is ready and selected.', templateProvisioningHint: 'A template is still being built. The status refreshes automatically.', @@ -1182,6 +1183,7 @@ export default { secretKeepHint: 'Configured — leave empty to keep it', e2bApiUrlOptional: 'Optional — the SDK default is used when empty', e2bDomainOptional: 'Optional — the SDK default is used when empty', + e2bProxyUrlOptional: 'Data-plane gateway of a self-hosted E2B-compatible cluster. Leave empty to reach sandboxes through the sandbox domain, as E2B Cloud expects.', backends: { disabled: 'Disabled', local: 'Local process', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index a6a0a93ecc..dfb334f8b3 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -4758,6 +4758,7 @@ export default { loadingTemplates: 'Loading templates from the cluster...', templateBuildingHint: 'The standard template is being built automatically. This list will refresh.', templateUntaggedHint: '빌드는 끝났지만 default 태그가 붙은 빌드가 없어 샌드박스 생성 시 이 템플릿을 찾을 수 없습니다. E2B에서 템플릿을 삭제하고 새로고침하면 WeKnora가 다시 빌드합니다.', + templateFailedReason: '빌드 실패: {reason}', noTemplates: 'No templates were returned by this cluster.', templateReadyHint: 'Template “{name}” is ready and selected.', templateProvisioningHint: 'A template is still being built. The status refreshes automatically.', @@ -4782,6 +4783,7 @@ export default { secretKeepHint: '설정됨 — 비워 두면 변경되지 않습니다', e2bApiUrlOptional: 'Optional — the SDK default is used when empty', e2bDomainOptional: 'Optional — the SDK default is used when empty', + e2bProxyUrlOptional: 'Data-plane gateway of a self-hosted E2B-compatible cluster. Leave empty to reach sandboxes through the sandbox domain, as E2B Cloud expects.', backends: { disabled: 'Disabled', local: 'Local process', diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 8f521d28e5..08520e7046 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -4758,6 +4758,7 @@ export default { loadingTemplates: 'Loading templates from the cluster...', templateBuildingHint: 'The standard template is being built automatically. This list will refresh.', templateUntaggedHint: 'Сборки завершены, но ни одна не имеет тега default, поэтому при создании песочницы шаблон не находится. Удалите его в E2B и обновите список — WeKnora пересоберёт шаблон.', + templateFailedReason: 'Сборка не удалась: {reason}', noTemplates: 'No templates were returned by this cluster.', templateReadyHint: 'Template “{name}” is ready and selected.', templateProvisioningHint: 'A template is still being built. The status refreshes automatically.', @@ -4782,6 +4783,7 @@ export default { secretKeepHint: 'Настроено — оставьте пустым, чтобы не менять', e2bApiUrlOptional: 'Optional — the SDK default is used when empty', e2bDomainOptional: 'Optional — the SDK default is used when empty', + e2bProxyUrlOptional: 'Data-plane gateway of a self-hosted E2B-compatible cluster. Leave empty to reach sandboxes through the sandbox domain, as E2B Cloud expects.', backends: { disabled: 'Disabled', local: 'Local process', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index e8ae3a762f..850c1cd610 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -4758,6 +4758,7 @@ export default { loadingTemplates: '正在从集群加载模板…', templateBuildingHint: '标准模板正在自动构建，列表会自动刷新。', templateUntaggedHint: '构建已完成，但没有构建带 default 标签，创建沙箱时无法解析。请在 E2B 删除该模板，刷新后 WeKnora 会重新构建。', + templateFailedReason: '构建失败：{reason}', noTemplates: '当前集群未返回可用模板。', templateReadyHint: '模板「{name}」已就绪并选中。', templateProvisioningHint: '模板仍在构建中，状态会自动刷新。', @@ -4782,6 +4783,7 @@ export default { secretKeepHint: '已配置，留空表示不修改', e2bApiUrlOptional: '可留空 —— 留空时使用 SDK 默认值', e2bDomainOptional: '可留空 —— 留空时使用 SDK 默认值', + e2bProxyUrlOptional: '自建 E2B 兼容集群的数据面网关地址；留空表示按 sandbox domain 直连（E2B Cloud 用法）', backends: { disabled: '禁用', local: '本地进程', diff --git a/internal/application/service/tenant_sandbox_config.go b/internal/application/service/tenant_sandbox_config.go index 294fae0208..516370f2f3 100644 --- a/internal/application/service/tenant_sandbox_config.go +++ b/internal/application/service/tenant_sandbox_config.go @@ -25,6 +25,8 @@ package service import ( "context" + "crypto/sha256" + "encoding/hex" stderrors "errors" "fmt" "sort" @@ -32,6 +34,7 @@ import ( "time" "github.com/google/uuid" + "golang.org/x/sync/singleflight" "github.com/Tencent/WeKnora/internal/application/repository" apperrors "github.com/Tencent/WeKnora/internal/errors" @@ -162,6 +165,12 @@ type TenantSandboxConfigService struct { // newClient is injectable so tests can supply a provider inventory. newClient func(*sandbox.Config) (sandbox.ConfigSandboxClient, error) + + // ensureTemplate collapses concurrent "make sure this cluster has our + // template" requests per cluster. Provisioning is idempotent only once the + // build shows up in the provider's catalog, and a double-click on refresh + // is fast enough to slip in before that. + ensureTemplate singleflight.Group } // NewTenantSandboxConfigService wires the config service. @@ -430,17 +439,25 @@ func (s *TenantSandboxConfigService) QueryTemplates( return nil, err } result := &SandboxTemplateCatalog{Templates: deduplicateSandboxTemplates(templates)} - for _, item := range result.Templates { - if item.Standard { - result.StandardTemplateID = item.ID - break - } - } - if in.EnsureStandard && result.StandardTemplateID == "" { - standard, ensureErr := catalog.EnsureStandardTemplate(ctx) + usable := pickStandardTemplate(result.Templates) + if usable != nil { + result.StandardTemplateID = usable.ID + } + // A template whose build failed cannot spawn anything, so it does not count + // as "this cluster already has one" — leaving it at that is what kept a + // broken cluster broken no matter how often the admin hit refresh. + if in.EnsureStandard && usable == nil { + key := ensureTemplateKey(tenantID, sandbox.IdentityOf(merged)) + ensured, ensureErr, _ := s.ensureTemplate.Do(key, func() (any, error) { + return catalog.EnsureStandardTemplate(ctx) + }) if ensureErr != nil { return nil, ensureErr } + standard, ok := ensured.(*sandbox.RemoteTemplate) + if !ok || standard == nil { + return nil, fmt.Errorf("sandbox: provider %q returned no standard template", effective.Type) + } result.Provisioned = true result.StandardTemplateID = standard.ID result.Templates = deduplicateSandboxTemplates(append(result.Templates, *standard)) @@ -454,6 +471,31 @@ func (s *TenantSandboxConfigService) QueryTemplates( return result, nil } +// pickStandardTemplate returns the WeKnora template the UI should preselect, or +// nil when the cluster has none that could ever spawn a sandbox. A failed build +// is skipped so the caller can reprovision instead of offering it. +func pickStandardTemplate(items []sandbox.RemoteTemplate) *sandbox.RemoteTemplate { + var best *sandbox.RemoteTemplate + for i := range items { + if !items[i].Standard || sandbox.IsTemplateBuildFailed(items[i].Status) { + continue + } + if best == nil || templateStatusRank(items[i].Status) > templateStatusRank(best.Status) { + best = &items[i] + } + } + return best +} + +// ensureTemplateKey names one cluster as seen by one tenant. The identity +// carries an API key, so it is hashed rather than formatted: this string is +// only ever compared, and it should not be able to surface a credential in a +// panic trace or a heap dump. +func ensureTemplateKey(tenantID uint64, identity sandbox.SandboxIdentity) string { + sum := sha256.Sum256([]byte(fmt.Sprintf("%d|%#v", tenantID, identity))) + return hex.EncodeToString(sum[:]) +} + func deduplicateSandboxTemplates(items []sandbox.RemoteTemplate) []sandbox.RemoteTemplate { if len(items) < 2 { return items @@ -478,6 +520,7 @@ func deduplicateSandboxTemplates(items []sandbox.RemoteTemplate) []sandbox.Remot current.Status = item.Status current.Version = item.Version current.UpdatedAt = item.UpdatedAt + current.Error = item.Error } if strings.TrimSpace(current.Name) == "" || (strings.EqualFold(current.Name, sandbox.StandardTemplateName) && strings.Contains(item.Name, "/")) { diff --git a/internal/application/service/tenant_sandbox_config_test.go b/internal/application/service/tenant_sandbox_config_test.go index 35593406d6..ef1fb15e2e 100644 --- a/internal/application/service/tenant_sandbox_config_test.go +++ b/internal/application/service/tenant_sandbox_config_test.go @@ -5,6 +5,8 @@ import ( stderrors "errors" "net/http" "strings" + "sync" + "sync/atomic" "testing" "time" @@ -281,6 +283,11 @@ type stubProviderClient struct { inventories [][]sandbox.RemoteSandboxSummary templates []sandbox.RemoteTemplate ensured *sandbox.RemoteTemplate + // ensureDelay widens the window in which concurrent provisioning requests + // overlap, which is the only way to observe whether they were collapsed. + ensureDelay time.Duration + + ensureCalls atomic.Int32 listCalls int deleted []string @@ -292,6 +299,10 @@ func (s *stubProviderClient) ListTemplates(context.Context) ([]sandbox.RemoteTem } func (s *stubProviderClient) EnsureStandardTemplate(context.Context) (*sandbox.RemoteTemplate, error) { + s.ensureCalls.Add(1) + if s.ensureDelay > 0 { + time.Sleep(s.ensureDelay) + } if s.ensured != nil { copy := *s.ensured return ©, nil @@ -355,6 +366,92 @@ func TestQueryTemplatesEnsuresMissingWeKnoraTemplate(t *testing.T) { require.True(t, result.Templates[0].Standard, "standard template should sort first") } +// Provisioning only becomes idempotent once the build shows up in the +// provider's catalog, so overlapping requests have to share one attempt or the +// cluster ends up with a template per click. +func TestQueryTemplatesCollapsesConcurrentProvisioning(t *testing.T) { + client := &stubProviderClient{ensureDelay: 50 * time.Millisecond} + svc := newTestConfigService(t, &fakeConfigRepo{}, client, stubAgentRepo{}) + + var group sync.WaitGroup + for range 4 { + group.Add(1) + go func() { + defer group.Done() + _, err := svc.QueryTemplates(context.Background(), 7, SandboxTemplateQueryInput{ + Config: e2bCfg("key-a", "https://api.e2b.app", "e2b.app", "", 300), + EnsureStandard: true, + }) + require.NoError(t, err) + }() + } + group.Wait() + + require.Equal(t, int32(1), client.ensureCalls.Load()) +} + +// Two tenants pointing at different clusters must not be serialised behind one +// another, and neither may receive the other's template. +func TestQueryTemplatesProvisionsPerClusterIndependently(t *testing.T) { + client := &stubProviderClient{} + svc := newTestConfigService(t, &fakeConfigRepo{}, client, stubAgentRepo{}) + + for _, key := range []string{"key-a", "key-b"} { + _, err := svc.QueryTemplates(context.Background(), 7, SandboxTemplateQueryInput{ + Config: e2bCfg(key, "https://api.e2b.app", "e2b.app", "", 300), + EnsureStandard: true, + }) + require.NoError(t, err) + } + + require.Equal(t, int32(2), client.ensureCalls.Load()) +} + +// A cluster whose only WeKnora template failed to build must be reprovisioned, +// not reported as already equipped. +func TestQueryTemplatesReprovisionsOverFailedStandardTemplate(t *testing.T) { + client := &stubProviderClient{ + templates: []sandbox.RemoteTemplate{ + {ID: "tpl-broken", Name: "weknora", Status: "failed", Standard: true, Error: "no space left"}, + }, + ensured: &sandbox.RemoteTemplate{ + ID: "tpl-broken", Name: "weknora", Status: "building", Standard: true, + }, + } + svc := newTestConfigService(t, &fakeConfigRepo{}, client, stubAgentRepo{}) + + result, err := svc.QueryTemplates(context.Background(), 7, SandboxTemplateQueryInput{ + Config: e2bCfg("key-a", "https://api.e2b.app", "e2b.app", "", 300), + EnsureStandard: true, + }) + + require.NoError(t, err) + require.Equal(t, int32(1), client.ensureCalls.Load()) + require.True(t, result.Provisioned) + require.Equal(t, "tpl-broken", result.StandardTemplateID) + require.Len(t, result.Templates, 1, "a rebuild must not add a catalog entry") + require.Equal(t, "building", result.Templates[0].Status) +} + +// Without EnsureStandard the catalog is read-only, so a failed template is +// reported as it is rather than silently hidden. +func TestQueryTemplatesReportsFailedStandardTemplateWithoutEnsure(t *testing.T) { + client := &stubProviderClient{templates: []sandbox.RemoteTemplate{ + {ID: "tpl-broken", Name: "weknora", Status: "failed", Standard: true, Error: "no space left"}, + }} + svc := newTestConfigService(t, &fakeConfigRepo{}, client, stubAgentRepo{}) + + result, err := svc.QueryTemplates(context.Background(), 7, SandboxTemplateQueryInput{ + Config: e2bCfg("key-a", "https://api.e2b.app", "e2b.app", "", 300), + }) + + require.NoError(t, err) + require.Equal(t, int32(0), client.ensureCalls.Load()) + require.False(t, result.Provisioned) + require.Empty(t, result.StandardTemplateID, "a failed template must not be preselected") + require.Equal(t, "no space left", result.Templates[0].Error) +} + func TestQueryTemplatesDeduplicatesSameProviderTemplateID(t *testing.T) { client := &stubProviderClient{templates: []sandbox.RemoteTemplate{ {ID: "tpl-weknora", Name: "weknora", Status: "building", Standard: true}, diff --git a/internal/sandbox/config_identity.go b/internal/sandbox/config_identity.go index 984ce99ca3..98fe2fb0ac 100644 --- a/internal/sandbox/config_identity.go +++ b/internal/sandbox/config_identity.go @@ -69,6 +69,7 @@ func IdentityOf(tenantCfg *types.TenantSandboxConfig) SandboxIdentity { if e2bCfg := tenantCfg.E2B; e2bCfg != nil { identity.APIURL, identity.APIKey = e2bCfg.APIURL, e2bCfg.APIKey identity.SandboxDomain = e2bCfg.SandboxDomain + identity.ProxyURL = e2bCfg.ProxyURL } } return identity diff --git a/internal/sandbox/cube_remote_client.go b/internal/sandbox/cube_remote_client.go index 34d063efd2..4bdc30240b 100644 --- a/internal/sandbox/cube_remote_client.go +++ b/internal/sandbox/cube_remote_client.go @@ -46,7 +46,7 @@ func NewCubeRemoteClient(config *Config) (*CubeRemoteClient, error) { // proxy dial rewrite for the data plane. A nil pool keeps the SDK defaults. func NewCubeRemoteClientWithPool( config *Config, - pool *CubeTransportPool, + pool *SandboxGatewayTransportPool, ) (*CubeRemoteClient, error) { if config == nil { return nil, errors.New("cube remote client config is required") @@ -149,8 +149,16 @@ func (c *CubeRemoteClient) ListTemplates(ctx context.Context) ([]RemoteTemplate, result := make([]RemoteTemplate, 0, len(items)) for _, item := range items { name := strings.TrimSpace(item.Name) + // Cube only reports a name when the template carries an alias, so fall + // back to the image before falling back to the opaque ID: recognising + // our own template is what keeps EnsureStandardTemplate idempotent. + standard := isStandardTemplate(name) || isStandardTemplateImage(item.ImageInfo) if name == "" { - name = item.TemplateID + if standard { + name = StandardTemplateName + } else { + name = item.TemplateID + } } result = append(result, RemoteTemplate{ ID: item.TemplateID, @@ -159,27 +167,40 @@ func (c *CubeRemoteClient) ListTemplates(ctx context.Context) ([]RemoteTemplate, Version: item.Version, Image: item.ImageInfo, CreatedAt: item.CreatedAt, - Standard: isStandardTemplate(name), + Standard: standard, + Error: strings.TrimSpace(item.LastError), }) } return result, nil } +// EnsureStandardTemplate makes the cluster hold exactly one WeKnora template. +// A healthy or still-building one is returned as is; a failed one is rebuilt in +// place, because building a second template would leave the failed one behind +// and repeat on every refresh. func (c *CubeRemoteClient) EnsureStandardTemplate(ctx context.Context) (*RemoteTemplate, error) { items, err := c.ListTemplates(ctx) if err != nil { return nil, err } + var failed *RemoteTemplate for i := range items { - if items[i].Standard { + if !items[i].Standard { + continue + } + if !IsTemplateBuildFailed(items[i].Status) { return &items[i], nil } + if failed == nil { + failed = &items[i] + } + } + if failed != nil { + return c.rebuildStandardTemplate(ctx, *failed) } job, err := c.client.BuildTemplate(ctx, cubesandbox.BuildTemplateOptions{ - Image: DefaultDockerImage, - Name: StandardTemplateName, - WritableLayerSize: "1G", - ExposedPorts: []uint16{49983}, + Image: DefaultCubeTemplateImage, + Extra: cubeStandardTemplateSpec(), }) if err != nil { return nil, normalizeCubeError("EnsureStandardTemplate", err) @@ -188,11 +209,50 @@ func (c *CubeRemoteClient) EnsureStandardTemplate(ctx context.Context) (*RemoteT ID: job.TemplateID, Name: StandardTemplateName, Status: job.Status, - Image: DefaultDockerImage, + Image: DefaultCubeTemplateImage, Standard: true, + Error: strings.TrimSpace(job.ErrorMessage), }, nil } +// rebuildStandardTemplate restarts the build of a template that already exists, +// keeping its ID so a retry never adds to the catalog. +func (c *CubeRemoteClient) rebuildStandardTemplate( + ctx context.Context, + current RemoteTemplate, +) (*RemoteTemplate, error) { + logger.Infof(ctx, "cube standard template %s failed (%s), rebuilding in place", + current.ID, current.Status) + job, err := c.client.RebuildTemplate(ctx, current.ID, cubeStandardTemplateSpec()) + if err != nil { + return nil, normalizeCubeError("EnsureStandardTemplate", err) + } + rebuilt := current + rebuilt.Status = job.Status + rebuilt.Error = strings.TrimSpace(job.ErrorMessage) + if strings.TrimSpace(job.TemplateID) != "" { + rebuilt.ID = job.TemplateID + } + return &rebuilt, nil +} + +// cubeStandardTemplateSpec is the single definition of how the WeKnora template +// is built. Both the first build and every rebuild send it verbatim — the +// rebuild endpoint takes a raw payload rather than BuildTemplateOptions, and +// two hand-kept copies of the spec would eventually disagree. +func cubeStandardTemplateSpec() map[string]any { + return map[string]any{ + "image": DefaultCubeTemplateImage, + "name": StandardTemplateName, + "writableLayerSize": "1G", + "exposedPorts": []uint16{CubeEnvdPort}, + // Cube defaults to probing envd, but naming the probe keeps the reason + // this image must ship envd visible at the call site. + "probePort": uint16(CubeEnvdPort), + "probePath": CubeEnvdHealthPath, + } +} + func (c *CubeRemoteClient) Create( ctx context.Context, request RemoteCreateRequest, diff --git a/internal/sandbox/cube_template_catalog_test.go b/internal/sandbox/cube_template_catalog_test.go new file mode 100644 index 0000000000..9cded5a638 --- /dev/null +++ b/internal/sandbox/cube_template_catalog_test.go @@ -0,0 +1,196 @@ +package sandbox + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// newCubeTemplateClient points a CubeRemoteClient at a bare template-API stub. +// The full cubeMockServer models sandboxes rather than the template catalog, +// and these tests only need the three /templates routes. +func newCubeTemplateClient(t *testing.T, handler http.HandlerFunc) *CubeRemoteClient { + t.Helper() + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + client, err := NewCubeRemoteClient(&Config{ + Type: SandboxTypeCube, + CubeAPIURL: server.URL, + CubeProxyURL: server.URL, + CubeSandboxDomain: "cube.app", + CubeHTTPTimeout: 5 * time.Second, + }) + require.NoError(t, err) + return client +} + +// Cube omits the name of a template that carries no alias, which used to make +// our own template unrecognisable and every catalog refresh build another one. +func TestCubeRemoteClientListTemplatesRecognisesStandardByImage(t *testing.T) { + client := newCubeTemplateClient(t, func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "/templates", r.URL.Path) + writeJSON(w, http.StatusOK, []map[string]any{ + { + "templateID": "tpl-nameless", + "status": "READY", + "imageInfo": DefaultDockerImage, + }, + { + "templateID": "tpl-other", + "status": "READY", + "imageInfo": "python:3.11", + }, + }) + }) + + templates, err := client.ListTemplates(context.Background()) + require.NoError(t, err) + require.Len(t, templates, 2) + + require.True(t, templates[0].Standard) + require.Equal(t, StandardTemplateName, templates[0].Name, + "a recognised template must be labelled, not shown as a bare ID") + require.False(t, templates[1].Standard) + require.Equal(t, "tpl-other", templates[1].Name) +} + +func TestCubeRemoteClientListTemplatesSurfacesLastError(t *testing.T) { + client := newCubeTemplateClient(t, func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, []map[string]any{{ + "templateID": "tpl-broken", + "status": "FAILED", + "imageInfo": DefaultDockerImage, + "lastError": "pull access denied for wechatopenai/weknora-sandbox", + }}) + }) + + templates, err := client.ListTemplates(context.Background()) + require.NoError(t, err) + require.Len(t, templates, 1) + require.Equal(t, "pull access denied for wechatopenai/weknora-sandbox", templates[0].Error) +} + +// The bug this guards: an unnamed WeKnora template was invisible to the +// idempotency check, so every visit to the template step queued another build. +func TestCubeRemoteClientEnsureStandardTemplateSkipsBuildForNamelessTemplate(t *testing.T) { + var builds atomic.Int32 + client := newCubeTemplateClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + builds.Add(1) + } + writeJSON(w, http.StatusOK, []map[string]any{{ + "templateID": "tpl-nameless", + "status": "READY", + "imageInfo": DefaultDockerImage, + }}) + }) + + for range 3 { + template, err := client.EnsureStandardTemplate(context.Background()) + require.NoError(t, err) + require.Equal(t, "tpl-nameless", template.ID) + } + require.Equal(t, int32(0), builds.Load()) +} + +// A failed template must be rebuilt in place. Building a fresh one would leave +// the failure behind and repeat on the next refresh. +func TestCubeRemoteClientEnsureStandardTemplateRebuildsFailedTemplate(t *testing.T) { + var created atomic.Int32 + var rebuilt atomic.Int32 + client := newCubeTemplateClient(t, func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/templates" && r.Method == http.MethodGet: + writeJSON(w, http.StatusOK, []map[string]any{{ + "templateID": "tpl-failed", + "status": "FAILED", + "imageInfo": DefaultDockerImage, + "lastError": "no space left on device", + }}) + case r.URL.Path == "/templates" && r.Method == http.MethodPost: + created.Add(1) + writeJSON(w, http.StatusAccepted, map[string]any{"templateID": "tpl-new"}) + case r.URL.Path == "/templates/tpl-failed" && r.Method == http.MethodPost: + rebuilt.Add(1) + var payload map[string]any + require.NoError(t, json.NewDecoder(r.Body).Decode(&payload)) + require.Equal(t, DefaultCubeTemplateImage, payload["image"]) + require.Equal(t, StandardTemplateName, payload["name"]) + writeJSON(w, http.StatusAccepted, map[string]any{ + "templateID": "tpl-failed", + "status": "PENDING", + }) + default: + http.NotFound(w, r) + } + }) + + template, err := client.EnsureStandardTemplate(context.Background()) + require.NoError(t, err) + require.Equal(t, "tpl-failed", template.ID) + require.Equal(t, "PENDING", template.Status) + require.Equal(t, int32(1), rebuilt.Load()) + require.Equal(t, int32(0), created.Load(), "a rebuild must not add a template") +} + +func TestCubeRemoteClientEnsureStandardTemplateBuildsWhenAbsent(t *testing.T) { + var payload map[string]any + client := newCubeTemplateClient(t, func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + writeJSON(w, http.StatusOK, []map[string]any{}) + case http.MethodPost: + require.NoError(t, json.NewDecoder(r.Body).Decode(&payload)) + writeJSON(w, http.StatusAccepted, map[string]any{ + "templateID": "tpl-fresh", + "status": "PENDING", + }) + } + }) + + template, err := client.EnsureStandardTemplate(context.Background()) + require.NoError(t, err) + require.Equal(t, "tpl-fresh", template.ID) + require.True(t, template.Standard) + // Cube probes envd to decide whether the build succeeded, so the template + // must be built from the variant that ships it. + require.Equal(t, DefaultCubeTemplateImage, payload["image"]) + require.Equal(t, StandardTemplateName, payload["name"]) + require.Equal(t, "1G", payload["writableLayerSize"]) + require.EqualValues(t, CubeEnvdPort, payload["probePort"]) + require.Equal(t, CubeEnvdHealthPath, payload["probePath"]) +} + +// The plain and Cube images share a repository, so a template built from either +// one is still recognised as ours — which is what lets an existing template +// built from the envd-less image be rebuilt in place rather than duplicated. +func TestCubeTemplateImageIsRecognisedAsStandard(t *testing.T) { + require.True(t, isStandardTemplateImage(DefaultCubeTemplateImage)) + require.True(t, isStandardTemplateImage(DefaultDockerImage)) +} + +func TestIsStandardTemplateImage(t *testing.T) { + for _, image := range []string{ + DefaultDockerImage, + "wechatopenai/weknora-sandbox", + "docker.io/wechatopenai/weknora-sandbox:latest", + "docker.io/wechatopenai/weknora-sandbox@sha256:abc", + "registry.internal:5000/wechatopenai/weknora-sandbox:v1", + } { + require.True(t, isStandardTemplateImage(image), image) + } + for _, image := range []string{ + "", + "python:3.11", + "wechatopenai/weknora-docreader:latest", + "someone-else/weknora-sandbox:latest", + } { + require.False(t, isStandardTemplateImage(image), image) + } +} diff --git a/internal/sandbox/cube_transport.go b/internal/sandbox/cube_transport.go deleted file mode 100644 index a189dcad06..0000000000 --- a/internal/sandbox/cube_transport.go +++ /dev/null @@ -1,140 +0,0 @@ -// Package sandbox: connection pooling for per-request Cube clients. -// -// Named Cube configs build a fresh client on every Resolve, so without an -// externally owned transport every request would open new TCP connections to -// both the CubeAPI control plane and the envd proxy. -// -// Cube cannot reuse the single shared transport E2B uses, because the SDK -// speaks two planes with different dialling rules: -// -// - control plane (Create/Connect/List) talks to CubeAPIURL directly; -// - data plane (exec, filesystem) addresses sandboxes as -// "49983-{id}.{domain}" but must dial CubeProxyURL, keeping the sandbox -// authority in the Host header so the proxy can route it. -// -// Handing the SDK one http.Client for both planes (WithHTTPClient overwrites -// controlHTTP and dataHTTP alike) drops that dial rewrite, which only appears -// to work when DNS happens to resolve the sandbox domain to the proxy node on -// the same port. This file keeps the rewrite by routing per request: control -// traffic rides the process-wide transport shared with E2B, data traffic rides -// a transport cached per proxy endpoint so configs pointing at the same proxy -// share one pool. -package sandbox - -import ( - "context" - "net" - "net/http" - "strconv" - "strings" - "sync" - "time" -) - -// CubeTransportPool owns the transports handed to per-request Cube clients. -// One instance lives for the process; clients built from it come and go. -type CubeTransportPool struct { - control http.RoundTripper - policy OutboundURLPolicy - - // data maps a proxy "host:port" to the transport that dials it. - data sync.Map -} - -// NewCubeTransportPool returns a pool whose control plane rides control. -// A nil control transport installs a guarded one. -func NewCubeTransportPool(control http.RoundTripper) *CubeTransportPool { - return NewCubeTransportPoolWithPolicy(control, DefaultOutboundURLPolicy()) -} - -func NewCubeTransportPoolWithPolicy(control http.RoundTripper, policy OutboundURLPolicy) *CubeTransportPool { - if control == nil { - control = NewGuardedTransportWithPolicy(policy) - } - return &CubeTransportPool{control: control, policy: policy} -} - -// RoundTripperFor returns the transport a client built from cfg should use. -// Configs without a usable proxy URL keep every request on the control -// transport, matching the SDK's behaviour when no proxy node is configured. -func (p *CubeTransportPool) RoundTripperFor(cfg *Config) http.RoundTripper { - split := &cubeSplitTransport{ - control: p.control, - sandboxDomain: strings.ToLower(strings.TrimSpace(cfg.CubeSandboxDomain)), - } - if host, port, _, ok := parseProxyURL(cfg.CubeProxyURL); ok { - split.data = p.dataTransport(net.JoinHostPort(host, strconv.Itoa(port))) - } - return split -} - -// dataTransport returns the transport dialling target, creating it once. -func (p *CubeTransportPool) dataTransport(target string) http.RoundTripper { - if existing, ok := p.data.Load(target); ok { - return existing.(http.RoundTripper) - } - actual, _ := p.data.LoadOrStore(target, newCubeDataTransportWithPolicy(target, p.policy)) - return actual.(http.RoundTripper) -} - -// newCubeDataTransport dials target regardless of the request's authority, -// mirroring the SDK's proxy rewrite while adding the outbound address guard -// the SDK has no notion of. -func newCubeDataTransport(target string) *http.Transport { - return newCubeDataTransportWithPolicy(target, DefaultOutboundURLPolicy()) -} - -func newCubeDataTransportWithPolicy(target string, policy OutboundURLPolicy) *http.Transport { - dialer := &net.Dialer{ - Timeout: 10 * time.Second, - KeepAlive: 30 * time.Second, - Control: SafeDialControlForPolicy(policy), - } - return &http.Transport{ - // The proxy is addressed directly; an ambient HTTP proxy would - // defeat the rewrite. - Proxy: nil, - DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { - return dialer.DialContext(ctx, network, target) - }, - MaxIdleConns: 100, - MaxIdleConnsPerHost: 4, - IdleConnTimeout: 90 * time.Second, - } -} - -// cubeSplitTransport routes a request to the control or the data transport by -// looking at the authority the SDK addressed. -type cubeSplitTransport struct { - control http.RoundTripper - data http.RoundTripper - sandboxDomain string -} - -func (t *cubeSplitTransport) RoundTrip(req *http.Request) (*http.Response, error) { - if t.data != nil && t.isDataPlane(req.URL.Hostname()) { - return t.data.RoundTrip(req) - } - return t.control.RoundTrip(req) -} - -// isDataPlane reports whether host addresses a sandbox rather than CubeAPI. -// Anything else - including an unset sandbox domain - stays on the control -// transport, so a misconfiguration cannot silently redirect API calls at the -// proxy. -func (t *cubeSplitTransport) isDataPlane(host string) bool { - if t.sandboxDomain == "" { - return false - } - host = strings.ToLower(host) - return host == t.sandboxDomain || strings.HasSuffix(host, "."+t.sandboxDomain) -} - -// CloseIdleConnections keeps the SDK's post-rollback reset meaningful. Only -// the data pool is dropped: the control transport is shared with every other -// tenant and with E2B, and one sandbox's restart is no reason to close it. -func (t *cubeSplitTransport) CloseIdleConnections() { - if closer, ok := t.data.(interface{ CloseIdleConnections() }); ok { - closer.CloseIdleConnections() - } -} diff --git a/internal/sandbox/e2b_compatible_integration_test.go b/internal/sandbox/e2b_compatible_integration_test.go new file mode 100644 index 0000000000..3272ddcbe0 --- /dev/null +++ b/internal/sandbox/e2b_compatible_integration_test.go @@ -0,0 +1,261 @@ +//go:build e2b_integration + +// Conformance test for E2B-protocol control planes. +// +// WeKnora treats "E2B protocol" as the single integration contract for remote +// sandboxes, so the same suite must pass against every implementation of it: +// E2B Cloud, a self-hosted e2b-dev/infra, CubeSandbox's CubeAPI, or a +// container-backed gateway such as Agent-Sandbox. It drives the same public +// surface the agent runtime uses — session-scoped script execution, shell +// commands, attachment staging, artifact listing, teardown — rather than the +// individual client methods, so a backend that passes here is usable by the +// product and not merely reachable. +// +// Run with: +// +// E2B_INTEGRATION_API_URL=http://127.0.0.1:18080/e2b/v1 \ +// E2B_INTEGRATION_API_KEY= \ +// E2B_INTEGRATION_TEMPLATE=code-interpreter \ +// E2B_INTEGRATION_SANDBOX_DOMAIN=localhost \ +// E2B_INTEGRATION_PROXY_URL=http://127.0.0.1:18080 \ +// go test -tags=e2b_integration ./internal/sandbox \ +// -run '^TestE2BCompatibleControlPlaneConformance' -count=1 -v -timeout=15m +// +// E2B_INTEGRATION_PROXY_URL is the data-plane gateway. Leave it empty for E2B +// Cloud, whose sandbox domain resolves through public DNS over TLS. +package sandbox + +import ( + "bytes" + "context" + "fmt" + "os" + "path" + "strings" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/types" +) + +const ( + conformanceTenantID = 1 + conformanceTTL = 10 * time.Minute + conformanceHTTPTimeut = 60 * time.Second + conformanceExecUser = "E2B_INTEGRATION_EXEC_USER" +) + +func TestE2BCompatibleControlPlaneConformance(t *testing.T) { + cfg := e2bCompatibleConfig(t) + client, err := NewE2BRemoteClientWithPool( + cfg, + NewSandboxGatewayTransportPoolWithPolicy(nil, OutboundURLPolicy{AllowPrivate: true}), + ) + if err != nil { + t.Fatalf("build E2B-protocol client: %v", err) + } + + ctx, cancel := context.WithTimeout( + types.WithSandboxTenantID(context.Background(), conformanceTenantID), + 12*time.Minute, + ) + defer cancel() + + if err := client.Health(ctx); err != nil { + t.Fatalf("control plane health: %v", err) + } + + manager, err := NewSessionBoundManager(SessionBoundManagerConfig{ + Config: cfg, + Client: client, + Store: NewMemorySessionSandboxBindingStore(), + Checker: PermissiveSessionExistenceChecker{}, + ConfigID: "conformance", + SkipHealthProbe: true, + }) + if err != nil { + t.Fatalf("NewSessionBoundManager: %v", err) + } + + sessionID := fmt.Sprintf("conformance-%d", time.Now().UnixNano()) + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout( + types.WithSandboxTenantID(context.Background(), conformanceTenantID), + 2*time.Minute, + ) + defer cleanupCancel() + if err := manager.DestroySession(cleanupCtx, sessionID); err != nil { + t.Errorf("DestroySession: %v", err) + } + }) + + // State is written under the artifact directory rather than an arbitrary + // path: that directory is the one WeKnora provisions and grants to the + // script account, so the assertion tests session persistence instead of a + // template's /workspace permissions. + counterPath := path.Join(SessionOutputRoot, "counter.txt") + t.Run("SessionScopedStatePersistsAcrossExecutions", func(t *testing.T) { + first := runConformanceScript(t, ctx, manager, sessionID, fmt.Sprintf(` +with open(%q, 'w') as handle: + handle.write('1') +print('wrote counter') +`, counterPath)) + if !first.IsSuccess() { + t.Fatalf("first execution failed: %#v", first) + } + + second := runConformanceScript(t, ctx, manager, sessionID, fmt.Sprintf(` +with open(%q) as handle: + print('counter=' + handle.read()) +`, counterPath)) + if !second.IsSuccess() { + t.Fatalf("second execution failed: %#v", second) + } + if !strings.Contains(second.Stdout, "counter=1") { + t.Fatalf("session state did not persist across executions: stdout=%q stderr=%q", + second.Stdout, second.Stderr) + } + }) + + t.Run("ShellExecSharesTheSessionSandbox", func(t *testing.T) { + executor := manager.SessionShellExecutor() + if executor == nil { + t.Fatal("session shell executor is unavailable on a healthy remote backend") + } + result, err := executor.ExecShellCommand( + ctx, sessionID, "cat "+counterPath, SessionWorkspaceRoot, + time.Minute, nil, + ) + if err != nil { + t.Fatalf("ExecShellCommand: %v", err) + } + if !result.IsSuccess() || !strings.Contains(result.Stdout, "1") { + t.Fatalf("shell command did not observe the session sandbox: %#v", result) + } + }) + + t.Run("AttachmentStagingAndArtifactCollection", func(t *testing.T) { + files := manager.SessionFileStore() + if files == nil { + t.Fatal("session file store is unavailable on a healthy remote backend") + } + inputPath := path.Join(SessionInputRoot, "attachment.txt") + payload := []byte("attachment payload\n") + if err := files.WriteSessionInputFile(ctx, sessionID, inputPath, payload); err != nil { + t.Fatalf("WriteSessionInputFile: %v", err) + } + content, err := files.ReadSessionFile(ctx, sessionID, inputPath) + if err != nil { + t.Fatalf("ReadSessionFile: %v", err) + } + if !bytes.Equal(content, payload) { + t.Fatalf("staged attachment mismatch: got=%q want=%q", content, payload) + } + + result := runConformanceScript(t, ctx, manager, sessionID, fmt.Sprintf(` +import os +target = os.path.join(os.environ['%s'], 'report.txt') +with open(target, 'w') as handle: + handle.write('artifact body') +print('artifact written') +`, skillOutputEnvVar)) + if !result.IsSuccess() { + t.Fatalf("artifact-producing execution failed: %#v", result) + } + + entries, err := files.ListSessionFiles(ctx, sessionID, SessionOutputRoot) + if err != nil { + t.Fatalf("ListSessionFiles: %v", err) + } + found := false + for _, entry := range entries { + if entry.Name == "report.txt" { + found = true + break + } + } + if !found { + t.Fatalf("artifact directory did not contain report.txt: %#v", entries) + } + + if err := files.RemoveSessionInputPath(ctx, sessionID, inputPath); err != nil { + t.Fatalf("RemoveSessionInputPath: %v", err) + } + }) + + t.Run("TimeoutIsReportedAsKilled", func(t *testing.T) { + result := runConformanceScriptWithTimeout(t, ctx, manager, sessionID, ` +import time +time.sleep(30) +`, 5*time.Second) + if !result.Killed { + t.Fatalf("expected a killed result for an over-running script: %#v", result) + } + }) +} + +// runConformanceScript executes source as a session-scoped Python script, +// mirroring how the skills runtime invokes the sandbox. +func runConformanceScript( + t *testing.T, + ctx context.Context, + manager *SessionBoundManager, + sessionID string, + source string, +) *ExecuteResult { + t.Helper() + return runConformanceScriptWithTimeout(t, ctx, manager, sessionID, source, 2*time.Minute) +} + +func runConformanceScriptWithTimeout( + t *testing.T, + ctx context.Context, + manager *SessionBoundManager, + sessionID string, + source string, + timeout time.Duration, +) *ExecuteResult { + t.Helper() + result, err := manager.Execute(ctx, &ExecuteConfig{ + Script: "conformance.py", + ScriptContent: source, + SessionID: sessionID, + Timeout: timeout, + SkipValidation: true, + Env: map[string]string{ + skillOutputEnvVar: SessionOutputRoot, + }, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result == nil { + t.Fatal("Execute returned no result") + } + t.Logf("execute exit=%d killed=%v stdout=%q stderr=%q err=%q", + result.ExitCode, result.Killed, result.Stdout, result.Stderr, result.Error) + return result +} + +func e2bCompatibleConfig(t *testing.T) *Config { + t.Helper() + apiKey := firstNonEmptyEnvironment("E2B_INTEGRATION_API_KEY", "E2B_API_KEY") + template := firstNonEmptyEnvironment("E2B_INTEGRATION_TEMPLATE", "E2B_TEMPLATE") + if apiKey == "" || template == "" { + t.Skip("E2B-protocol conformance requires an API key and a template") + } + + cfg := DefaultConfig() + cfg.Type = SandboxTypeE2B + cfg.FallbackEnabled = false + cfg.AllowPrivateEndpoints = true + cfg.E2BAPIKey = apiKey + cfg.E2BTemplate = template + cfg.E2BAPIURL = strings.TrimSpace(os.Getenv("E2B_INTEGRATION_API_URL")) + cfg.E2BSandboxDomain = strings.TrimSpace(os.Getenv("E2B_INTEGRATION_SANDBOX_DOMAIN")) + cfg.E2BProxyURL = strings.TrimSpace(os.Getenv("E2B_INTEGRATION_PROXY_URL")) + cfg.E2BSandboxTTL = conformanceTTL + cfg.E2BHTTPTimeout = conformanceHTTPTimeut + cfg.DefaultTimeout = 2 * time.Minute + return cfg +} diff --git a/internal/sandbox/e2b_remote_client.go b/internal/sandbox/e2b_remote_client.go index 5c006658dc..222bbf1ddf 100644 --- a/internal/sandbox/e2b_remote_client.go +++ b/internal/sandbox/e2b_remote_client.go @@ -44,6 +44,31 @@ func NewE2BRemoteClient(cfg *Config) (*E2BRemoteClient, error) { func NewE2BRemoteClientWithTransport( cfg *Config, transport *http.Transport, +) (*E2BRemoteClient, error) { + if transport == nil { + return newE2BRemoteClient(cfg, nil) + } + return newE2BRemoteClient(cfg, transport) +} + +// NewE2BRemoteClientWithPool builds the client on top of the shared gateway +// pool. It is what self-hosted E2B-compatible control planes need: the pool +// keeps control-plane traffic on the process-wide transport while dialling +// data-plane traffic at the configured gateway (see gateway_transport.go). +// A nil pool falls back to the SDK defaults. +func NewE2BRemoteClientWithPool( + cfg *Config, + pool *SandboxGatewayTransportPool, +) (*E2BRemoteClient, error) { + if pool == nil { + return newE2BRemoteClient(cfg, nil) + } + return newE2BRemoteClient(cfg, pool.RoundTripperFor(cfg)) +} + +func newE2BRemoteClient( + cfg *Config, + transport http.RoundTripper, ) (*E2BRemoteClient, error) { if cfg == nil { return nil, errors.New("e2b remote client config is required") @@ -55,9 +80,12 @@ func NewE2BRemoteClientWithTransport( if timeout <= 0 { timeout = DefaultE2BHTTPTimeout } - httpClient := &http.Client{Timeout: timeout} - if transport != nil { - httpClient.Transport = transport + // Every E2B client speaks to envd through the compatibility shim, whether + // or not a gateway is configured: the two details it rewrites belong to the + // envd protocol itself, not to any one deployment. See envd_compat_transport.go. + httpClient := &http.Client{ + Timeout: timeout, + Transport: NewEnvdCompatTransport(transport, DefaultSandboxExecUser), } client, err := e2b.NewClient(e2b.ClientConfig{ APIKey: cfg.E2BAPIKey, @@ -119,11 +147,16 @@ func (c *E2BRemoteClient) Capabilities() RemoteSandboxCapabilities { } } -// Health probes the E2B control plane via ListSandboxes. The SDK does not -// expose a dedicated health endpoint, and ListSandboxes is the smallest -// authenticated call that will detect a bad API key or a dead API. +// Health probes the control plane by listing sandboxes. The protocol has no +// dedicated health endpoint, and a list is the smallest authenticated call +// that detects a bad API key or a dead API. +// +// It deliberately uses the v2 listing rather than the legacy one: v2 is what +// every other call in this client already depends on, and E2B-compatible +// control planes (Agent-Sandbox, for one) implement only that one — probing +// the legacy path would report a perfectly healthy backend as unreachable. func (c *E2BRemoteClient) Health(ctx context.Context) error { - if _, err := c.client.ListSandboxes(ctx); err != nil { + if _, err := c.client.ListSandboxesV2(ctx, e2b.WithSandboxLimit(1)); err != nil { return normalizeE2BError("Health", err) } return nil @@ -280,13 +313,18 @@ func isE2BTemplateBuildPending(status string) bool { } } +// EnsureStandardTemplate returns the cluster's WeKnora template, building it +// when absent. A failed or untagged template is not returned as is: it can +// never spawn a sandbox, so it falls through to the build below. E2B resolves +// the build by name, so that is a rebuild of the same template rather than a +// second entry in the catalog. func (c *E2BRemoteClient) EnsureStandardTemplate(ctx context.Context) (*RemoteTemplate, error) { items, err := c.ListTemplates(ctx) if err != nil { return nil, err } for i := range items { - if items[i].Standard { + if items[i].Standard && !IsTemplateBuildFailed(items[i].Status) { return &items[i], nil } } @@ -744,6 +782,11 @@ func (c *E2BRemoteClient) Exec( }, nil } +// Filesystem operations name DefaultSandboxExecUser explicitly rather than +// relying on the daemon's default account. It keeps ownership aligned with the +// account scripts run as, and it is required for interoperability: E2B Cloud +// falls back to "user" when the request omits it, while other E2B-compatible +// control planes reject the call outright. func (c *E2BRemoteClient) WriteFile( ctx context.Context, handle RemoteSandboxHandle, @@ -757,7 +800,7 @@ func (c *E2BRemoteClient) WriteFile( if strings.TrimSpace(path) == "" { return e2bInvalidRequest("WriteFile", "path is required", nil) } - if _, err := sandbox.Filesystem.WriteBytes(ctx, path, content); err != nil { + if _, err := sandbox.Filesystem.WriteBytes(ctx, path, content, e2b.WithFileUser(DefaultSandboxExecUser)); err != nil { return normalizeE2BError("WriteFile", err) } return nil @@ -775,7 +818,7 @@ func (c *E2BRemoteClient) ReadFile( if strings.TrimSpace(path) == "" { return nil, e2bInvalidRequest("ReadFile", "path is required", nil) } - content, err := sandbox.Filesystem.ReadBytes(ctx, path) + content, err := sandbox.Filesystem.ReadBytes(ctx, path, e2b.WithFileUser(DefaultSandboxExecUser)) if err != nil { return nil, normalizeE2BError("ReadFile", err) } @@ -794,7 +837,7 @@ func (c *E2BRemoteClient) ListDir( if strings.TrimSpace(path) == "" { return nil, e2bInvalidRequest("ListDir", "path is required", nil) } - entries, err := sandbox.Filesystem.List(ctx, path) + entries, err := sandbox.Filesystem.List(ctx, path, e2b.WithFileUser(DefaultSandboxExecUser)) if err != nil { return nil, normalizeE2BError("ListDir", err) } @@ -823,7 +866,7 @@ func (c *E2BRemoteClient) MakeDir( if strings.TrimSpace(path) == "" { return e2bInvalidRequest("MakeDir", "path is required", nil) } - if err := sandbox.Filesystem.MakeDir(ctx, path); err != nil { + if err := sandbox.Filesystem.MakeDir(ctx, path, e2b.WithFileUser(DefaultSandboxExecUser)); err != nil { return normalizeE2BError("MakeDir", err) } return nil @@ -841,7 +884,7 @@ func (c *E2BRemoteClient) Remove( if strings.TrimSpace(path) == "" { return e2bInvalidRequest("Remove", "path is required", nil) } - if err := sandbox.Filesystem.Remove(ctx, path); err != nil { + if err := sandbox.Filesystem.Remove(ctx, path, e2b.WithFileUser(DefaultSandboxExecUser)); err != nil { return normalizeE2BError("Remove", err) } return nil @@ -859,7 +902,7 @@ func (c *E2BRemoteClient) Stat( if strings.TrimSpace(path) == "" { return nil, e2bInvalidRequest("Stat", "path is required", nil) } - info, err := sandbox.Filesystem.Stat(ctx, path) + info, err := sandbox.Filesystem.Stat(ctx, path, e2b.WithFileUser(DefaultSandboxExecUser)) if err != nil { return nil, normalizeE2BError("Stat", err) } diff --git a/internal/sandbox/envd_compat_transport.go b/internal/sandbox/envd_compat_transport.go new file mode 100644 index 0000000000..4f3783c5f7 --- /dev/null +++ b/internal/sandbox/envd_compat_transport.go @@ -0,0 +1,141 @@ +// Package sandbox: envd protocol compatibility for the E2B data plane. +// +// The sandbox-side daemon (envd) authenticates every data-plane call with HTTP +// Basic auth carrying the sandbox account name, and accepts file uploads only +// as multipart/form-data. github.com/matiasinsaurralde/go-e2b predates both: +// it sends the account in an X-User-ID header and POSTs file contents as a raw +// octet-stream body. +// +// E2B Cloud's own gateway is lenient enough to hide the difference, so the gap +// only surfaces against other implementations of the protocol — the very +// backends WeKnora wants to support without carrying one adapter per vendor. +// Rather than fork the SDK, this transport rewrites the two data-plane details +// on the way out: +// +// - it adds "Authorization: Basic base64(user:)" when the request carries no +// credentials of its own; +// - it re-wraps a non-multipart /files upload as multipart/form-data. +// +// Requests are recognised by path, not by host, so the shim works whether the +// data plane is reached directly or through a gateway (gateway_transport.go). +// Control-plane calls never use these paths and pass through untouched. +package sandbox + +import ( + "bytes" + "encoding/base64" + "fmt" + "io" + "mime/multipart" + "net/http" + "path" + "strings" +) + +// envdFilesRoute is envd's upload/download endpoint. +const envdFilesRoute = "/files" + +// envdUploadFormField is the multipart field name envd reads the payload from. +const envdUploadFormField = "file" + +// NewEnvdCompatTransport wraps next so E2B data-plane requests satisfy the +// current envd contract. A blank user leaves authentication untouched. +func NewEnvdCompatTransport(next http.RoundTripper, user string) http.RoundTripper { + if next == nil { + next = http.DefaultTransport + } + return &envdCompatTransport{next: next, user: strings.TrimSpace(user)} +} + +type envdCompatTransport struct { + next http.RoundTripper + user string +} + +func (t *envdCompatTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if !isEnvdDataPlaneRequest(req) { + return t.next.RoundTrip(req) + } + rewritten := req.Clone(req.Context()) + if t.user != "" && rewritten.Header.Get("Authorization") == "" { + rewritten.Header.Set("Authorization", basicAuthorizationFor(t.user)) + } + if err := rewriteEnvdUpload(rewritten); err != nil { + return nil, err + } + return t.next.RoundTrip(rewritten) +} + +func (t *envdCompatTransport) CloseIdleConnections() { + if closer, ok := t.next.(interface{ CloseIdleConnections() }); ok { + closer.CloseIdleConnections() + } +} + +// isEnvdDataPlaneRequest reports whether req addresses envd rather than the +// control plane. envd serves /files plus the ConnectRPC services generated +// from its proto package. +func isEnvdDataPlaneRequest(req *http.Request) bool { + if req == nil || req.URL == nil { + return false + } + requestPath := req.URL.Path + if requestPath == envdFilesRoute { + return true + } + return strings.HasPrefix(requestPath, "/filesystem.Filesystem/") || + strings.HasPrefix(requestPath, "/process.Process/") +} + +// basicAuthorizationFor builds envd's credential: the account name as the +// username with an empty password. +func basicAuthorizationFor(user string) string { + return "Basic " + base64.StdEncoding.EncodeToString([]byte(user+":")) +} + +// rewriteEnvdUpload converts a raw-body upload into the multipart form envd +// expects. Uploads that already are multipart, and every non-upload request, +// are left alone. +func rewriteEnvdUpload(req *http.Request) error { + if req.Method != http.MethodPost || req.URL.Path != envdFilesRoute { + return nil + } + if strings.HasPrefix(req.Header.Get("Content-Type"), "multipart/") { + return nil + } + if req.Body == nil { + return nil + } + payload, err := io.ReadAll(req.Body) + _ = req.Body.Close() + if err != nil { + return fmt.Errorf("sandbox: read envd upload body: %w", err) + } + + filename := path.Base(strings.TrimSpace(req.URL.Query().Get("path"))) + if filename == "" || filename == "." || filename == "/" { + filename = envdUploadFormField + } + + var form bytes.Buffer + writer := multipart.NewWriter(&form) + part, err := writer.CreateFormFile(envdUploadFormField, filename) + if err != nil { + return fmt.Errorf("sandbox: build envd upload form: %w", err) + } + if _, err := part.Write(payload); err != nil { + return fmt.Errorf("sandbox: write envd upload form: %w", err) + } + if err := writer.Close(); err != nil { + return fmt.Errorf("sandbox: close envd upload form: %w", err) + } + + body := form.Bytes() + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.ContentLength = int64(len(body)) + req.Body = io.NopCloser(bytes.NewReader(body)) + req.GetBody = func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(body)), nil + } + return nil +} diff --git a/internal/sandbox/envd_compat_transport_test.go b/internal/sandbox/envd_compat_transport_test.go new file mode 100644 index 0000000000..d06226f4d6 --- /dev/null +++ b/internal/sandbox/envd_compat_transport_test.go @@ -0,0 +1,163 @@ +package sandbox + +import ( + "bytes" + "encoding/base64" + "io" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// recordingRoundTripper captures the request the shim produced. +type recordingRoundTripper struct { + request *http.Request + body []byte +} + +func (r *recordingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + r.request = req + if req.Body != nil { + body, err := io.ReadAll(req.Body) + if err != nil { + return nil, err + } + r.body = body + } + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader("")), + Header: make(http.Header), + Request: req, + }, nil +} + +func TestEnvdCompatTransportAuthenticatesDataPlaneCalls(t *testing.T) { + recorder := &recordingRoundTripper{} + transport := NewEnvdCompatTransport(recorder, "user") + + request := httptest.NewRequest( + http.MethodPost, + "https://49983-sbx.example.com/filesystem.Filesystem/MakeDir", + strings.NewReader("{}"), + ) + _, err := transport.RoundTrip(request) + require.NoError(t, err) + + require.Equal(t, + "Basic "+base64.StdEncoding.EncodeToString([]byte("user:")), + recorder.request.Header.Get("Authorization"), + ) +} + +// A caller that already authenticated must win: the shim fills a gap, it does +// not override credentials. +func TestEnvdCompatTransportKeepsExistingAuthorization(t *testing.T) { + recorder := &recordingRoundTripper{} + transport := NewEnvdCompatTransport(recorder, "user") + + request := httptest.NewRequest( + http.MethodGet, + "https://49983-sbx.example.com/files?path=/workspace/a.txt", + nil, + ) + request.Header.Set("Authorization", "Basic preset") + _, err := transport.RoundTrip(request) + require.NoError(t, err) + + require.Equal(t, "Basic preset", recorder.request.Header.Get("Authorization")) +} + +// Control-plane traffic shares the same transport, so the shim must leave it +// untouched - an unexpected Authorization header there would replace the API +// key the SDK sends. +func TestEnvdCompatTransportIgnoresControlPlaneCalls(t *testing.T) { + recorder := &recordingRoundTripper{} + transport := NewEnvdCompatTransport(recorder, "user") + + request := httptest.NewRequest(http.MethodGet, "https://api.e2b.app/v2/sandboxes", nil) + _, err := transport.RoundTrip(request) + require.NoError(t, err) + + require.Empty(t, recorder.request.Header.Get("Authorization")) +} + +func TestEnvdCompatTransportRewritesUploadAsMultipart(t *testing.T) { + recorder := &recordingRoundTripper{} + transport := NewEnvdCompatTransport(recorder, "user") + + payload := []byte("print('hi')\n") + request := httptest.NewRequest( + http.MethodPost, + "https://49983-sbx.example.com/files?path=/workspace/script.py&username=user", + bytes.NewReader(payload), + ) + request.Header.Set("Content-Type", "application/octet-stream") + _, err := transport.RoundTrip(request) + require.NoError(t, err) + + contentType := recorder.request.Header.Get("Content-Type") + mediaType, params, err := mime.ParseMediaType(contentType) + require.NoError(t, err) + require.Equal(t, "multipart/form-data", mediaType) + require.EqualValues(t, len(recorder.body), recorder.request.ContentLength) + + reader := multipart.NewReader(bytes.NewReader(recorder.body), params["boundary"]) + part, err := reader.NextPart() + require.NoError(t, err) + require.Equal(t, envdUploadFormField, part.FormName()) + require.Equal(t, "script.py", part.FileName()) + content, err := io.ReadAll(part) + require.NoError(t, err) + require.Equal(t, payload, content) +} + +// Uploads that already are multipart must pass through byte-for-byte. +func TestEnvdCompatTransportPreservesMultipartUploads(t *testing.T) { + recorder := &recordingRoundTripper{} + transport := NewEnvdCompatTransport(recorder, "user") + + var form bytes.Buffer + writer := multipart.NewWriter(&form) + part, err := writer.CreateFormFile(envdUploadFormField, "already.txt") + require.NoError(t, err) + _, err = part.Write([]byte("body")) + require.NoError(t, err) + require.NoError(t, writer.Close()) + original := form.Bytes() + + request := httptest.NewRequest( + http.MethodPost, + "https://49983-sbx.example.com/files?path=/workspace/already.txt", + bytes.NewReader(original), + ) + request.Header.Set("Content-Type", writer.FormDataContentType()) + _, err = transport.RoundTrip(request) + require.NoError(t, err) + + require.Equal(t, writer.FormDataContentType(), recorder.request.Header.Get("Content-Type")) + require.Equal(t, original, recorder.body) +} + +// A download shares the /files route with uploads; only the POST body is +// rewritten. +func TestEnvdCompatTransportLeavesDownloadsAlone(t *testing.T) { + recorder := &recordingRoundTripper{} + transport := NewEnvdCompatTransport(recorder, "user") + + request := httptest.NewRequest( + http.MethodGet, + "https://49983-sbx.example.com/files?path=/workspace/a.txt", + nil, + ) + _, err := transport.RoundTrip(request) + require.NoError(t, err) + + require.Empty(t, recorder.request.Header.Get("Content-Type")) + require.Empty(t, recorder.body) +} diff --git a/internal/sandbox/gateway_transport.go b/internal/sandbox/gateway_transport.go new file mode 100644 index 0000000000..90525d26b3 --- /dev/null +++ b/internal/sandbox/gateway_transport.go @@ -0,0 +1,184 @@ +// Package sandbox: connection pooling and data-plane routing for per-request +// remote clients. +// +// Named configs build a fresh client on every Resolve, so without an +// externally owned transport every request would open new TCP connections to +// both the control plane and the envd data plane. +// +// Remote backends speak two planes with different dialling rules: +// +// - control plane (Create/Connect/List) talks to the API URL directly; +// - data plane (exec, filesystem) addresses sandboxes as +// "49983-{id}.{domain}", which E2B Cloud resolves through public DNS and +// TLS. Self-hosted E2B-compatible control planes (CubeSandbox's CubeProxy, +// Agent-Sandbox's gateway, e2b-dev/infra's client proxy) instead front +// every sandbox with one gateway address and route on the Host header. +// +// Handing an SDK one http.Client for both planes drops that distinction, which +// only appears to work when DNS happens to resolve the sandbox domain to the +// gateway on the same port. This file keeps the two planes apart by routing per +// request: control traffic rides the process-wide transport shared by every +// backend, data traffic rides a transport cached per gateway endpoint so +// configs pointing at the same gateway share one pool. +// +// The gateway may also be plain HTTP. Both SDKs pin the data-plane scheme to +// https, so a http:// gateway URL additionally rewrites the request scheme +// rather than forcing operators to terminate TLS in front of a local cluster. +package sandbox + +import ( + "context" + "net" + "net/http" + "strconv" + "strings" + "sync" + "time" +) + +// SandboxGatewayTransportPool owns the transports handed to per-request remote +// clients. One instance lives for the process; clients built from it come and +// go. +type SandboxGatewayTransportPool struct { + control http.RoundTripper + policy OutboundURLPolicy + + // data maps a gateway "host:port" to the transport that dials it. + data sync.Map +} + +// NewSandboxGatewayTransportPool returns a pool whose control plane rides +// control. A nil control transport installs a guarded one. +func NewSandboxGatewayTransportPool(control http.RoundTripper) *SandboxGatewayTransportPool { + return NewSandboxGatewayTransportPoolWithPolicy(control, DefaultOutboundURLPolicy()) +} + +func NewSandboxGatewayTransportPoolWithPolicy( + control http.RoundTripper, + policy OutboundURLPolicy, +) *SandboxGatewayTransportPool { + if control == nil { + control = NewGuardedTransportWithPolicy(policy) + } + return &SandboxGatewayTransportPool{control: control, policy: policy} +} + +// RoundTripperFor returns the transport a client built from cfg should use. +// Configs without a usable gateway URL keep every request on the control +// transport, matching the SDKs' behaviour when no gateway is configured. +func (p *SandboxGatewayTransportPool) RoundTripperFor(cfg *Config) http.RoundTripper { + gatewayURL, sandboxDomain := gatewayEndpointFor(cfg) + split := &gatewaySplitTransport{ + control: p.control, + sandboxDomain: strings.ToLower(strings.TrimSpace(sandboxDomain)), + } + if host, port, scheme, ok := parseProxyURL(gatewayURL); ok { + split.data = p.dataTransport(net.JoinHostPort(host, strconv.Itoa(port))) + split.dataScheme = scheme + } + return split +} + +// gatewayEndpointFor reads the active provider's data-plane fields. Reading +// them per provider (rather than merging both) keeps a stale sub-struct left +// behind by an earlier provider switch from routing today's traffic. +func gatewayEndpointFor(cfg *Config) (gatewayURL, sandboxDomain string) { + if cfg == nil { + return "", "" + } + switch cfg.Type { + case SandboxTypeE2B: + return cfg.E2BProxyURL, cfg.E2BSandboxDomain + default: + return cfg.CubeProxyURL, cfg.CubeSandboxDomain + } +} + +// dataTransport returns the transport dialling target, creating it once. +func (p *SandboxGatewayTransportPool) dataTransport(target string) http.RoundTripper { + if existing, ok := p.data.Load(target); ok { + return existing.(http.RoundTripper) + } + actual, _ := p.data.LoadOrStore(target, newGatewayDataTransportWithPolicy(target, p.policy)) + return actual.(http.RoundTripper) +} + +// newGatewayDataTransport dials target regardless of the request's authority, +// mirroring the SDK's proxy rewrite while adding the outbound address guard +// the SDKs have no notion of. +func newGatewayDataTransport(target string) *http.Transport { + return newGatewayDataTransportWithPolicy(target, DefaultOutboundURLPolicy()) +} + +func newGatewayDataTransportWithPolicy(target string, policy OutboundURLPolicy) *http.Transport { + dialer := &net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 30 * time.Second, + Control: SafeDialControlForPolicy(policy), + } + return &http.Transport{ + // The gateway is addressed directly; an ambient HTTP proxy would + // defeat the rewrite. + Proxy: nil, + DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { + return dialer.DialContext(ctx, network, target) + }, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 4, + IdleConnTimeout: 90 * time.Second, + } +} + +// gatewaySplitTransport routes a request to the control or the data transport +// by looking at the authority the SDK addressed. +type gatewaySplitTransport struct { + control http.RoundTripper + data http.RoundTripper + sandboxDomain string + + // dataScheme is the gateway's scheme. When it differs from the scheme the + // SDK hardcoded, data-plane requests are rewritten before dialling. + dataScheme string +} + +func (t *gatewaySplitTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if t.data == nil || !t.isDataPlane(req.URL.Hostname()) { + return t.control.RoundTrip(req) + } + return t.data.RoundTrip(t.applyGatewayScheme(req)) +} + +// applyGatewayScheme returns req addressed with the gateway's scheme. The +// sandbox authority is preserved so the gateway can still route on Host. +func (t *gatewaySplitTransport) applyGatewayScheme(req *http.Request) *http.Request { + if t.dataScheme == "" || t.dataScheme == req.URL.Scheme { + return req + } + rewritten := req.Clone(req.Context()) + url := *req.URL + url.Scheme = t.dataScheme + rewritten.URL = &url + return rewritten +} + +// isDataPlane reports whether host addresses a sandbox rather than the control +// plane. Anything else - including an unset sandbox domain - stays on the +// control transport, so a misconfiguration cannot silently redirect API calls +// at the gateway. +func (t *gatewaySplitTransport) isDataPlane(host string) bool { + if t.sandboxDomain == "" { + return false + } + host = strings.ToLower(host) + return host == t.sandboxDomain || strings.HasSuffix(host, "."+t.sandboxDomain) +} + +// CloseIdleConnections keeps the SDK's post-rollback reset meaningful. Only +// the data pool is dropped: the control transport is shared with every other +// tenant and every other backend, and one sandbox's restart is no reason to +// close it. +func (t *gatewaySplitTransport) CloseIdleConnections() { + if closer, ok := t.data.(interface{ CloseIdleConnections() }); ok { + closer.CloseIdleConnections() + } +} diff --git a/internal/sandbox/cube_transport_test.go b/internal/sandbox/gateway_transport_test.go similarity index 61% rename from internal/sandbox/cube_transport_test.go rename to internal/sandbox/gateway_transport_test.go index af465deb77..db09985e0b 100644 --- a/internal/sandbox/cube_transport_test.go +++ b/internal/sandbox/gateway_transport_test.go @@ -35,7 +35,7 @@ func (c *countingRoundTripper) seen() []string { // the configured proxy. Sharing one transport across both planes drops that // rewrite, which is exactly the regression this guards: the proxy has to see // the request, and it has to see the sandbox authority in the Host header. -func TestCubeTransportPoolRoutesDataPlaneThroughProxy(t *testing.T) { +func TestSandboxGatewayTransportPoolRoutesDataPlaneThroughProxy(t *testing.T) { api := newCubeMockServer(t) var mu sync.Mutex @@ -54,7 +54,7 @@ func TestCubeTransportPoolRoutesDataPlaneThroughProxy(t *testing.T) { policy := OutboundURLPolicy{AllowPrivate: true} control := &countingRoundTripper{next: NewGuardedTransportWithPolicy(policy)} - client, err := NewCubeRemoteClientWithPool(cfg, NewCubeTransportPoolWithPolicy(control, policy)) + client, err := NewCubeRemoteClientWithPool(cfg, NewSandboxGatewayTransportPoolWithPolicy(control, policy)) require.NoError(t, err) ctx := context.Background() @@ -90,8 +90,8 @@ func TestCubeTransportPoolRoutesDataPlaneThroughProxy(t *testing.T) { // Configs pointing at the same proxy must share one pool - otherwise building // a client per request pools nothing. -func TestCubeTransportPoolReusesTransportPerProxyEndpoint(t *testing.T) { - pool := NewCubeTransportPoolWithPolicy( +func TestSandboxGatewayTransportPoolReusesTransportPerProxyEndpoint(t *testing.T) { + pool := NewSandboxGatewayTransportPoolWithPolicy( NewGuardedTransportWithPolicy(OutboundURLPolicy{AllowPrivate: true}), OutboundURLPolicy{AllowPrivate: true}, ) @@ -99,15 +99,15 @@ func TestCubeTransportPoolReusesTransportPerProxyEndpoint(t *testing.T) { first := pool.RoundTripperFor(&Config{ CubeProxyURL: "http://127.0.0.1:8080", CubeSandboxDomain: "cube.app", - }).(*cubeSplitTransport) + }).(*gatewaySplitTransport) second := pool.RoundTripperFor(&Config{ CubeProxyURL: "http://127.0.0.1:8080", CubeSandboxDomain: "cube.app", - }).(*cubeSplitTransport) + }).(*gatewaySplitTransport) other := pool.RoundTripperFor(&Config{ CubeProxyURL: "http://127.0.0.1:9090", CubeSandboxDomain: "cube.app", - }).(*cubeSplitTransport) + }).(*gatewaySplitTransport) require.Same(t, first.data, second.data) require.NotSame(t, first.data, other.data) @@ -116,17 +116,60 @@ func TestCubeTransportPoolReusesTransportPerProxyEndpoint(t *testing.T) { // Without a usable proxy URL the SDK dials the sandbox authority directly, so // the split transport must not invent a data plane. -func TestCubeTransportPoolWithoutProxyKeepsEverythingOnControl(t *testing.T) { - pool := NewCubeTransportPool(NewGuardedTransport()) +func TestSandboxGatewayTransportPoolWithoutProxyKeepsEverythingOnControl(t *testing.T) { + pool := NewSandboxGatewayTransportPool(NewGuardedTransport()) - split := pool.RoundTripperFor(&Config{CubeSandboxDomain: "cube.app"}).(*cubeSplitTransport) + split := pool.RoundTripperFor(&Config{CubeSandboxDomain: "cube.app"}).(*gatewaySplitTransport) // A nil data transport is what sends sandbox authorities back to control. require.Nil(t, split.data) } -func TestCubeSplitTransportClassifiesAuthorities(t *testing.T) { - split := &cubeSplitTransport{ +// A self-hosted E2B-compatible control plane fronts every sandbox with one +// gateway, so the E2B provider must read its own gateway fields - and a plain +// HTTP gateway has to survive the SDK pinning the data-plane scheme to https. +func TestSandboxGatewayTransportPoolRoutesE2BDataPlane(t *testing.T) { + pool := NewSandboxGatewayTransportPoolWithPolicy( + NewGuardedTransportWithPolicy(OutboundURLPolicy{AllowPrivate: true}), + OutboundURLPolicy{AllowPrivate: true}, + ) + + split := pool.RoundTripperFor(&Config{ + Type: SandboxTypeE2B, + E2BProxyURL: "http://127.0.0.1:18080", + E2BSandboxDomain: "localhost", + // Cube fields must be ignored for an E2B config. + CubeProxyURL: "http://127.0.0.1:9999", + CubeSandboxDomain: "cube.app", + }).(*gatewaySplitTransport) + + require.NotNil(t, split.data) + require.Equal(t, "http", split.dataScheme) + require.True(t, split.isDataPlane("49983-sbx.localhost")) + require.False(t, split.isDataPlane("49983-sbx.cube.app")) +} + +func TestGatewaySplitTransportAppliesGatewayScheme(t *testing.T) { + recorder := &recordingRoundTripper{} + split := &gatewaySplitTransport{ + control: &recordingRoundTripper{}, + data: recorder, + sandboxDomain: "localhost", + dataScheme: "http", + } + + request := httptest.NewRequest(http.MethodGet, "https://49983-sbx.localhost/files", nil) + _, err := split.RoundTrip(request) + require.NoError(t, err) + + require.Equal(t, "http", recorder.request.URL.Scheme) + // The sandbox authority is what the gateway routes on; it must survive. + require.Equal(t, "49983-sbx.localhost", recorder.request.URL.Host) + require.Equal(t, "https", request.URL.Scheme, "the caller's request must not be mutated") +} + +func TestGatewaySplitTransportClassifiesAuthorities(t *testing.T) { + split := &gatewaySplitTransport{ control: NewGuardedTransport(), data: NewGuardedTransport(), sandboxDomain: "cube.app", diff --git a/internal/sandbox/sandbox.go b/internal/sandbox/sandbox.go index 4925962ba3..e1ae03c7f9 100644 --- a/internal/sandbox/sandbox.go +++ b/internal/sandbox/sandbox.go @@ -47,6 +47,25 @@ const ( DefaultCPULimit = 1.0 // 1 CPU core DefaultDockerImage = "wechatopenai/weknora-sandbox:latest" + // DefaultCubeTemplateImage is the same environment with Cube's envd daemon + // baked in (target "cube" of docker/Dockerfile.sandbox). + // + // Cube turns an OCI image into a template directly and gates the build on + // GET :49983/health, which only envd answers. Building a Cube template from + // DefaultDockerImage therefore always fails the probe with "connection + // refused" — E2B gets away with that image because its own builder injects + // envd, and the Docker backend never needs one. + DefaultCubeTemplateImage = "wechatopenai/weknora-sandbox:latest-cube" + + // CubeEnvdPort is the port envd listens on inside a Cube sandbox. It carries + // the readiness probe as well as every exec and filesystem call, and the + // data plane addresses sandboxes as "49983-{id}.{domain}". + CubeEnvdPort = 49983 + + // CubeEnvdHealthPath is the envd endpoint Cube probes to decide whether a + // template build succeeded. + CubeEnvdHealthPath = "/health" + // DefaultCubeAPIURL is retained for SDK tests and explicit local helpers; // workspace configs must still provide their endpoint. DefaultCubeAPIURL = "http://127.0.0.1:33000" @@ -259,6 +278,12 @@ type Config struct { // "e2b.app". Empty defaults to the SDK's built-in. E2BSandboxDomain string + // E2BProxyURL is the data-plane gateway that fronts envd for self-hosted + // E2B-compatible control planes. Empty keeps the SDK's behaviour of + // resolving the sandbox authority through DNS over TLS, which is what E2B + // Cloud expects. See types.E2BSandboxConfig.ProxyURL. + E2BProxyURL string + // E2BTemplate is the E2B template ID used at sandbox creation. E2BTemplate string diff --git a/internal/sandbox/template_catalog.go b/internal/sandbox/template_catalog.go index a0444353be..7e7c00447f 100644 --- a/internal/sandbox/template_catalog.go +++ b/internal/sandbox/template_catalog.go @@ -28,6 +28,10 @@ type RemoteTemplate struct { CreatedAt string `json:"created_at,omitempty"` UpdatedAt string `json:"updated_at,omitempty"` Standard bool `json:"standard"` + // Error carries the provider's own explanation for a failed build. Without + // it a failed template is a red badge with no way to tell a registry + // credential problem from an out-of-disk node. + Error string `json:"error,omitempty"` } // RemoteTemplateCatalog is an optional provider capability used by the @@ -46,3 +50,53 @@ func isStandardTemplate(name string) bool { parts := strings.Split(trimmed, "/") return len(parts) > 1 && strings.EqualFold(parts[len(parts)-1], StandardTemplateName) } + +// isStandardTemplateImage recognises our template by the image it was built +// from. Names are the primary key, but a provider that drops them — Cube omits +// the field entirely when a template carries no alias — would otherwise make +// every catalog refresh look at a cluster with no standard template and build +// yet another one. +func isStandardTemplateImage(image string) bool { + candidate := normalizeImageRepository(image) + return candidate != "" && candidate == normalizeImageRepository(DefaultDockerImage) +} + +// normalizeImageRepository reduces an image reference to its repository path so +// that "docker.io/wechatopenai/weknora-sandbox:latest", +// "wechatopenai/weknora-sandbox@sha256:…" and the bare name all compare equal. +func normalizeImageRepository(image string) string { + ref := strings.TrimSpace(image) + if ref == "" { + return "" + } + if at := strings.Index(ref, "@"); at >= 0 { + ref = ref[:at] + } + // A colon before the last slash belongs to a registry port, not a tag. + if colon := strings.LastIndex(ref, ":"); colon > strings.LastIndex(ref, "/") { + ref = ref[:colon] + } + ref = strings.Trim(ref, "/") + parts := strings.Split(ref, "/") + // Registry hosts are recognisable by a dot, a port, or being "localhost"; + // anything else at the head is a namespace we must keep. + if len(parts) > 1 && (strings.ContainsAny(parts[0], ".:") || parts[0] == "localhost") { + parts = parts[1:] + } + if len(parts) > 1 && strings.EqualFold(parts[0], "library") { + parts = parts[1:] + } + return strings.ToLower(strings.Join(parts, "/")) +} + +// IsTemplateBuildFailed reports whether a template's build ended in a state no +// amount of waiting will improve. Such a template must be rebuilt rather than +// treated as an existing standard template. +func IsTemplateBuildFailed(status string) bool { + switch strings.ToLower(strings.TrimSpace(status)) { + case "failed", "failure", "error", "cancelled", "canceled", TemplateStatusUntagged: + return true + default: + return false + } +} diff --git a/internal/sandbox/tenant_config.go b/internal/sandbox/tenant_config.go index 567eddc6db..e1d1ac710d 100644 --- a/internal/sandbox/tenant_config.go +++ b/internal/sandbox/tenant_config.go @@ -80,6 +80,9 @@ func ResolveEffectiveConfig( if err := overrideURL(&effective.E2BAPIURL, e2bCfg.APIURL, effective.AllowPrivateEndpoints); err != nil { return nil, err } + if err := overrideURL(&effective.E2BProxyURL, e2bCfg.ProxyURL, effective.AllowPrivateEndpoints); err != nil { + return nil, err + } overrideString(&effective.E2BSandboxDomain, e2bCfg.SandboxDomain) overrideString(&effective.E2BAPIKey, e2bCfg.APIKey) overrideString(&effective.E2BTemplate, e2bCfg.TemplateID) @@ -121,6 +124,7 @@ func clearProviderFields(cfg *Config) { cfg.CubeHTTPTimeout = 0 cfg.E2BAPIURL = "" + cfg.E2BProxyURL = "" cfg.E2BSandboxDomain = "" cfg.E2BAPIKey = "" cfg.E2BTemplate = "" diff --git a/internal/sandbox/tenant_resolver.go b/internal/sandbox/tenant_resolver.go index 5e4c956863..69d2ed8cee 100644 --- a/internal/sandbox/tenant_resolver.go +++ b/internal/sandbox/tenant_resolver.go @@ -18,7 +18,7 @@ // That left the construction-time Health probe as the only real cost, which // SkipHealthProbe removes. Connection reuse is preserved by sharing one // http.Transport across tenants (Cube additionally routes its data plane -// through CubeTransportPool; see cube_transport.go). The upshot: no cache, no +// through SandboxGatewayTransportPool; see gateway_transport.go). The upshot: no cache, no // eviction, no invalidation plumbing, and a config change takes effect on the // next request. package sandbox @@ -95,10 +95,10 @@ type tenantSandboxResolver struct { transport *http.Transport privateTransport *http.Transport - // cubeTransports must outlive the per-request clients it serves, which is + // gatewayTransports must outlive the per-request clients it serves, which is // the whole point of holding it here rather than building it per Resolve. - cubeTransports *CubeTransportPool - privateCubeTransports *CubeTransportPool + gatewayTransports *SandboxGatewayTransportPool + privateGatewayTransports *SandboxGatewayTransportPool } // NewTenantSandboxResolver validates the wiring and returns a resolver. @@ -120,11 +120,11 @@ func NewTenantSandboxResolver(deps TenantSandboxResolverDeps) (TenantSandboxReso transport = NewGuardedTransport() } return &tenantSandboxResolver{ - deps: deps, - transport: transport, - privateTransport: NewGuardedTransportWithPolicy(OutboundURLPolicy{AllowPrivate: true}), - cubeTransports: NewCubeTransportPool(transport), - privateCubeTransports: NewCubeTransportPoolWithPolicy(nil, OutboundURLPolicy{AllowPrivate: true}), + deps: deps, + transport: transport, + privateTransport: NewGuardedTransportWithPolicy(OutboundURLPolicy{AllowPrivate: true}), + gatewayTransports: NewSandboxGatewayTransportPool(transport), + privateGatewayTransports: NewSandboxGatewayTransportPoolWithPolicy(nil, OutboundURLPolicy{AllowPrivate: true}), }, nil } @@ -210,14 +210,17 @@ func (r *tenantSandboxResolver) buildClient(cfg *Config) (RemoteSandboxClient, e switch cfg.Type { case SandboxTypeCube: if cfg.AllowPrivateEndpoints { - return NewCubeRemoteClientWithPool(cfg, r.privateCubeTransports) + return NewCubeRemoteClientWithPool(cfg, r.privateGatewayTransports) } - return NewCubeRemoteClientWithPool(cfg, r.cubeTransports) + return NewCubeRemoteClientWithPool(cfg, r.gatewayTransports) case SandboxTypeE2B: + // The gateway pool is used even without a gateway URL: it then keeps + // every request on the shared control transport, which is exactly what + // a plain E2B Cloud config wants. if cfg.AllowPrivateEndpoints { - return NewE2BRemoteClientWithTransport(cfg, r.privateTransport) + return NewE2BRemoteClientWithPool(cfg, r.privateGatewayTransports) } - return NewE2BRemoteClientWithTransport(cfg, r.transport) + return NewE2BRemoteClientWithPool(cfg, r.gatewayTransports) default: return nil, fmt.Errorf("sandbox: provider %q has no remote client", cfg.Type) } @@ -237,10 +240,13 @@ func NewRemoteClientForCheck(cfg *Config) (RemoteSandboxClient, error) { } switch cfg.Type { case SandboxTypeCube: - return NewCubeRemoteClientWithPool(cfg, NewCubeTransportPoolWithPolicy(nil, + return NewCubeRemoteClientWithPool(cfg, NewSandboxGatewayTransportPoolWithPolicy(nil, OutboundURLPolicy{AllowPrivate: cfg.AllowPrivateEndpoints})) case SandboxTypeE2B: - return NewE2BRemoteClientWithTransport(cfg, NewGuardedTransportWithPolicy( + // Probing through the gateway pool is what makes the check meaningful + // for a self-hosted control plane: it exercises the same data-plane + // routing the resolved manager will use. + return NewE2BRemoteClientWithPool(cfg, NewSandboxGatewayTransportPoolWithPolicy(nil, OutboundURLPolicy{AllowPrivate: cfg.AllowPrivateEndpoints})) default: return nil, fmt.Errorf("sandbox: provider %q cannot be probed", cfg.Type) diff --git a/internal/types/tenant.go b/internal/types/tenant.go index 452a26a74b..037ff47c95 100644 --- a/internal/types/tenant.go +++ b/internal/types/tenant.go @@ -676,15 +676,27 @@ type CubeSandboxConfig struct { CubeSandboxTTLSeconds int `json:"cube_sandbox_ttl_seconds,omitempty"` } -// E2BSandboxConfig addresses one E2B account. APIKey and TemplateID are -// required; APIURL and SandboxDomain are optional because go-e2b resolves both -// on its own when they are empty. +// E2BSandboxConfig addresses one E2B-protocol control plane: E2B Cloud, a +// self-hosted E2B Infrastructure, or any E2B-compatible implementation +// (CubeSandbox, Agent-Sandbox, …). APIKey and TemplateID are required; APIURL +// and SandboxDomain are optional because go-e2b resolves both on its own when +// they are empty. type E2BSandboxConfig struct { APIURL string `json:"api_url,omitempty"` SandboxDomain string `json:"sandbox_domain,omitempty"` APIKey string `json:"api_key,omitempty"` // 加密 TemplateID string `json:"template_id,omitempty"` + // ProxyURL is the data-plane gateway that fronts envd. E2B Cloud resolves + // "-." through public DNS and TLS, so it + // needs no value here. Self-hosted E2B-compatible control planes usually + // serve every sandbox from one gateway address and expect the sandbox + // authority in the Host header; setting this makes WeKnora dial the + // gateway directly instead of requiring wildcard DNS and a certificate + // for the sandbox domain. An "http://" gateway also downgrades the + // data-plane scheme, which the E2B SDK otherwise pins to https. + ProxyURL string `json:"proxy_url,omitempty"` + // HTTPTimeoutSec bounds each HTTP call to the sandbox control plane. // 0 means use the built-in default (30s), never the deployment's value. HTTPTimeoutSec int `json:"http_timeout_sec,omitempty"` diff --git a/scripts/build_images.sh b/scripts/build_images.sh index b448d86570..47a8f0bbcc 100755 --- a/scripts/build_images.sh +++ b/scripts/build_images.sh @@ -215,14 +215,32 @@ build_sandbox_image() { docker build \ --platform $PLATFORM \ -f docker/Dockerfile.sandbox \ + --target sandbox \ -t wechatopenai/weknora-sandbox:latest \ . + if [ $? -ne 0 ]; then + log_error "沙箱镜像构建失败" + return 1 + fi + + # Cube 从镜像直接构建模板，并以 :49983/health 探活，缺 envd 必然失败， + # 因此 Cube 用的是注入了 envd 的变体镜像。详见 docs/sandbox-cluster.md。 + # 固定 linux/amd64：envd 的来源镜像 cubesandbox-base 不发布 arm64。 + log_info "构建沙箱镜像 Cube 变体 (weknora-sandbox:latest-cube)..." + + docker build \ + --platform linux/amd64 \ + -f docker/Dockerfile.sandbox \ + --target cube \ + -t wechatopenai/weknora-sandbox:latest-cube \ + . + if [ $? -eq 0 ]; then log_success "沙箱镜像构建成功" return 0 else - log_error "沙箱镜像构建失败" + log_error "沙箱镜像 Cube 变体构建失败" return 1 fi } @@ -310,6 +328,7 @@ clean_images() { docker rmi wechatopenai/weknora-docreader:latest 2>/dev/null || true docker rmi wechatopenai/weknora-ui:latest 2>/dev/null || true docker rmi wechatopenai/weknora-sandbox:latest 2>/dev/null || true + docker rmi wechatopenai/weknora-sandbox:latest-cube 2>/dev/null || true docker image prune -f diff --git a/website-docs/03-features/07-agent.md b/website-docs/03-features/07-agent.md index 1182eef37c..85d88acec1 100644 --- a/website-docs/03-features/07-agent.md +++ b/website-docs/03-features/07-agent.md @@ -408,7 +408,7 @@ Agent 侧的启停在 `configureSkillsFromAgent`（`internal/application/service ### 5.3 与沙箱（internal/sandbox）的关系 -`execute_skill_script` → `skills.Manager.ExecuteScript` → `sandbox.Manager.Execute`。Docker、Local、CubeSandbox、E2B 均通过「设置 → 沙箱后端」的同一套空间配置与检查接口维护；远端模板从目标 Cube/E2B 集群实时拉取，缺少 WeKnora 标准模板时自动创建。Docker/Local 每次独立执行，不写入会话沙箱绑定。 +`execute_skill_script` → `skills.Manager.ExecuteScript` → `sandbox.Manager.Execute`。Docker、Local、CubeSandbox、E2B 均通过「设置 → 沙箱后端」的同一套空间配置与检查接口维护；远端模板从目标集群实时拉取，缺少 WeKnora 标准模板时自动创建。Docker/Local 每次独立执行，不写入会话沙箱绑定，也不提供 shell_exec、附件暂存与产物收集，仅适合本机开发调试。生产环境使用 E2B 协议后端：E2B Cloud、CubeSandbox，或任意 E2B 兼容控制面，接入方式见 `docs/sandbox-protocol.md`。 **Manager 与校验器**（`internal/sandbox/manager.go`、`validator.go`）：每次执行前，除非 `SkipValidation`，`ScriptValidator` 会做四类静态校验，任一命中即拒绝执行并返回 `ErrSecurityViolation`：