From adf77fe0ef8d12b69d0cda45cb764a86664eba86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iwo=20Strzebo=C5=84ski?= Date: Tue, 20 Oct 2026 08:57:37 +0000 Subject: [PATCH] Add an e2e suite that runs against the real external services The other e2e scripts stand in for S3, TURN and the auth issuer, and the stand-ins hide the failures that matter. e2e-authz.sh presigns with the literal credentials "e2e"/"e2e" and never sends the URL anywhere - but presigning is local HMAC, so it succeeds against any key, and a wrong endpoint, a missing bucket or a revoked token all pass it. The failure that motivated this already happened once: a gateway downloaded S3's XML 403 error document and handed it to a printer as a project file. So this one talks to the live devstack: a presigned PUT/HEAD/GET round-trip, a tampered signature answering the XML error document download.rs exists to refuse, Cloudflare minting ICE credentials for our key id, and a real Supabase user's ES256 token verified against the project's published JWKS before and after tampering, then spent on a route of the booted backend. The first draft of this script announced PASS while printing nothing: it captured results with `grep -E "^(ok|FAILED)\t"`, and POSIX ERE has no \t escape, so every check was silently dropped and FAILURES stayed 0. run_probe now matches a real tab, and a probe that returns fewer results than expected is itself a failure with its raw output shown - a check that cannot fail is worse than no check. Both modes verified: a nonexistent bucket and a probe that will not import each exit 1. Credentials come from the environment and are never printed; the devstack's live in the workspace root, outside every repo. --- scripts/e2e-real-services.sh | 237 +++++++++++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100755 scripts/e2e-real-services.sh diff --git a/scripts/e2e-real-services.sh b/scripts/e2e-real-services.sh new file mode 100755 index 0000000..363105e --- /dev/null +++ b/scripts/e2e-real-services.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash +# End-to-end test of the three external services the other e2e scripts fake, +# against the real thing. +# +# This is the test the others cannot be. e2e-authz.sh signs presigned URLs +# with the literal credentials "e2e"/"e2e" and never sends them anywhere, +# because presigning is local HMAC and succeeds against any key - so a wrong +# endpoint, a missing bucket or a revoked token all still pass. The failure +# that motivated this script already happened once: a gateway downloading +# S3's XML 403 error document and handing it to a printer as a project file. +# Likewise it authenticates against a local dev issuer, which proves the role +# ladder but never exercises a real JWKS fetch or ES256. +# +# What it proves, each against live infrastructure: +# 1. S3 - a presigned PUT, HEAD and GET round-trip real bytes, and a +# tampered signature yields the XML error document that +# continuum-proxy's download.rs refuses to treat as a print. +# 2. TURN - Cloudflare mints short-lived ICE credentials for our key id. +# 3. Auth - a real Supabase user's ES256 access token verifies against the +# project's published JWKS and opens an authenticated route, +# while an unsigned request does not. +# +# Credentials come from the environment and are never printed. This repo holds +# none of them: /home/coder/workspace/.env holds the devstack's, and +# ../dev-secrets.sh maps them onto the names read here. +# +# Usage: +# scripts/dev-env.sh up && scripts/dev-env.sh schema +# set -a; . ../dev-secrets.sh; set +a +# scripts/e2e-real-services.sh +# +# SKIP_AUTH=1 skip section 3 (it creates a user in the Supabase project) +set -euo pipefail +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +PORT="${PORT:-3031}" +FAILURES=0 +fail() { echo "FAIL: $*" >&2; exit 1; } +ok() { echo " ok: $*"; } +bad() { echo " FAILED: $*" >&2; FAILURES=$((FAILURES + 1)); } + +need() { + for name in "$@"; do + [ -n "${!name:-}" ] || fail "$name is not set - source ../dev-secrets.sh first (see the header)" + done +} +need R2_ACCESS_KEY_ID R2_SECRET_ACCESS_KEY R2_BUCKET DATABASE_URL VALKEY_URL +[ -n "${R2_ENDPOINT:-}" ] || [ -n "${R2_ACCOUNT_ID:-}" ] || fail "neither R2_ENDPOINT nor R2_ACCOUNT_ID is set" + +# Runs a TypeScript probe and reports each `oklabel` line it printed. +# +# The "at least this many results" argument is not defensive padding. The +# first version of this script grepped for `^(ok|FAILED)\t`, and POSIX ERE has +# no `\t` escape - so it matched nothing, every section printed no results, +# and the script announced PASS. A check that cannot fail is worse than no +# check, so a probe that goes quiet is now a failure and its raw output is +# printed. +run_probe() { + local name="$1" expected="$2" src="$3" out seen=0 verdict label + cp "$src" "./.e2e-probe.ts" + out="$(bun run ./.e2e-probe.ts 2>&1 || true)" + rm -f "./.e2e-probe.ts" + while IFS=$'\t' read -r verdict label; do + case "$verdict" in + ok) ok "$label"; seen=$((seen + 1)) ;; + FAILED) bad "$label"; seen=$((seen + 1)) ;; + esac + done < <(printf '%s\n' "$out" | grep -E "^(ok|FAILED)$(printf '\t')") + if [ "$seen" -lt "$expected" ]; then + bad "$name produced $seen of $expected expected results - the probe itself failed" + printf '%s\n' "$out" | sed 's/^/ | /' >&2 + fi +} + +WORK="$(mktemp -d)" +BACKEND_PID="" +cleanup() { + [ -n "$BACKEND_PID" ] && kill "$BACKEND_PID" 2>/dev/null || true + rm -rf "$WORK" +} +trap cleanup EXIT + +echo "==> 1. object storage: the presign path, against a live endpoint" +cat > "$WORK/s3.ts" <<'TS' +import { HeadBucketCommand } from "@aws-sdk/client-s3"; +import { s3, getUploadUrl, getDownloadUrl, headObject, jobFileKey } from "./src/s3"; + +const expect = (label: string, cond: boolean, detail = "") => { + console.log(`${cond ? "ok" : "FAILED"}\t${label}${detail ? ` (${detail})` : ""}`); + if (!cond) process.exitCode = 1; +}; + +try { + await s3.send(new HeadBucketCommand({ Bucket: process.env.R2_BUCKET! })); + expect(`bucket '${process.env.R2_BUCKET}' exists and the credentials may read it`, true); +} catch (err) { + expect(`bucket '${process.env.R2_BUCKET}' reachable`, false, (err as Error).name); + process.exit(1); +} + +const key = jobFileKey("e2e-farm", crypto.randomUUID(), "verify.3mf"); +const payload = `continuum e2e ${new Date().toISOString()}\n`; +const body = new TextEncoder().encode(payload); + +const put = await getUploadUrl(key, "application/octet-stream", 300); +const up = await fetch(put, { method: "PUT", body, headers: { "content-type": "application/octet-stream" } }); +expect("a presigned PUT uploads real bytes", up.ok, `HTTP ${up.status}`); +if (!up.ok) process.exit(1); + +const head = await headObject(key); +expect("headObject reports the size the scheduler checks before dispatching", head?.size === body.length, `got ${head?.size}, want ${body.length}`); + +const down = await fetch(await getDownloadUrl(key, 300)); +const text = await down.text(); +expect("a presigned GET returns those bytes unchanged", down.ok && text === payload, `HTTP ${down.status}`); + +expect("a missing object is undefined, not a throw", (await headObject(`${key}.nope`)) === undefined); + +// The documented trap: a bad signature answers 200-shaped XML, not an error +// the HTTP layer notices. continuum-proxy/src/uplink/download.rs checks the +// status before the body for exactly this reason. +const bad = await fetch(put.replace(/Signature=[^&]+/, "Signature=deadbeef"), { method: "PUT", body }); +const badBody = await bad.text(); +expect("a tampered signature is refused with an XML error document", !bad.ok && badBody.includes(" 2. TURN: Cloudflare mints ICE credentials for this key id" +if [ -z "${CLOUDFLARE_TURN_KEY_ID:-}" ] || [ -z "${CLOUDFLARE_TURN_API_TOKEN:-}" ]; then + ok "skipped - TURN is unconfigured, which the API reports rather than failing on" +else + cat > "$WORK/turn.ts" <<'TS' +import { turnConfigured, mintIceServers } from "./src/camera/ice"; +const servers = await mintIceServers(); +// Never print the credentials themselves: they are live for an hour. +const relay = (servers ?? []).find((s) => { + const urls = Array.isArray(s.urls) ? s.urls : [s.urls]; + return urls.some((u) => u.startsWith("turn:") || u.startsWith("turns:")); +}); +console.log(`${turnConfigured() ? "ok" : "FAILED"}\tTURN reports itself configured`); +console.log(`${servers && servers.length > 0 ? "ok" : "FAILED"}\tCloudflare minted ${servers?.length ?? 0} ICE server entr(ies)`); +console.log(`${relay?.username && relay?.credential ? "ok" : "FAILED"}\ta relay entry carries a username and credential`); +TS + run_probe "TURN" 3 "$WORK/turn.ts" +fi + +echo "==> 3. auth: a real Supabase token against the real JWKS" +if [ "${SKIP_AUTH:-0}" = "1" ]; then + ok "skipped - SKIP_AUTH=1" +elif [ -z "${SUPABASE_URL:-}" ] || [ -z "${SUPABASE_SECRET_KEY:-}" ] || [ -z "${SUPABASE_PUBLISHABLE_KEY:-}" ]; then + ok "skipped - SUPABASE_URL / SUPABASE_SECRET_KEY / SUPABASE_PUBLISHABLE_KEY not all set" +else + export PORT + bun run src/index.ts > "$WORK/backend.log" 2>&1 & + BACKEND_PID=$! + for _ in $(seq 1 60); do + curl -fsS -m 1 -o /dev/null "http://localhost:$PORT/health" 2>/dev/null && break + sleep 0.25 + done + curl -fsS -m 5 -o /dev/null "http://localhost:$PORT/health" \ + || { sed -n '1,20p' "$WORK/backend.log" >&2; fail "the backend did not start on :$PORT"; } + health="$(curl -fsS -m 5 "http://localhost:$PORT/health")" + case "$health" in + *'"db":true'*'"valkey":true'*) ok "the backend booted on real services; /health has db and valkey up" ;; + *) bad "/health reports a dependency down: $health" ;; + esac + + cat > "$WORK/auth.ts" <<'TS' +const url = process.env.SUPABASE_URL!; +const secret = process.env.SUPABASE_SECRET_KEY!; +const pub = process.env.SUPABASE_PUBLISHABLE_KEY!; +const port = process.env.PORT!; +// `.invalid` is reserved by RFC 2606, so this address can never be a real +// person's, and the account is reused rather than piling up per run. +const email = "continuum-e2e@example.invalid"; +const password = `e2e-${crypto.randomUUID()}`; +const admin = { apikey: secret, Authorization: `Bearer ${secret}`, "Content-Type": "application/json" }; +const expect = (label: string, cond: boolean, detail = "") => { + console.log(`${cond ? "ok" : "FAILED"}\t${label}${detail ? ` (${detail})` : ""}`); + if (!cond) process.exitCode = 1; +}; + +const made = await fetch(`${url}/auth/v1/admin/users`, { + method: "POST", headers: admin, + body: JSON.stringify({ email, password, email_confirm: true }), +}); +if (!made.ok) { + // Already there from a previous run: reset the password so this one can + // still sign in. Keeps the script re-runnable without leaving users behind. + const list = (await fetch(`${url}/auth/v1/admin/users?per_page=200`, { headers: admin }).then((r) => r.json())) as any; + const existing = (list.users ?? []).find((u: any) => u.email === email); + if (!existing) { expect("the test user could be created or found", false, `HTTP ${made.status}`); process.exit(1); } + const reset = await fetch(`${url}/auth/v1/admin/users/${existing.id}`, { + method: "PUT", headers: admin, body: JSON.stringify({ password, email_confirm: true }), + }); + expect("the existing test user's password was reset", reset.ok, `HTTP ${reset.status}`); +} + +const session = (await fetch(`${url}/auth/v1/token?grant_type=password`, { + method: "POST", headers: { apikey: pub, "Content-Type": "application/json" }, + body: JSON.stringify({ email, password }), +}).then((r) => r.json())) as any; +const token: string | undefined = session.access_token; +expect("the test user signs in and receives an access token", Boolean(token)); +if (!token) process.exit(1); + +const alg = JSON.parse(atob(token.split(".")[0].replace(/-/g, "+").replace(/_/g, "/"))).alg; +expect("the token is asymmetric, so the backend needs no shared secret", alg === "ES256" || alg === "RS256", `alg=${alg}`); + +const { jwtVerifier } = await import("./src/auth/verifier"); +const claims = (await jwtVerifier.verify(token)) as any; +expect("it verifies against the project's published JWKS", claims.aud === "authenticated", `aud=${claims.aud}`); + +try { + await jwtVerifier.verify(`${token.slice(0, -4)}AAAA`); + expect("a token with a broken signature is refused", false); +} catch { + expect("a token with a broken signature is refused", true); +} + +const anon = await fetch(`http://localhost:${port}/orgs`); +expect("an unauthenticated request to /orgs is refused", anon.status === 401, `HTTP ${anon.status}`); +const authed = await fetch(`http://localhost:${port}/orgs`, { headers: { Authorization: `Bearer ${token}` } }); +expect("the same route answers that token", authed.ok, `HTTP ${authed.status}`); +TS + run_probe "auth" 6 "$WORK/auth.ts" +fi + +echo +if [ "$FAILURES" -eq 0 ]; then + echo "PASS - object storage, TURN and the real Supabase auth path all verified" + echo " against live services, not stand-ins." +else + echo "FAIL - $FAILURES check(s) failed above." >&2 + exit 1 +fi