diff --git a/package.json b/package.json index 06868d7..27365c1 100644 --- a/package.json +++ b/package.json @@ -29,8 +29,9 @@ "ioredis": "^5.4.1" }, "devDependencies": { - "drizzle-kit": "0.24.2", "bun-types": "^1.1.10", - "typescript": "^5.6.2" + "drizzle-kit": "0.24.2", + "typescript": "^5.6.2", + "werift": "^0.25.1" } } diff --git a/scripts/e2e-camera-peer.ts b/scripts/e2e-camera-peer.ts new file mode 100644 index 0000000..ce3dcb7 --- /dev/null +++ b/scripts/e2e-camera-peer.ts @@ -0,0 +1,128 @@ +/** + * A WebRTC receiver that is not a browser, for scripts/e2e-camera.sh. + * + * What it proves that nothing else here does: that video actually arrives. + * The signalling relay has always been testable with a canned SDP, and the + * go2rtc-to-printer leg was confirmed by hand, but "the offer round-trips" + * says nothing about whether a viewer would see a picture. This runs the + * browser's half of the exchange for real - createOffer, relay through the + * backend, setRemoteDescription, then ICE, DTLS and SRTP to go2rtc - and + * counts the RTP packets that come out the other end. + * + * It also exists because the suite's earlier stub offer (`v=0 ... t=0 0`, no + * media section, no ice-ufrag) could only ever be refused, and the test + * counted that refusal as success. A real offer has to be accepted. + * + * Results are printed as `oklabel` / `FAILEDlabel`, the convention + * run_probe in e2e-real-services.sh also reads. werift is a devDependency of + * this repo for this file alone; nothing under src/ imports it. + * + * Environment: BASE, PRINTER_UUID, ACCESS_TOKEN. The token never reaches a + * log line. + */ +import { RTCPeerConnection, RTCRtpCodecParameters } from "werift"; + +const base = process.env.BASE!; +const printer = process.env.PRINTER_UUID!; +const token = process.env.ACCESS_TOKEN!; +const MIN_PACKETS = Number(process.env.MIN_RTP_PACKETS ?? 50); +const WAIT_MS = Number(process.env.MEDIA_WAIT_MS ?? 25_000); + +let failed = false; +const say = (good: boolean, label: string, detail = "") => { + if (!good) failed = true; + console.log(`${good ? "ok" : "FAILED"}\t${label}${detail ? ` (${detail})` : ""}`); +}; + +// werift offers only VP8 by default, and a Bambu camera is H.264 (High, level +// 4.1), so go2rtc refused the first version of this offer with "codecs not +// matched: video:H264 => video:VP8" - correctly. Every browser offers H.264, +// so offering it here is what a real viewer does, not a concession to the test. +// The parameters are Chrome's: packetization-mode=1, constrained baseline. +const pc = new RTCPeerConnection({ + iceServers: [], + codecs: { + video: [ + new RTCRtpCodecParameters({ + mimeType: "video/H264", + clockRate: 90000, + rtcpFeedback: [{ type: "nack" }, { type: "nack", parameter: "pli" }, { type: "goog-remb" }], + parameters: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", + }), + ], + }, +}); +const transceiver = pc.addTransceiver("video", { direction: "recvonly" }); + +let packets = 0; +let bytes = 0; +transceiver.onTrack.subscribe((track) => { + track.onReceiveRtp.subscribe((rtp) => { + packets += 1; + bytes += rtp.payload.length; + }); +}); + +const offer = await pc.createOffer(); +await pc.setLocalDescription(offer); +const offerSdp = pc.localDescription!.sdp; +say(/^m=video /m.test(offerSdp) && /^a=ice-ufrag:/m.test(offerSdp), + "the offer has a video section and an ice-ufrag, unlike the stub it replaces"); + +// A player that gets "stream not found" just asks again, and so does this: the +// gateway advertises a camera as soon as the printer reports its URL, a moment +// before go2rtc has been restarted with the stream in its config. Bounded, and +// the attempt count is reported, so a window that grows shows up as a number +// rather than as a flaky pass. Only that one reason is retried - a codec +// mismatch or any other refusal is a real failure and must fail at once. +const RETRY_MS = Number(process.env.OFFER_RETRY_MS ?? 20_000); +const started = Date.now(); +let attempts = 0; +let res: Response; +let bodyText = ""; +do { + attempts += 1; + res = await fetch(`${base}/printers/${printer}/webrtc`, { + method: "POST", + headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, + body: JSON.stringify({ sdp: offerSdp }), + }); + bodyText = await res.text(); + if (res.status !== 503 || !bodyText.includes("stream not found")) break; + await Bun.sleep(1000); +} while (Date.now() - started < RETRY_MS); +say(res.status === 200, "the relayed offer is answered 200", + `HTTP ${res.status} after ${attempts} attempt(s), ${Math.round((Date.now() - started) / 1000)}s`); +if (res.status !== 200) { + // The gateway's reason is the whole diagnosis, and it carries no secrets. + console.log(`# ${bodyText.slice(0, 300).replace(/\s+/g, " ")}`); + await pc.close(); + process.exit(1); +} + +const answer = (JSON.parse(bodyText) as { sdp?: string }).sdp ?? ""; +say(/^m=video /m.test(answer) && /H264\/90000/i.test(answer), + "the answer negotiates H264 video, which is what the printer sends"); + +await pc.setRemoteDescription({ type: "answer", sdp: answer }); + +const connected = await new Promise((resolve) => { + const timer = setTimeout(() => resolve(false), WAIT_MS); + pc.connectionStateChange.subscribe((state) => { + if (state === "connected") { clearTimeout(timer); resolve(true); } + if (state === "failed") { clearTimeout(timer); resolve(false); } + }); + if (pc.connectionState === "connected") { clearTimeout(timer); resolve(true); } +}); +say(connected, "ICE and DTLS completed with go2rtc", `state=${pc.connectionState}`); + +if (connected) { + const deadline = Date.now() + WAIT_MS; + while (packets < MIN_PACKETS && Date.now() < deadline) await Bun.sleep(250); +} +say(packets >= MIN_PACKETS, + `video RTP reached the viewer: at least ${MIN_PACKETS} packets`, + `${packets} packets, ${bytes} payload bytes`); + +await pc.close(); +process.exit(failed ? 1 : 0); diff --git a/scripts/e2e-camera.sh b/scripts/e2e-camera.sh new file mode 100755 index 0000000..a925ea5 --- /dev/null +++ b/scripts/e2e-camera.sh @@ -0,0 +1,229 @@ +#!/usr/bin/env bash +# End-to-end test that a camera actually delivers video through the whole +# stack, not just that its signalling round-trips. +# +# e2e-edge.sh covers the relay with go2rtc deliberately absent: it posts a +# stub offer and treats a 503 that came back *from the gateway* as proof the +# message travelled. That is honest about what it proves, which is only the +# round trip. It cannot say a viewer would see anything, and it once read as +# more than it was because a stray go2rtc left over from an earlier run +# answered the stub with an HTTP 500 and the test still printed "ok". +# +# This one runs the real thing end to end: +# +# werift peer (stands in for the browser) +# -> POST /printers/:id/webrtc on the real backend +# -> /ws/edge/v1 -> the real continuum-proxy -> its real go2rtc +# -> RTSPS to the printer's camera +# and the RTP that comes back is counted. +# +# SAFETY: read-only against the printers. It sends no printer command of any +# kind; the proxy's MQTT subscription is passive and the camera is a pull of +# the existing stream. Printer credentials stay in continuum-proxy/printers.toml +# (gitignored) and are never read here beyond the id and name lines. +# +# Needs: a printer with an RTSP camera (an H2C - a P1P exposes none, see +# continuum-pending-hardware-tests finding 14), go2rtc, and these ports free: +# 1984 (go2rtc API, loopback) and 8555 (its WebRTC listener). +# +# Usage: +# scripts/dev-env.sh up && scripts/dev-env.sh schema +# scripts/e2e-camera.sh +# +# CAMERA_ID=h2c the gateway id (in printers.toml) with a camera +# GO2RTC_BIN=~/.local/bin/go2rtc +# PROXY_DIR=../continuum-proxy +# KEEP_DATA=1 leave the seeded farm behind for inspection +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +CAMERA_ID="${CAMERA_ID:-h2c}" +GO2RTC_BIN="${GO2RTC_BIN:-$HOME/.local/bin/go2rtc}" +PROXY_DIR="${PROXY_DIR:-$REPO_ROOT/../continuum-proxy}" +PORT="${PORT:-3040}" +AUTH_PORT="${AUTH_PORT:-3041}" +BASE="http://localhost:$PORT" +DATABASE_URL="${DATABASE_URL:-postgres://continuum:continuum@localhost:5432/continuum}" +VALKEY_URL="${VALKEY_URL:-redis://localhost:6379}" +EDGE_JWT_SECRET="${EDGE_JWT_SECRET:-e2e-edge-secret}" +# Nothing here presigns or uploads; the backend just refuses to boot without +# object-storage credentials (src/s3/index.ts). +R2_ACCESS_KEY_ID="${R2_ACCESS_KEY_ID:-e2e}" +R2_SECRET_ACCESS_KEY="${R2_SECRET_ACCESS_KEY:-e2e}" +R2_ENDPOINT="${R2_ENDPOINT:-http://localhost:9000}" +export DATABASE_URL VALKEY_URL EDGE_JWT_SECRET PORT +export R2_ACCESS_KEY_ID R2_SECRET_ACCESS_KEY R2_ENDPOINT + +WORK="$(mktemp -d)" +BACKEND_PID="" +PROXY_PID="" +AUTH_PID="" + +fail() { echo "FAIL: $*" >&2; exit 1; } +ok() { echo " ok: $*"; } +psql_q() { psql "$DATABASE_URL" -tAq -c "$1"; } +new_uuid() { cat /proc/sys/kernel/random/uuid; } +USER_ID="$(new_uuid)" + +# The proxy supervises go2rtc, so it is the proxy's PID that has to die, and +# go2rtc with it. Anything that survives holds a printer access code in memory +# and answers the next run's offers - that exact leftover is how this script +# came to exist - so cleanup checks for it rather than assuming. +cleanup() { + local rc=$? + for pid in "$PROXY_PID" "$BACKEND_PID" "$AUTH_PID"; do + [ -n "$pid" ] && kill "$pid" 2>/dev/null || true + done + for _ in $(seq 1 20); do + [ -n "$PROXY_PID" ] && kill -0 "$PROXY_PID" 2>/dev/null || break + sleep 0.3 + done + for pid in "$PROXY_PID" "$BACKEND_PID" "$AUTH_PID"; do + [ -n "$pid" ] && kill -9 "$pid" 2>/dev/null || true + done + if [ -z "${KEEP_DATA:-}" ] && [ -n "${ORG_ID:-}" ]; then + psql_q "DELETE FROM organisations WHERE id = '$ORG_ID'" >/dev/null 2>&1 || true + fi + if [ $rc -ne 0 ]; then + echo; echo "--- backend log (tail) ---"; tail -20 "$WORK/backend.log" 2>/dev/null || true + echo "--- proxy log (tail) ---"; tail -30 "$WORK/proxy.log" 2>/dev/null || true + fi + rm -rf "$WORK" + # A go2rtc that outlived us is a leak, not a detail. + sleep 0.5 + if ps -eo comm | grep -qx go2rtc; then + echo "WARNING: a go2rtc process is still running after cleanup:" >&2 + ps -eo pid,etime,cmd | awk '$3 ~ /go2rtc/' >&2 + [ $rc -eq 0 ] && rc=1 + fi + exit $rc +} +trap cleanup EXIT + +# --- 0. preconditions ------------------------------------------------------ + +echo "==> checking preconditions" +psql_q "SELECT org_id FROM farms LIMIT 0" >/dev/null 2>&1 \ + || fail "no migrated database at DATABASE_URL - run scripts/dev-env.sh up && schema" +[ -x "$GO2RTC_BIN" ] || fail "no go2rtc at $GO2RTC_BIN (set GO2RTC_BIN)" +[ -f "$PROXY_DIR/printers.toml" ] || fail "no $PROXY_DIR/printers.toml" +grep -q "^id *= *\"$CAMERA_ID\"" "$PROXY_DIR/printers.toml" \ + || fail "printers.toml has no printer with id '$CAMERA_ID' (set CAMERA_ID)" + +# A leftover go2rtc would answer for the one under test and make this pass or +# fail for reasons that have nothing to do with the code. Refuse to start. +if ps -eo comm | grep -qx go2rtc; then + fail "a go2rtc is already running - it would answer instead of the one under test: $(ps -eo pid,etime,cmd | awk '$3 ~ /go2rtc/' | head -2)" +fi +for p in 1984 8555; do + ss -ltnu 2>/dev/null | grep -q ":$p " && fail "port $p is already in use" +done +CAM_HOST="$(awk -v id="$CAMERA_ID" -F'"' '/^id *=/ {cur=$2} cur==id && /^(host|address|ip) *=/ {print $2; exit}' "$PROXY_DIR/printers.toml")" +if [ -n "$CAM_HOST" ]; then + timeout 4 bash -c "/dev/null \ + || fail "$CAMERA_ID ($CAM_HOST) refuses TCP 322: it has no RTSP camera, or is off" + ok "$CAMERA_ID accepts RTSP on 322" +else + ok "could not read $CAMERA_ID's address from printers.toml; skipping the 322 pre-check" +fi + +# --- 1. seed --------------------------------------------------------------- + +echo "==> seeding an organisation and a farm claiming '$CAMERA_ID'" +EDGE_PROXY_TOKEN="e2e-$(date +%s)" +ORG_ID="$(psql_q "INSERT INTO organisations (name) VALUES ('e2e camera org') RETURNING id")" +# `viewer`: the relay route asks for viewer, so seeding the weakest role that +# should work shows the camera is watchable without any authority to command. +psql_q "INSERT INTO org_members (org_id, user_id, role) VALUES ('$ORG_ID', '$USER_ID', 'viewer')" >/dev/null +FARM_ID="$(psql_q "INSERT INTO farms (name, org_id, edge_proxy_token) + VALUES ('e2e camera farm', '$ORG_ID', '$EDGE_PROXY_TOKEN') RETURNING id")" +psql_q "INSERT INTO printers (farm_id, name, vendor, gateway_printer_id) + VALUES ('$FARM_ID', 'camera under test', 'bambu', '$CAMERA_ID')" >/dev/null +CAM_UUID="$(psql_q "SELECT id FROM printers WHERE farm_id='$FARM_ID' AND gateway_printer_id='$CAMERA_ID'")" +ok "org $ORG_ID, farm $FARM_ID, viewer $USER_ID" + +# --- 2. backend ------------------------------------------------------------ + +echo "==> starting the dev-auth issuer on :$AUTH_PORT and the backend on :$PORT" +bun scripts/dev-auth.ts --port "$AUTH_PORT" > "$WORK/auth.log" 2>&1 & +AUTH_PID=$! +AUTH_URL="http://localhost:$AUTH_PORT" +for _ in $(seq 1 40); do curl -sf "$AUTH_URL/auth/v1/.well-known/jwks.json" >/dev/null 2>&1 && break; sleep 0.25; done +curl -sf "$AUTH_URL/auth/v1/.well-known/jwks.json" >/dev/null 2>&1 || fail "dev-auth did not come up" +export SUPABASE_URL="$AUTH_URL" +USER_TOKEN="$(curl -sf "$AUTH_URL/token?sub=$USER_ID" | python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])')" +bun run src/index.ts > "$WORK/backend.log" 2>&1 & +BACKEND_PID=$! +for _ in $(seq 1 60); do curl -sf "$BASE/health" >/dev/null 2>&1 && break; sleep 0.5; done +curl -sf "$BASE/health" >/dev/null || fail "backend did not become healthy" +TOKEN="$(curl -sf -X POST "$BASE/auth/edge-token" -H 'content-type: application/json' \ + -d "{\"farmId\":\"$FARM_ID\",\"edgeProxyToken\":\"$EDGE_PROXY_TOKEN\"}" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin).get("token",""))')" +[ -n "$TOKEN" ] || fail "no edge token" +ok "backend healthy, edge token minted" + +# --- 3. the real gateway, with the real go2rtc ------------------------------ + +echo "==> starting the real continuum-proxy, which spawns the real go2rtc" +( cd "$PROXY_DIR" && cargo build --quiet 2>&1 | tail -3 ) || fail "proxy did not build" +# STREAM_BASE_URL has to be set for /stream to answer 200; it is never +# dialled, because the media path here is the relay, not a gateway URL. +env -C "$PROXY_DIR" \ + CONTINUUM_GATEWAY_ID=e2e-cam-gw \ + CONTINUUM_FARM_ID="$FARM_ID" \ + CONTINUUM_GATEWAY_TOKEN="$TOKEN" \ + CONTINUUM_UPLINK_URL="ws://localhost:$PORT/ws/edge/v1" \ + CONTINUUM_GO2RTC_BIN="$GO2RTC_BIN" \ + CONTINUUM_GO2RTC_CONFIG="$WORK/go2rtc.yaml" \ + CONTINUUM_STREAM_BASE_URL="http://e2e-gateway.invalid:1984" \ + CONTINUUM_PRINTERS_PATH=printers.toml \ + RUST_LOG=info \ + "$PROXY_DIR/target/debug/continuum-proxy" > "$WORK/proxy.log" 2>&1 & +PROXY_PID=$! + +echo "==> waiting for the camera to be advertised" +STREAM="" +for _ in $(seq 1 45); do + STREAM="$(curl -sf -H "authorization: Bearer $USER_TOKEN" "$BASE/printers/$CAM_UUID/stream" 2>/dev/null || true)" + [ -n "$STREAM" ] && break + sleep 1 +done +[ -n "$STREAM" ] || fail "GET /printers/$CAM_UUID/stream never answered 200 - the gateway did not report a camera" +# Deliberately not echoed: with TURN configured the body carries a live +# username and credential, and printing it put them in a transcript once. +ok "the gateway advertises camera '$(printf '%s' "$STREAM" | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d["cameraId"], "- iceServers:", len(d["iceServers"]))')'" + +# The generated config holds the printer's access code and go2rtc's API +# password. $WORK is mode 0700, which hides a bad mode here, so the file's own +# mode is what is asserted - it was 0644 until this check caught it. +[ -f "$WORK/go2rtc.yaml" ] || fail "the proxy never wrote a go2rtc config" +mode="$(stat -c %a "$WORK/go2rtc.yaml")" +[ "$mode" = "600" ] || fail "go2rtc.yaml is mode $mode, but it holds an access code and must be 600" +ok "the generated go2rtc config is owner-only (mode $mode)" + +# --- 4. the viewer --------------------------------------------------------- + +echo "==> a real WebRTC peer plays the camera through the relay" +RESULTS="$(BASE="$BASE" PRINTER_UUID="$CAM_UUID" ACCESS_TOKEN="$USER_TOKEN" bun scripts/e2e-camera-peer.ts 2>&1 || true)" +SEEN=0; BAD=0 +while IFS=$'\t' read -r verdict label; do + case "$verdict" in + ok) ok "$label"; SEEN=$((SEEN + 1)) ;; + FAILED) echo " FAILED: $label" >&2; SEEN=$((SEEN + 1)); BAD=$((BAD + 1)) ;; + esac +done < <(printf '%s\n' "$RESULTS" | grep -E "^(ok|FAILED)$(printf '\t')") +# Fewer results than the peer's five checks means it died before finishing, +# and a silent peer must not read as a pass. +if [ "$SEEN" -lt 5 ] && [ "$BAD" -eq 0 ]; then + BAD=1; echo " FAILED: the peer produced only $SEEN of 5 results" >&2 +fi +if [ "$BAD" -ne 0 ]; then + printf '%s\n' "$RESULTS" | sed 's/^/ | /' >&2 + fail "the camera did not deliver video" +fi + +echo +echo "PASS - a real WebRTC peer received video from the printer's camera" +echo " through the backend relay, the uplink, the proxy and go2rtc." diff --git a/scripts/e2e-edge.sh b/scripts/e2e-edge.sh index 29c4509..138feee 100755 --- a/scripts/e2e-edge.sh +++ b/scripts/e2e-edge.sh @@ -47,6 +47,10 @@ R2_SECRET_ACCESS_KEY="${R2_SECRET_ACCESS_KEY:-e2e}" R2_ENDPOINT="${R2_ENDPOINT:-http://localhost:9000}" export DATABASE_URL VALKEY_URL EDGE_JWT_SECRET PORT export R2_ACCESS_KEY_ID R2_SECRET_ACCESS_KEY R2_ENDPOINT +# This script asserts TURN is unconfigured (below). Sourcing dev-secrets.sh +# turns it on, which used to fail the assertion - and print the live +# credentials in the failure message. Make the assumption true. +unset CLOUDFLARE_TURN_KEY_ID CLOUDFLARE_TURN_API_TOKEN WORK="$(mktemp -d)" BACKEND_PID="" @@ -255,7 +259,7 @@ fi ok "every printer id this gateway reported mapped to a row" echo "==> what landed in the database:" -PGPASSWORD=continuum psql -h localhost -U continuum -d continuum -c \ +psql "$DATABASE_URL" -c \ "SELECT gateway_printer_id AS gw, status, vendor_state, progress_percent AS pct, current_layer || '/' || total_layers AS layers, remaining_minutes AS mins, nozzle_temp_c AS nozzle, bed_temp_c AS bed, active_alerts AS alerts,