#!/usr/bin/env bash # End-to-end test that a camera actually delivers video through the whole # stack, not just that its signalling round-trips. # # e2e-edge.sh covers the relay with go2rtc deliberately absent: it posts a # stub offer and treats a 503 that came back *from the gateway* as proof the # message travelled. That is honest about what it proves, which is only the # round trip. It cannot say a viewer would see anything, and it once read as # more than it was because a stray go2rtc left over from an earlier run # answered the stub with an HTTP 500 and the test still printed "ok". # # This one runs the real thing end to end, twice - once over direct candidates # and once pinned to relay-only through Cloudflare TURN, which is how the real # browser is configured: # # werift peer (stands in for the browser) # -> POST /printers/:id/webrtc on the real backend # -> /ws/edge/v1 -> the real continuum-proxy -> its real go2rtc # -> RTSPS to the printer's camera # and the RTP that comes back is counted. The relay phase needs # CLOUDFLARE_TURN_KEY_ID / CLOUDFLARE_TURN_API_TOKEN and is skipped, with a # message saying the TURN path went unverified, when they are absent. # # Not covered: the viewer and the gateway share this host, so what is proven is # that ICE, DTLS and media work via TURN, not that they survive two distinct # NATs; and a real browser has still not rendered a frame. # # SAFETY: read-only against the printers. It sends no printer command of any # kind; the proxy's MQTT subscription is passive and the camera is a pull of # the existing stream. Printer credentials stay in continuum-proxy/printers.toml # (gitignored) and are never read here beyond the id and name lines. # # Needs: a printer with an RTSP camera (an H2C - a P1P exposes none, see # continuum-pending-hardware-tests finding 14), go2rtc, and these ports free: # 1984 (go2rtc API, loopback) and 8555 (its WebRTC listener). # # Usage: # scripts/dev-env.sh up && scripts/dev-env.sh schema # scripts/e2e-camera.sh # # CAMERA_ID=h2c the gateway id (in printers.toml) with a camera # GO2RTC_BIN=~/.local/bin/go2rtc # PROXY_DIR=../continuum-proxy # KEEP_DATA=1 leave the seeded farm behind for inspection # # BROWSER=1 also drive the real web console in a real Chrome # (../continuum-app) and check the picture on # screen. Implies AUTH=supabase, because the app's # client needs a real Supabase session; needs # SUPABASE_URL, SUPABASE_SECRET_KEY and # SUPABASE_PUBLISHABLE_KEY (source ../dev-secrets.sh). # It creates/reuses one Supabase user, # continuum-e2e@example.invalid, and resets its # password each run. Port 3042 must be free. # AUTH=supabase use that real user's token for the API phases too # (default is the local dev-auth issuer) set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$REPO_ROOT" CAMERA_ID="${CAMERA_ID:-h2c}" GO2RTC_BIN="${GO2RTC_BIN:-$HOME/.local/bin/go2rtc}" PROXY_DIR="${PROXY_DIR:-$REPO_ROOT/../continuum-proxy}" PORT="${PORT:-3040}" AUTH_PORT="${AUTH_PORT:-3041}" BASE="http://localhost:$PORT" DATABASE_URL="${DATABASE_URL:-postgres://continuum:continuum@localhost:5432/continuum}" VALKEY_URL="${VALKEY_URL:-redis://localhost:6379}" EDGE_JWT_SECRET="${EDGE_JWT_SECRET:-e2e-edge-secret}" # Nothing here presigns or uploads; the backend just refuses to boot without # object-storage credentials (src/s3/index.ts). R2_ACCESS_KEY_ID="${R2_ACCESS_KEY_ID:-e2e}" R2_SECRET_ACCESS_KEY="${R2_SECRET_ACCESS_KEY:-e2e}" R2_ENDPOINT="${R2_ENDPOINT:-http://localhost:9000}" export DATABASE_URL VALKEY_URL EDGE_JWT_SECRET PORT export R2_ACCESS_KEY_ID R2_SECRET_ACCESS_KEY R2_ENDPOINT BROWSER="${BROWSER:-0}" AUTH="${AUTH:-dev}" [ "$BROWSER" = "1" ] && AUTH=supabase APP_DIR="${APP_DIR:-$REPO_ROOT/../continuum-app}" APP_PORT="${APP_PORT:-3042}" CHROME_BIN="${CHROME_BIN:-/usr/bin/google-chrome}" WORK="$(mktemp -d)" BACKEND_PID="" PROXY_PID="" AUTH_PID="" APP_PID="" fail() { echo "FAIL: $*" >&2; exit 1; } ok() { echo " ok: $*"; } psql_q() { psql "$DATABASE_URL" -tAq -c "$1"; } new_uuid() { cat /proc/sys/kernel/random/uuid; } USER_ID="$(new_uuid)" # The proxy supervises go2rtc, so it is the proxy's PID that has to die, and # go2rtc with it. Anything that survives holds a printer access code in memory # and answers the next run's offers - that exact leftover is how this script # came to exist - so cleanup checks for it rather than assuming. cleanup() { local rc=$? for pid in "$PROXY_PID" "$BACKEND_PID" "$AUTH_PID"; do [ -n "$pid" ] && kill "$pid" 2>/dev/null || true done # The dev server forks workers, so it is signalled as a group: it was started # with setsid, which makes its PID the group id. `kill 0` would hit us. [ -n "$APP_PID" ] && kill -- "-$APP_PID" 2>/dev/null || true for _ in $(seq 1 20); do [ -n "$PROXY_PID" ] && kill -0 "$PROXY_PID" 2>/dev/null || break sleep 0.3 done for pid in "$PROXY_PID" "$BACKEND_PID" "$AUTH_PID"; do [ -n "$pid" ] && kill -9 "$pid" 2>/dev/null || true done [ -n "$APP_PID" ] && kill -9 -- "-$APP_PID" 2>/dev/null || true if [ -z "${KEEP_DATA:-}" ] && [ -n "${ORG_ID:-}" ]; then psql_q "DELETE FROM organisations WHERE id = '$ORG_ID'" >/dev/null 2>&1 || true fi if [ $rc -ne 0 ]; then echo; echo "--- backend log (tail) ---"; tail -20 "$WORK/backend.log" 2>/dev/null || true echo "--- proxy log (tail) ---"; tail -30 "$WORK/proxy.log" 2>/dev/null || true fi rm -rf "$WORK" # A go2rtc that outlived us is a leak, not a detail. sleep 0.5 if ps -eo comm | grep -qx go2rtc; then echo "WARNING: a go2rtc process is still running after cleanup:" >&2 ps -eo pid,etime,cmd | awk '$3 ~ /go2rtc/' >&2 [ $rc -eq 0 ] && rc=1 fi exit $rc } trap cleanup EXIT # --- 0. preconditions ------------------------------------------------------ echo "==> checking preconditions" psql_q "SELECT org_id FROM farms LIMIT 0" >/dev/null 2>&1 \ || fail "no migrated database at DATABASE_URL - run scripts/dev-env.sh up && schema" [ -x "$GO2RTC_BIN" ] || fail "no go2rtc at $GO2RTC_BIN (set GO2RTC_BIN)" [ -f "$PROXY_DIR/printers.toml" ] || fail "no $PROXY_DIR/printers.toml" grep -q "^id *= *\"$CAMERA_ID\"" "$PROXY_DIR/printers.toml" \ || fail "printers.toml has no printer with id '$CAMERA_ID' (set CAMERA_ID)" # A leftover go2rtc would answer for the one under test and make this pass or # fail for reasons that have nothing to do with the code. Refuse to start. if ps -eo comm | grep -qx go2rtc; then fail "a go2rtc is already running - it would answer instead of the one under test: $(ps -eo pid,etime,cmd | awk '$3 ~ /go2rtc/' | head -2)" fi for p in 1984 8555; do ss -ltnu 2>/dev/null | grep -q ":$p " && fail "port $p is already in use" done CAM_HOST="$(awk -v id="$CAMERA_ID" -F'"' '/^id *=/ {cur=$2} cur==id && /^(host|address|ip) *=/ {print $2; exit}' "$PROXY_DIR/printers.toml")" if [ -n "$CAM_HOST" ]; then timeout 4 bash -c "/dev/null \ || fail "$CAMERA_ID ($CAM_HOST) refuses TCP 322: it has no RTSP camera, or is off" ok "$CAMERA_ID accepts RTSP on 322" else ok "could not read $CAMERA_ID's address from printers.toml; skipping the 322 pre-check" fi if [ "$AUTH" = "supabase" ]; then for name in SUPABASE_URL SUPABASE_SECRET_KEY SUPABASE_PUBLISHABLE_KEY; do [ -n "${!name:-}" ] || fail "$name is not set - source ../dev-secrets.sh (AUTH=supabase / BROWSER=1)" done fi if [ "$BROWSER" = "1" ]; then [ -x "$CHROME_BIN" ] || fail "no Chrome at $CHROME_BIN (set CHROME_BIN)" [ -d "$APP_DIR/apps/web/node_modules" ] || fail "$APP_DIR has no installed web app - run pnpm install there" ss -ltn 2>/dev/null | grep -q ":$APP_PORT " && fail "port $APP_PORT is already in use" fi # The real Supabase user, when one is needed. Captured, never echoed: the JSON # holds a password and a live access token. if [ "$AUTH" = "supabase" ]; then SB_USER="$(bun scripts/e2e-supabase-user.ts)" || fail "could not provision the Supabase test user" sb_field() { printf '%s' "$SB_USER" | python3 -c 'import json,sys; print(json.load(sys.stdin)[sys.argv[1]])' "$1"; } USER_ID="$(sb_field id)" USER_TOKEN="$(sb_field accessToken)" SB_EMAIL="$(sb_field email)" SB_PASSWORD="$(sb_field password)" unset SB_USER ok "signed in as the real Supabase test user $USER_ID" fi # --- 1. seed --------------------------------------------------------------- echo "==> seeding an organisation and a farm claiming '$CAMERA_ID'" EDGE_PROXY_TOKEN="e2e-$(date +%s)" ORG_ID="$(psql_q "INSERT INTO organisations (name) VALUES ('e2e camera org') RETURNING id")" # `viewer`: the relay route asks for viewer, so seeding the weakest role that # should work shows the camera is watchable without any authority to command. psql_q "INSERT INTO org_members (org_id, user_id, role) VALUES ('$ORG_ID', '$USER_ID', 'viewer')" >/dev/null FARM_ID="$(psql_q "INSERT INTO farms (name, org_id, edge_proxy_token) VALUES ('e2e camera farm', '$ORG_ID', '$EDGE_PROXY_TOKEN') RETURNING id")" psql_q "INSERT INTO printers (farm_id, name, vendor, gateway_printer_id) VALUES ('$FARM_ID', 'camera under test', 'bambu', '$CAMERA_ID')" >/dev/null CAM_UUID="$(psql_q "SELECT id FROM printers WHERE farm_id='$FARM_ID' AND gateway_printer_id='$CAMERA_ID'")" ok "org $ORG_ID, farm $FARM_ID, viewer $USER_ID" # --- 2. backend ------------------------------------------------------------ if [ "$AUTH" = "dev" ]; then echo "==> starting the dev-auth issuer on :$AUTH_PORT and the backend on :$PORT" bun scripts/dev-auth.ts --port "$AUTH_PORT" > "$WORK/auth.log" 2>&1 & AUTH_PID=$! AUTH_URL="http://localhost:$AUTH_PORT" for _ in $(seq 1 40); do curl -sf "$AUTH_URL/auth/v1/.well-known/jwks.json" >/dev/null 2>&1 && break; sleep 0.25; done curl -sf "$AUTH_URL/auth/v1/.well-known/jwks.json" >/dev/null 2>&1 || fail "dev-auth did not come up" export SUPABASE_URL="$AUTH_URL" USER_TOKEN="$(curl -sf "$AUTH_URL/token?sub=$USER_ID" | python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])')" else echo "==> starting the backend on :$PORT, verifying tokens against the real Supabase JWKS" fi bun run src/index.ts > "$WORK/backend.log" 2>&1 & BACKEND_PID=$! for _ in $(seq 1 60); do curl -sf "$BASE/health" >/dev/null 2>&1 && break; sleep 0.5; done curl -sf "$BASE/health" >/dev/null || fail "backend did not become healthy" TOKEN="$(curl -sf -X POST "$BASE/auth/edge-token" -H 'content-type: application/json' \ -d "{\"farmId\":\"$FARM_ID\",\"edgeProxyToken\":\"$EDGE_PROXY_TOKEN\"}" \ | python3 -c 'import json,sys; print(json.load(sys.stdin).get("token",""))')" [ -n "$TOKEN" ] || fail "no edge token" ok "backend healthy, edge token minted" # --- 3. the real gateway, with the real go2rtc ------------------------------ echo "==> starting the real continuum-proxy, which spawns the real go2rtc" ( cd "$PROXY_DIR" && cargo build --quiet 2>&1 | tail -3 ) || fail "proxy did not build" # STREAM_BASE_URL has to be set for /stream to answer 200; it is never # dialled, because the media path here is the relay, not a gateway URL. env -C "$PROXY_DIR" \ CONTINUUM_GATEWAY_ID=e2e-cam-gw \ CONTINUUM_FARM_ID="$FARM_ID" \ CONTINUUM_GATEWAY_TOKEN="$TOKEN" \ CONTINUUM_UPLINK_URL="ws://localhost:$PORT/ws/edge/v1" \ CONTINUUM_GO2RTC_BIN="$GO2RTC_BIN" \ CONTINUUM_GO2RTC_CONFIG="$WORK/go2rtc.yaml" \ CONTINUUM_STREAM_BASE_URL="http://e2e-gateway.invalid:1984" \ CONTINUUM_PRINTERS_PATH=printers.toml \ RUST_LOG=info \ "$PROXY_DIR/target/debug/continuum-proxy" > "$WORK/proxy.log" 2>&1 & PROXY_PID=$! echo "==> waiting for the camera to be advertised" STREAM="" for _ in $(seq 1 45); do STREAM="$(curl -sf -H "authorization: Bearer $USER_TOKEN" "$BASE/printers/$CAM_UUID/stream" 2>/dev/null || true)" [ -n "$STREAM" ] && break sleep 1 done [ -n "$STREAM" ] || fail "GET /printers/$CAM_UUID/stream never answered 200 - the gateway did not report a camera" # Deliberately not echoed: with TURN configured the body carries a live # username and credential, and printing it put them in a transcript once. ok "the gateway advertises camera '$(printf '%s' "$STREAM" | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d["cameraId"], "- iceServers:", len(d["iceServers"]))')'" # The generated config holds the printer's access code and go2rtc's API # password. $WORK is mode 0700, which hides a bad mode here, so the file's own # mode is what is asserted - it was 0644 until this check caught it. [ -f "$WORK/go2rtc.yaml" ] || fail "the proxy never wrote a go2rtc config" mode="$(stat -c %a "$WORK/go2rtc.yaml")" [ "$mode" = "600" ] || fail "go2rtc.yaml is mode $mode, but it holds an access code and must be 600" ok "the generated go2rtc config is owner-only (mode $mode)" # --- 4. the viewer --------------------------------------------------------- # Runs the peer once and reads its `oklabel` / `FAILEDlabel` lines. # $1 names the phase, $2 is how many results a complete run produces: fewer # means the peer died before finishing, and a silent peer must not read as a # pass. run_viewer() { local phase="$1" expected="$2" results seen=0 bad=0 verdict label shift 2 results="$(env BASE="$BASE" PRINTER_UUID="$CAM_UUID" ACCESS_TOKEN="$USER_TOKEN" "$@" \ bun scripts/e2e-camera-peer.ts 2>&1 || true)" while IFS=$'\t' read -r verdict label; do case "$verdict" in ok) ok "$label"; seen=$((seen + 1)) ;; FAILED) echo " FAILED: $label" >&2; seen=$((seen + 1)); bad=$((bad + 1)) ;; esac done < <(printf '%s\n' "$results" | grep -E "^(ok|FAILED)$(printf '\t')") if [ "$seen" -lt "$expected" ] && [ "$bad" -eq 0 ]; then bad=1; echo " FAILED: the $phase peer produced only $seen of $expected results" >&2 fi if [ "$bad" -ne 0 ]; then printf '%s\n' "$results" | sed 's/^/ | /' >&2 fail "the camera did not deliver video ($phase)" fi } echo "==> a real WebRTC peer plays the camera through the relay (direct candidates)" run_viewer "direct" 5 # The path the product actually uses: the browser is pinned to relay-only, so # media goes through Cloudflare TURN. Needs real credentials, which come from # the environment, so it is skipped - loudly - rather than failed without them. if [ -n "${CLOUDFLARE_TURN_KEY_ID:-}" ] && [ -n "${CLOUDFLARE_TURN_API_TOKEN:-}" ]; then echo "==> the same camera, viewer pinned to relay-only through Cloudflare TURN" run_viewer "relay-only" 7 ICE_MODE=relay else echo " SKIPPED: relay-only phase - CLOUDFLARE_TURN_KEY_ID / CLOUDFLARE_TURN_API_TOKEN not set." echo " The TURN-relayed path, which the browser is pinned to, is NOT verified by this run." fi # --- 5. the real web console in a real browser ------------------------------- if [ "$BROWSER" = "1" ]; then echo "==> starting the web console on :$APP_PORT" # setsid so the whole dev server (it forks workers) shares one process group # that cleanup can signal; the app is told where the backend is and given the # real Supabase project it signs in against. setsid env -C "$APP_DIR/apps/web" \ NUXT_PUBLIC_API_BASE="$BASE" \ NUXT_PUBLIC_WS_BASE="ws://localhost:$PORT" \ SUPABASE_URL="$SUPABASE_URL" \ SUPABASE_KEY="$SUPABASE_PUBLISHABLE_KEY" \ pnpm exec nuxt dev --host 127.0.0.1 --port "$APP_PORT" > "$WORK/app.log" 2>&1 & APP_PID=$! for _ in $(seq 1 120); do curl -sf -o /dev/null "http://127.0.0.1:$APP_PORT/login" 2>/dev/null && break sleep 1 done curl -sf -o /dev/null "http://127.0.0.1:$APP_PORT/login" \ || { tail -20 "$WORK/app.log" >&2; fail "the web console did not start on :$APP_PORT"; } ok "the web console is up" echo "==> the real console in real Chrome: sign in, open the fleet, watch the camera" BROWSER_OUT="$(env APP_URL="http://127.0.0.1:$APP_PORT" SUPABASE_E2E_EMAIL="$SB_EMAIL" \ SUPABASE_E2E_PASSWORD="$SB_PASSWORD" CHROME_BIN="$CHROME_BIN" \ timeout 240 bun scripts/e2e-camera-browser.ts 2>&1 || true)" seen=0; bad=0 while IFS=$'\t' read -r verdict label; do case "$verdict" in ok) ok "$label"; seen=$((seen + 1)) ;; FAILED) echo " FAILED: $label" >&2; seen=$((seen + 1)); bad=$((bad + 1)) ;; esac done < <(printf '%s\n' "$BROWSER_OUT" | grep -E "^(ok|FAILED)$(printf '\t')") # A complete run is nine results. Fewer means the driver died part way, and a # silent driver must not read as a pass. if [ "$seen" -lt 9 ] && [ "$bad" -eq 0 ]; then bad=1; echo " FAILED: the browser run produced only $seen of 9 results" >&2 fi if [ "$bad" -ne 0 ]; then printf '%s\n' "$BROWSER_OUT" | sed 's/^/ | /' >&2 echo "--- web console log (tail) ---" >&2; tail -15 "$WORK/app.log" >&2 fail "the real browser did not show the camera" fi fi echo echo "PASS - a real WebRTC peer received video from the printer's camera" echo " through the backend relay, the uplink, the proxy and go2rtc." if [ -n "${CLOUDFLARE_TURN_KEY_ID:-}" ] && [ -n "${CLOUDFLARE_TURN_API_TOKEN:-}" ]; then echo " Both directly and pinned to relay-only through Cloudflare TURN." fi if [ "$BROWSER" = "1" ]; then echo " And the real web console, in real Chrome, showed the picture." fi