diff --git a/README.md b/README.md index 0b76e71..e4b5ec8 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,34 @@ with pgvector for frame embeddings), Valkey Streams, and Cloudflare R2 storage. ```sh bun install -cp .env.example .env # fill in DATABASE_URL, VALKEY_URL, R2_*, SUPABASE_JWT_SECRET -bun run db:generate && bun run db:migrate +scripts/dev-env.sh up # Valkey + Postgres/pgvector, and print the URLs +scripts/dev-env.sh schema # or: bun run db:migrate +cp .env.example .env # fill in the URLs it printed, plus R2_*, SUPABASE_JWT_SECRET bun run dev ``` +`scripts/dev-env.sh` uses `docker-compose.yml` when a Docker daemon is +answering, and falls back to the same two services from apt when there is no +usable container runtime - some sandboxes cannot run Docker at all, because +dockerd needs `CAP_NET_ADMIN` for its bridge and user namespaces to unpack an +image, and neither can be granted from inside. `scripts/dev-env.sh status` +says which backend is in use. + +## Test + +```sh +bun test # unit + protocol contract tests, no services needed +scripts/e2e-edge.sh # the whole edge path: real database, real printers +``` + +`scripts/e2e-edge.sh` seeds a farm, claims its printers, mints an edge token, +runs the real `continuum-proxy` against this backend, and asserts that live +telemetry reached Postgres and that a command reached a printer. It only ever +sends a fan speed, and only to a printer reporting `IDLE`. +`scripts/capture-gateway-frames.js` is the smaller tool: it stands in for this +backend and records what a gateway sends, for refreshing the fixtures in +`tests/fixtures/`. + ## Layout - `src/db/` - Drizzle schema (`printers`, `farms`, `print_jobs`, `frame_embeddings`) @@ -30,7 +53,24 @@ same RESP protocol, so `ioredis` is unchanged - only the server differs. See `continuum-common/docs/LICENSES.md`. `docker-compose.yml` here brings up Valkey and Postgres+pgvector for local development. - `src/routes/` - `/auth`, `/farms`, `/printers`, `/jobs`, and the edge-gateway - WebSocket at `/ws/edge/v1`. + WebSocket at `/ws/edge/v1`. That route's wire format is not defined here: it + is `schemas/continuum/v1/edge/` in continuum-schemas, validated through its + TypeBox mirror in `@continuum/ts-types`, because both ends of it are separate + programs in different languages and they had already drifted apart once. + `printers.gateway_printer_id` is what maps a gateway's own printer ids + (`"p1p"`) onto rows here; telemetry for an unclaimed id is logged and skipped, + never inserted. `frame_embeddings.embedding` requires the `vector` extension: -`CREATE EXTENSION IF NOT EXISTS vector;` +`CREATE EXTENSION IF NOT EXISTS vector;` (`scripts/dev-env.sh up` does this). + +## Dependency pins + +`elysia`, `@sinclair/typebox` and the `drizzle-*` packages are pinned tightly +on purpose, not out of caution. Caret ranges here resolved into a combination +that could not boot: `elysia: ^1.1.26` installed 1.4.30, which calls +`t.Module` and so needs TypeBox >=0.34, while `drizzle-typebox: ^0.2.1` +installed a version whose peers demand `drizzle-orm >=0.36` against the +`^0.33.0` pinned here. Typechecking passed throughout - only starting the +server failed. `tests/dependency_identity.test.ts` guards the part of this that +can silently come back. diff --git a/package.json b/package.json index 17ccd3f..1ec749e 100644 --- a/package.json +++ b/package.json @@ -15,20 +15,20 @@ }, "dependencies": { "@continuum/ts-types": "file:../continuum-schemas/packages/ts-types", - "elysia": "^1.1.26", - "@elysiajs/cors": "^1.1.1", - "@elysiajs/jwt": "^1.1.1", - "@elysiajs/swagger": "^1.1.6", + "elysia": "~1.4.30", + "@elysiajs/cors": "~1.4.2", + "@elysiajs/jwt": "~1.4.2", + "@elysiajs/swagger": "~1.3.1", "@aws-sdk/client-s3": "^3.658.1", "@aws-sdk/s3-request-presigner": "^3.658.1", - "drizzle-orm": "^0.33.0", - "drizzle-typebox": "^0.2.1", - "@sinclair/typebox": "^0.33.12", + "drizzle-orm": "0.33.0", + "drizzle-typebox": "0.1.1", + "@sinclair/typebox": "0.34.52", "postgres": "^3.4.4", "ioredis": "^5.4.1" }, "devDependencies": { - "drizzle-kit": "^0.24.2", + "drizzle-kit": "0.24.2", "bun-types": "^1.1.10", "typescript": "^5.6.2" } diff --git a/scripts/capture-gateway-frames.js b/scripts/capture-gateway-frames.js new file mode 100644 index 0000000..c9cdb61 --- /dev/null +++ b/scripts/capture-gateway-frames.js @@ -0,0 +1,108 @@ +// Stands in for this backend's /ws/edge/v1 and records every frame a real +// gateway sends, one JSON object per line. +// +// What it is for: refreshing tests/fixtures/gateway_frames.json. Those fixtures +// are deliberately real captures rather than hand-written examples - a +// hand-written example agrees with whoever wrote it, which is how the two ends +// of this protocol drifted into incompatible shapes without anything failing. +// So when the protocol changes, re-capture rather than editing the fixture. +// +// It needs no database and no auth, which is the point: it isolates the +// gateway's side of the wire from everything else. For the full path through a +// real database and real printers, use e2e-edge.sh instead. +// +// Usage: +// bun run scripts/capture-gateway-frames.js > frames.jsonl +// # then, in continuum-proxy, with its CONTINUUM_* env set: +// CONTINUUM_UPLINK_URL=ws://localhost:59999/ws/edge/v1 cargo run +// +// PORT=59999 where to listen +// FAN_TEST_PRINTER=p1p send one set_fan_speed to this printer, to capture a +// command_result. Leave unset to stay read-only. +// +// SAFETY: with FAN_TEST_PRINTER set this writes to a real printer. Only ever a +// fan speed, and only to the printer you name - but name an idle one. +const PORT = Number(process.env.PORT ?? 59999); +const FAN_TEST_PRINTER = process.env.FAN_TEST_PRINTER; + +let commandSent = false; + +Bun.serve({ + port: PORT, + fetch(req, server) { + if (server.upgrade(req)) return; + return new Response("this endpoint only speaks WebSocket\n", { status: 400 }); + }, + websocket: { + open() { + console.error(`[capture] gateway connected`); + }, + message(ws, raw) { + // stdout is the capture, stderr is the commentary, so the two can be + // redirected separately. + console.log(typeof raw === "string" ? raw : raw.toString()); + + let msg; + try { + msg = JSON.parse(typeof raw === "string" ? raw : raw.toString()); + } catch (err) { + console.error(`[capture] not JSON, recorded anyway: ${err}`); + return; + } + + switch (msg.type) { + case "hello": + // No token check: this is not the backend, it is a recorder. + console.error(`[capture] hello from ${msg.gateway_id} (farm ${msg.farm_id})`); + ws.send(JSON.stringify({ type: "hello_ack", session_id: "capture" })); + break; + + case "heartbeat": + ws.send(JSON.stringify({ type: "heartbeat_ack", seq: msg.seq })); + break; + + case "telemetry": + console.error( + `[capture] telemetry: ${msg.printers + .map((p) => `${p.printer_id}=${p.state ?? "?"}`) + .join(" ")}`, + ); + if (FAN_TEST_PRINTER && !commandSent) { + commandSent = true; + const reported = msg.printers.find((p) => p.printer_id === FAN_TEST_PRINTER); + // Refuse to touch a printer that is not idle, and refuse to guess + // at one the gateway has not reported at all. + if (!reported) { + console.error(`[capture] '${FAN_TEST_PRINTER}' not in this batch, not sending`); + } else if (reported.state !== "IDLE" && reported.state !== "FINISH") { + console.error(`[capture] '${FAN_TEST_PRINTER}' is ${reported.state}, not sending`); + } else { + console.error(`[capture] -> set_fan_speed 0 to ${FAN_TEST_PRINTER}`); + ws.send(JSON.stringify({ + type: "command", + command_id: "capture-1", + printer_id: FAN_TEST_PRINTER, + command: { action: "set_fan_speed", percent: 0 }, + })); + } + } + break; + + case "command_result": + console.error(`[capture] command_result ${msg.command_id}: ok=${msg.ok}`); + break; + + default: + console.error(`[capture] ${msg.type}`); + } + }, + close() { + console.error("[capture] gateway disconnected"); + }, + }, +}); + +console.error(`[capture] listening on :${PORT} - frames to stdout, commentary to stderr`); +if (!FAN_TEST_PRINTER) { + console.error("[capture] read-only (set FAN_TEST_PRINTER to also capture a command_result)"); +} diff --git a/scripts/dev-env.sh b/scripts/dev-env.sh new file mode 100755 index 0000000..9e50df7 --- /dev/null +++ b/scripts/dev-env.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +# Brings the local dependency stack (Valkey + Postgres/pgvector) up and down, +# and applies the Drizzle schema to it. +# +# Two backends behind one interface, chosen automatically: +# +# docker docker-compose.yml - Valkey 8 + pgvector/pgvector:pg17. What CI +# and a normal dev machine should use; closest to production. +# native apt packages started directly - postgresql-16 + pgvector + Valkey +# 7.2 from Ubuntu's own repositories. +# +# The native path exists because Docker provably cannot run in some sandboxes, +# including the one this was written in: dockerd needs CAP_NET_ADMIN to build +# its bridge, and even with `--bridge=none --iptables=false +# --storage-driver=vfs` it cannot unpack an image, because registering a layer +# needs user namespaces. Both are host-level grants, so there is nothing a +# script can do about it. Rather than leave the database untestable there, this +# falls back to running the same two services from apt. Set CONTINUUM_DEV_BACKEND +# to force one. +# +# Usage: +# scripts/dev-env.sh up start the services, wait until they answer +# scripts/dev-env.sh down stop them (keeps data) +# scripts/dev-env.sh reset stop them AND delete all data +# scripts/dev-env.sh status what is running, and which backend +# scripts/dev-env.sh psql open a shell on the database +# scripts/dev-env.sh schema apply the Drizzle schema (drizzle-kit push) +# scripts/dev-env.sh env print the DATABASE_URL/VALKEY_URL to use +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +PG_VERSION=16 +PG_PORT=5432 +VALKEY_PORT=6379 +# Straight at Postgres, not pgCat on 6432: .env.example points at the pooler for +# production, and neither backend here runs one. +DEV_DATABASE_URL="postgres://continuum:continuum@localhost:${PG_PORT}/continuum" +DEV_VALKEY_URL="redis://localhost:${VALKEY_PORT}" + +NATIVE_STATE="${XDG_RUNTIME_DIR:-/tmp}/continuum-dev-env" + +# --- backend selection ----------------------------------------------------- + +# Docker is used when its daemon is already answering. Deliberately does not +# try to start dockerd: on a machine where it works there is a service manager +# doing that, and on a machine where it does not, starting it just produces a +# daemon that cannot unpack an image (see the header). Force either backend +# with CONTINUUM_DEV_BACKEND=docker|native. +docker_available() { + command -v docker >/dev/null 2>&1 || return 1 + sudo -n docker info >/dev/null 2>&1 +} + +backend() { + if [ -n "${CONTINUUM_DEV_BACKEND:-}" ]; then + echo "$CONTINUUM_DEV_BACKEND" + elif docker_available; then + echo docker + else + echo native + fi +} + +# --- waiting --------------------------------------------------------------- + +# Poll for readiness rather than sleeping a fixed amount: Postgres accepts +# connections before it will serve them, and a failure should report why. +wait_for_postgres() { + for _ in $(seq 1 60); do + if pg_isready -h localhost -p "$PG_PORT" -U continuum -d continuum >/dev/null 2>&1; then + return 0 + fi + sleep 0.5 + done + echo "postgres did not become ready on port $PG_PORT" >&2 + return 1 +} + +wait_for_valkey() { + for _ in $(seq 1 40); do + if valkey-cli -p "$VALKEY_PORT" ping 2>/dev/null | grep -q PONG; then + return 0 + fi + sleep 0.5 + done + echo "valkey did not answer on port $VALKEY_PORT" >&2 + return 1 +} + +# --- native backend -------------------------------------------------------- + +native_up() { + mkdir -p "$NATIVE_STATE" + + if ! pg_isready -h localhost -p "$PG_PORT" >/dev/null 2>&1; then + echo "==> starting postgresql-$PG_VERSION" + # pg_ctlcluster rather than systemctl: there is no service manager here. + sudo pg_ctlcluster "$PG_VERSION" main start + fi + wait_for_postgres_socket + + # Idempotent: both of these are no-ops on a second run. + if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_roles WHERE rolname='continuum'" | grep -q 1; then + echo "==> creating role 'continuum'" + sudo -u postgres psql -qc "CREATE ROLE continuum LOGIN PASSWORD 'continuum' SUPERUSER" >/dev/null + fi + if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_database WHERE datname='continuum'" | grep -q 1; then + echo "==> creating database 'continuum'" + sudo -u postgres createdb -O continuum continuum + fi + # frame_embeddings needs it, and it is why the compose file uses the + # pgvector image rather than plain postgres. + sudo -u postgres psql -d continuum -qc "CREATE EXTENSION IF NOT EXISTS vector" >/dev/null + + if ! valkey-cli -p "$VALKEY_PORT" ping >/dev/null 2>&1; then + echo "==> starting valkey" + valkey-server --port "$VALKEY_PORT" --daemonize yes \ + --dir "$NATIVE_STATE" --appendonly yes \ + --logfile "$NATIVE_STATE/valkey.log" + fi + wait_for_valkey +} + +# Postgres has to be up before the role exists, so this waits on the server +# rather than on a successful `continuum` login. +wait_for_postgres_socket() { + for _ in $(seq 1 60); do + if sudo -u postgres pg_isready -q 2>/dev/null; then return 0; fi + sleep 0.5 + done + echo "postgres did not start" >&2 + return 1 +} + +native_down() { + if valkey-cli -p "$VALKEY_PORT" ping >/dev/null 2>&1; then + echo "==> stopping valkey" + valkey-cli -p "$VALKEY_PORT" shutdown nosave 2>/dev/null || true + fi + if sudo -u postgres pg_isready -q 2>/dev/null; then + echo "==> stopping postgresql-$PG_VERSION" + sudo pg_ctlcluster "$PG_VERSION" main stop || true + fi +} + +native_reset() { + native_down + echo "==> dropping the continuum database and valkey's data" + sudo pg_ctlcluster "$PG_VERSION" main start + wait_for_postgres_socket + sudo -u postgres dropdb --if-exists continuum + sudo pg_ctlcluster "$PG_VERSION" main stop || true + rm -rf "$NATIVE_STATE" +} + +native_status() { + echo "backend: native (apt packages, no container runtime)" + if sudo -u postgres pg_isready -q 2>/dev/null; then + echo " postgres: up on :$PG_PORT ($(sudo -u postgres psql -tAc 'SELECT version()' | cut -d, -f1))" + else + echo " postgres: down" + fi + if valkey-cli -p "$VALKEY_PORT" ping >/dev/null 2>&1; then + echo " valkey: up on :$VALKEY_PORT" + else + echo " valkey: down" + fi +} + +# --- docker backend -------------------------------------------------------- + +docker_up() { + sudo docker compose up -d + echo "==> waiting for containers to report healthy" + for _ in $(seq 1 60); do + local unhealthy + unhealthy="$(sudo docker compose ps --format '{{.Service}} {{.Health}}' \ + | awk '$2 != "healthy" { print $1 }' || true)" + [ -z "$unhealthy" ] && return 0 + sleep 1 + done + echo "timed out: $(sudo docker compose ps --format '{{.Service}} {{.Health}}')" >&2 + return 1 +} + +# --- commands -------------------------------------------------------------- + +cmd_up() { + case "$(backend)" in + docker) echo "==> backend: docker compose"; docker_up ;; + native) echo "==> backend: native (docker is unavailable here - see this script's header)"; native_up ;; + esac + echo + cmd_env + echo + echo "Next: scripts/dev-env.sh schema" +} + +cmd_down() { case "$(backend)" in docker) sudo docker compose down ;; native) native_down ;; esac; } +cmd_reset() { case "$(backend)" in docker) sudo docker compose down -v ;; native) native_reset ;; esac; } +cmd_status() { case "$(backend)" in docker) echo "backend: docker"; sudo docker compose ps ;; native) native_status ;; esac; } +cmd_psql() { PGPASSWORD=continuum psql -h localhost -p "$PG_PORT" -U continuum -d continuum "$@"; } +cmd_env() { echo "DATABASE_URL=$DEV_DATABASE_URL"; echo "VALKEY_URL=$DEV_VALKEY_URL"; } + +cmd_schema() { + # `push` rather than generate+migrate: no migrations are committed and the + # schema is still moving. Switch once it settles - push diffs the live + # database against the schema and will happily drop a column, which is fine + # locally and nowhere else. + echo "==> applying the Drizzle schema to $DEV_DATABASE_URL" + DATABASE_URL="$DEV_DATABASE_URL" bunx drizzle-kit push --force +} + +case "${1:-}" in + up) cmd_up ;; + down) cmd_down ;; + reset) cmd_reset ;; + status) cmd_status ;; + psql) shift; cmd_psql "$@" ;; + schema) cmd_schema ;; + env) cmd_env ;; + *) sed -n '2,30p' "${BASH_SOURCE[0]}" | sed 's/^# \?//'; exit 1 ;; +esac diff --git a/scripts/e2e-edge.sh b/scripts/e2e-edge.sh new file mode 100755 index 0000000..446efad --- /dev/null +++ b/scripts/e2e-edge.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +# End-to-end test of the /ws/edge/v1 contract against a real database and real +# printers: seed a farm, claim its printers, mint an edge token, run the actual +# continuum-proxy daemon, and assert that live telemetry reached Postgres and +# that a command sent over HTTP reached a printer. +# +# This is the test that the unit tests cannot be: tests/edge_protocol.test.ts +# only proves the two ends agree about the wire format. Nothing but this proves +# the printer-id translation, persistTelemetry and sendPrinterCommand actually +# work against a database. +# +# SAFETY: the only command it sends is a fan speed, and only to the printer +# named by PRINTER_ID (default p1p), which it refuses to send unless that +# printer reports IDLE. It never starts, pauses or cancels a print. Point it at +# a printer you can see. +# +# Usage: +# scripts/dev-env.sh up && scripts/dev-env.sh schema +# scripts/e2e-edge.sh +# +# PRINTER_ID=p1p which printer to send the fan command to +# PROXY_DIR=../continuum-proxy +# KEEP_DATA=1 leave the seeded farm behind for inspection +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +PRINTER_ID="${PRINTER_ID:-p1p}" +PROXY_DIR="${PROXY_DIR:-$REPO_ROOT/../continuum-proxy}" +PORT="${PORT:-3010}" +BASE="http://localhost:$PORT" +DATABASE_URL="${DATABASE_URL:-postgres://continuum:continuum@localhost:5432/continuum}" +VALKEY_URL="${VALKEY_URL:-redis://localhost:6379}" +EDGE_JWT_SECRET="${EDGE_JWT_SECRET:-e2e-edge-secret}" +export DATABASE_URL VALKEY_URL EDGE_JWT_SECRET PORT + +WORK="$(mktemp -d)" +BACKEND_PID="" +PROXY_PID="" + +psql_q() { PGPASSWORD=continuum psql -h localhost -U continuum -d continuum -tAq -c "$1"; } + +cleanup() { + local rc=$? + for pid in "$PROXY_PID" "$BACKEND_PID"; do + [ -z "$pid" ] && continue + kill "$pid" 2>/dev/null || true + done + # A gateway that outlives this script keeps reconnecting with a token for a + # farm that is about to be deleted, and pollutes the next run's log. + for _ in $(seq 1 10); do + kill -0 "${PROXY_PID:-0}" 2>/dev/null || break + sleep 0.3 + done + kill -9 "${PROXY_PID:-0}" 2>/dev/null || true + kill -9 "${BACKEND_PID:-0}" 2>/dev/null || true + if [ -z "${KEEP_DATA:-}" ] && [ -n "${FARM_ID:-}" ]; then + # Printers cascade from farms. + psql_q "DELETE FROM farms WHERE id = '$FARM_ID'" >/dev/null 2>&1 || true + fi + if [ $rc -ne 0 ]; then + echo + echo "--- backend log (tail) ---"; tail -30 "$WORK/backend.log" 2>/dev/null || true + echo "--- proxy log (tail) ---"; tail -30 "$WORK/proxy.log" 2>/dev/null || true + fi + rm -rf "$WORK" + exit $rc +} +trap cleanup EXIT + +fail() { echo "FAIL: $*" >&2; exit 1; } +ok() { echo " ok: $*"; } + +# --- 0. preconditions ------------------------------------------------------ + +echo "==> checking the database is up and migrated" +psql_q "SELECT 1" >/dev/null 2>&1 || fail "no database at $DATABASE_URL - run scripts/dev-env.sh up" +psql_q "SELECT gateway_printer_id FROM printers LIMIT 0" >/dev/null 2>&1 \ + || fail "printers.gateway_printer_id is missing - run scripts/dev-env.sh schema" +ok "schema has the edge columns" + +[ -f "$PROXY_DIR/printers.toml" ] || fail "no $PROXY_DIR/printers.toml - the proxy needs real printers" + +# --- 1. seed --------------------------------------------------------------- + +echo "==> seeding a farm and claiming its printers" +EDGE_PROXY_TOKEN="e2e-$(date +%s)" +FARM_ID="$(psql_q "INSERT INTO farms (name, owner_id, edge_proxy_token) + VALUES ('e2e farm', gen_random_uuid(), '$EDGE_PROXY_TOKEN') RETURNING id")" +[ -n "$FARM_ID" ] || fail "could not create a farm" +ok "farm $FARM_ID" + +# gateway_printer_id is the whole point: it maps the gateway's own ids from +# printers.toml onto these rows. Telemetry for an id nothing claims is skipped. +while read -r pid pname; do + [ -z "$pid" ] && continue + psql_q "INSERT INTO printers (farm_id, name, vendor, gateway_printer_id) + VALUES ('$FARM_ID', '$pname', 'bambu', '$pid')" >/dev/null + ok "claimed '$pid' as '$pname'" +done < <(awk -F'"' '/^id *=/ {id=$2} /^name *=/ {print id, $2}' "$PROXY_DIR/printers.toml") + +# --- 2. backend ------------------------------------------------------------ + +echo "==> starting continuum-backend on :$PORT" +bun run src/index.ts > "$WORK/backend.log" 2>&1 & +BACKEND_PID=$! +for _ in $(seq 1 60); do + curl -sf "$BASE/health" >/dev/null 2>&1 && break + sleep 0.5 +done +HEALTH="$(curl -sf "$BASE/health")" || fail "backend did not become healthy" +echo "$HEALTH" | grep -q '"db":true' || fail "backend reports the database is down: $HEALTH" +echo "$HEALTH" | grep -q '"valkey":true' || fail "backend reports valkey is down: $HEALTH" +ok "health: $HEALTH" + +# --- 3. edge token --------------------------------------------------------- + +echo "==> minting an edge token" +TOKEN="$(curl -sf -X POST "$BASE/auth/edge-token" \ + -H 'content-type: application/json' \ + -d "{\"farmId\":\"$FARM_ID\",\"edgeProxyToken\":\"$EDGE_PROXY_TOKEN\"}" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin).get("token",""))')" +[ -n "$TOKEN" ] || fail "no token returned from POST /auth/edge-token" +ok "token minted (${#TOKEN} chars)" + +# --- 4. the real gateway --------------------------------------------------- + +echo "==> starting the real continuum-proxy against it" +( cd "$PROXY_DIR" && cargo build --quiet 2>&1 | tail -3 ) || fail "proxy did not build" +# env -C rather than a `( cd ... ) &` subshell: $! has to be the daemon itself, +# or cleanup kills only the wrapper and leaves an orphan reconnecting with a +# stale token. An earlier version did exactly that, and the orphan then +# reported a deleted farm's printers into the next run. +env -C "$PROXY_DIR" \ + CONTINUUM_GATEWAY_ID=e2e-gw \ + CONTINUUM_FARM_ID="$FARM_ID" \ + CONTINUUM_GATEWAY_TOKEN="$TOKEN" \ + CONTINUUM_UPLINK_URL="ws://localhost:$PORT/ws/edge/v1" \ + CONTINUUM_GO2RTC_BIN=/nonexistent-go2rtc \ + CONTINUUM_GO2RTC_CONFIG=./go2rtc.yaml \ + CONTINUUM_PRINTERS_PATH=printers.toml \ + RUST_LOG=info \ + "$PROXY_DIR/target/debug/continuum-proxy" > "$WORK/proxy.log" 2>&1 & +PROXY_PID=$! + +# Wait for *every* claimed printer, not just the first. Telemetry is on a 5s +# timer and a Bambu on protocol v1 reports one tick behind the rest (it only +# sends a full status when asked, so each tick reads the previous tick's +# answer) - stopping at the first row that appears is how an earlier version of +# this script silently tested only the v2 printer. +CLAIMED="$(psql_q "SELECT count(*) FROM printers WHERE farm_id='$FARM_ID' AND gateway_printer_id IS NOT NULL")" +echo "==> waiting for live telemetry from all $CLAIMED claimed printer(s)" +for _ in $(seq 1 45); do + COUNT="$(psql_q "SELECT count(*) FROM printers + WHERE farm_id='$FARM_ID' AND gateway_printer_id IS NOT NULL + AND last_seen_at IS NOT NULL AND vendor_state IS NOT NULL")" + [ "${COUNT:-0}" -ge "$CLAIMED" ] && break + sleep 1 +done + +[ "${COUNT:-0}" -ge 1 ] || fail "no telemetry was persisted at all (see the proxy log below)" + +if [ "${COUNT:-0}" -lt "$CLAIMED" ]; then + # Not fatal: a printer that is genuinely switched off cannot report, and that + # should not fail a run against the ones that can. Name them so the + # difference between "off" and "broken" is visible. + echo " WARNING: only $COUNT of $CLAIMED printers reported. Silent:" + psql_q "SELECT ' - ' || gateway_printer_id FROM printers + WHERE farm_id='$FARM_ID' AND gateway_printer_id IS NOT NULL + AND (last_seen_at IS NULL OR vendor_state IS NULL)" +else + ok "all $COUNT claimed printer(s) reported live telemetry" +fi + +if grep "reported unknown printer" "$WORK/backend.log" | grep -q "$FARM_ID"; then + fail "the backend could not map a gateway printer id for this farm - check gateway_printer_id" +fi +ok "every printer id this gateway reported mapped to a row" + +echo "==> what landed in the database:" +PGPASSWORD=continuum psql -h localhost -U continuum -d continuum -c \ + "SELECT gateway_printer_id AS gw, status, vendor_state, progress_percent AS pct, + current_layer || '/' || total_layers AS layers, remaining_minutes AS mins, + nozzle_temp_c AS nozzle, bed_temp_c AS bed, active_alerts AS alerts, + coalesce(current_job_name, '-') AS job + FROM printers WHERE farm_id = '$FARM_ID' ORDER BY gateway_printer_id" + +# Proves the telemetry is real rather than a row touched with nulls. +psql_q "SELECT 1 FROM printers WHERE farm_id='$FARM_ID' AND vendor_state IS NOT NULL + AND nozzle_temp_c IS NOT NULL LIMIT 1" | grep -q 1 \ + || fail "rows were touched but carry no actual telemetry" +ok "rows carry real vendor state and temperatures" + +# --- 5. a command, the whole way down -------------------------------------- + +TARGET_UUID="$(psql_q "SELECT id FROM printers WHERE farm_id='$FARM_ID' AND gateway_printer_id='$PRINTER_ID'")" +[ -n "$TARGET_UUID" ] || fail "no printer claimed as '$PRINTER_ID'" + +GW="$(curl -sf "$BASE/printers/$TARGET_UUID/gateway")" +echo "$GW" | grep -q '"gatewayOnline":true' || fail "the gateway is not registered as online: $GW" +ok "gateway reports online for $PRINTER_ID" + +COMMAND_TESTED=no +STATE="$(psql_q "SELECT vendor_state FROM printers WHERE id='$TARGET_UUID'")" +if [ "$STATE" != "IDLE" ] && [ "$STATE" != "FINISH" ]; then + echo " SKIPPED the command test: $PRINTER_ID reports '${STATE:-no telemetry yet}', not IDLE/FINISH." + echo " Refusing to touch a printer that may be mid-print." +else + echo "==> sending set_fan_speed to $PRINTER_ID ($STATE) over HTTP" + RESP="$(curl -s -o "$WORK/cmd.json" -w '%{http_code}' -X POST \ + "$BASE/printers/$TARGET_UUID/commands" -H 'content-type: application/json' \ + -d '{"action":"set_fan_speed","percent":40}')" + [ "$RESP" = "200" ] || fail "command returned HTTP $RESP: $(cat "$WORK/cmd.json")" + grep -q '"ok":true' "$WORK/cmd.json" || fail "command was not acknowledged: $(cat "$WORK/cmd.json")" + ok "command acknowledged by the printer: $(cat "$WORK/cmd.json")" + + # Put it back, so this leaves the printer as it found it. + curl -s -X POST "$BASE/printers/$TARGET_UUID/commands" \ + -H 'content-type: application/json' \ + -d '{"action":"set_fan_speed","percent":0}' > /dev/null + ok "fan returned to 0" + COMMAND_TESTED=yes +fi + +# --- 6. the negative case -------------------------------------------------- + +echo "==> an unclaimed printer must be refused, not guessed at" +ORPHAN="$(psql_q "INSERT INTO printers (farm_id, name, vendor) + VALUES ('$FARM_ID', 'unclaimed', 'bambu') RETURNING id")" +CODE="$(curl -s -o "$WORK/orphan.json" -w '%{http_code}' -X POST \ + "$BASE/printers/$ORPHAN/commands" -H 'content-type: application/json' \ + -d '{"action":"set_fan_speed","percent":0}')" +[ "$CODE" = "409" ] || fail "expected HTTP 409 for an unclaimed printer, got $CODE" +grep -q 'PRINTER_UNREACHABLE' "$WORK/orphan.json" || fail "wrong error: $(cat "$WORK/orphan.json")" +ok "unclaimed printer refused with 409 PRINTER_UNREACHABLE" + +echo +# Say what was actually exercised. Claiming commands work when the command test +# was skipped is worse than not running it. +if [ "$COMMAND_TESTED" = yes ]; then + echo "PASS - telemetry and commands both verified against a real database and real printers." +else + echo "PARTIAL PASS - telemetry verified against a real database and real printers." + echo " The command path was NOT exercised: no printer was idle enough to send to." + echo " Rerun with PRINTER_ID= once one is free." +fi diff --git a/scripts/install-docker.sh b/scripts/install-docker.sh new file mode 100755 index 0000000..1e93666 --- /dev/null +++ b/scripts/install-docker.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# One-time host setup: installs Docker Engine + the compose plugin from +# Docker's own apt repository. +# +# Separate from dev-env.sh on purpose - this touches the machine (apt, a new +# apt source, a usermod), dev-env.sh only touches containers. Run this once, +# then use dev-env.sh for everything else. +# +# Ubuntu-specific, and written for this workspace's WSL2 environment where +# PID 1 is not systemd: nothing here enables a service, because there is no +# service manager to enable it with. dev-env.sh starts dockerd itself. +set -euo pipefail + +if command -v docker >/dev/null 2>&1; then + echo "docker is already installed: $(docker --version)" + exit 0 +fi + +. /etc/os-release +echo "==> installing Docker Engine for ${ID} ${VERSION_CODENAME}" + +sudo apt-get update -qq +sudo apt-get install -y -qq ca-certificates curl gnupg iptables + +sudo install -m 0755 -d /etc/apt/keyrings +curl -fsSL "https://download.docker.com/linux/${ID}/gpg" \ + | sudo gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg +sudo chmod a+r /etc/apt/keyrings/docker.gpg + +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ +https://download.docker.com/linux/${ID} ${VERSION_CODENAME} stable" \ + | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + +sudo apt-get update -qq +sudo apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-compose-plugin + +# So dev-env.sh can talk to the daemon without sudo. Needs a fresh login to +# take effect, which is why dev-env.sh also falls back to sudo. +sudo usermod -aG docker "$USER" || true + +echo "==> installed: $(docker --version)" +echo "==> $(docker compose version)" +echo +echo "Next: scripts/dev-env.sh up" diff --git a/src/db/validators.ts b/src/db/validators.ts index 3ba4344..925c006 100644 --- a/src/db/validators.ts +++ b/src/db/validators.ts @@ -11,12 +11,22 @@ import { printers, printJobs } from "./schema"; // The second argument overrides fields Drizzle can't infer correctly for an // insert - `id` is DB-generated (defaultRandom()) so it must be optional on // the way in, even though it's required on the way out. +// +// `metadata` needs an override for a different reason: drizzle-typebox widens a +// `jsonb` column to "any JSON value", so the generated schema admits a bare +// string or number. The column is declared `$type>()`, +// so a validated body would then not be assignable to `.values()` - the object +// shape has to be restated here. +const JsonObject = t.Optional(t.Record(t.String(), t.Unknown())); + export const PrinterInsertSchema = createInsertSchema(printers, { id: t.Optional(t.String({ format: "uuid" })), + metadata: JsonObject, }); export const PrinterSelectSchema = createSelectSchema(printers); export const PrintJobInsertSchema = createInsertSchema(printJobs, { id: t.Optional(t.String({ format: "uuid" })), + metadata: JsonObject, }); export const PrintJobSelectSchema = createSelectSchema(printJobs); diff --git a/src/routes/ws/edge.ts b/src/routes/ws/edge.ts index 96f4cae..4459a8c 100644 --- a/src/routes/ws/edge.ts +++ b/src/routes/ws/edge.ts @@ -307,7 +307,10 @@ export const edgeWsRoute = new Elysia() } clearTimeout(pending.timer); conn.pending.delete(message.command_id); - pending.resolve({ ok: message.ok, error: message.error }); + // `?? undefined` because the wire allows either absent or null for + // "no error" (see the Maybe() helper in @continuum/ts-types) and + // callers should only have to check one of them. + pending.resolve({ ok: message.ok, error: message.error ?? undefined }); break; } diff --git a/tests/dependency_identity.test.ts b/tests/dependency_identity.test.ts new file mode 100644 index 0000000..0f05348 --- /dev/null +++ b/tests/dependency_identity.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "bun:test"; +import { dirname, join } from "node:path"; +import { existsSync, readFileSync } from "node:fs"; + +/** + * Guards the one dependency mistake that broke the shared schemas, because + * nothing else catches it and the symptom points somewhere else entirely. + * + * `@continuum/ts-types` is linked from continuum-schemas and resolves its own + * dependencies from that repo's node_modules. It had TypeBox 0.32.37 while + * this backend had 0.33.24, and two *different* TypeBox versions mean two + * structurally incompatible `TSchema` types - so Elysia rejected every schema + * from there, with a message that pointed somewhere else entirely. + * + * The misleading part: Elysia's validator accepts + * `TSchema | StandardSchemaV1Like`, so when the TSchema branch failed + * TypeScript reported the *other* branch's failure, `'~standard' is missing`. + * That reads like Elysia wants a Standard Schema validator and TypeBox is the + * wrong tool, which is not true at all - Elysia is TypeBox-native (`t` is a + * TypeBox JavaScriptTypeBuilder, and the dist `require`s @sinclair/typebox at + * runtime without even declaring it as a dependency). + * + * Note what this does *not* check: that both resolve to the same file. A + * `file:` dependency keeps its own node_modules, so there are legitimately two + * copies on disk. Two copies of the *same version* are interchangeable here, + * because TypeBox's types are structural and its kind markers use + * `Symbol.for()`, which is global across copies. It is the version skew that + * breaks, not the duplication. + */ + +/** The installed version of `name`, resolved as `from` would resolve it. */ +function installedVersion(name: string, from: string): string { + let dir = dirname(Bun.resolveSync(name, from)); + // Walk up from the entry file to the package root - resolveSync returns e.g. + // build/esm/index.mjs, and package.json is not always an allowed export. + for (let i = 0; i < 10; i++) { + const candidate = join(dir, "package.json"); + if (existsSync(candidate)) { + const pkg = JSON.parse(readFileSync(candidate, "utf8")); + if (pkg.name === name) return pkg.version; + } + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + throw new Error(`could not find the installed package.json for ${name} from ${from}`); +} + +describe("@sinclair/typebox version is identical everywhere", () => { + it("matches between this backend and @continuum/ts-types", () => { + const here = installedVersion("@sinclair/typebox", import.meta.dir); + const tsTypesDir = dirname(Bun.resolveSync("@continuum/ts-types", import.meta.dir)); + const there = installedVersion("@sinclair/typebox", tsTypesDir); + + expect(there).toBe(here); + }); + + it("is pinned exactly, not as a range", () => { + // A caret range is what let the two drift apart in the first place: this + // backend and ts-types each resolved their own newest match. + const pkg = JSON.parse(readFileSync(`${import.meta.dir}/../package.json`, "utf8")); + const pinned = pkg.dependencies["@sinclair/typebox"]; + expect(pinned).not.toMatch(/^[\^~]/); + expect(installedVersion("@sinclair/typebox", import.meta.dir)).toBe(pinned); + }); +}); diff --git a/tests/edge_protocol.test.ts b/tests/edge_protocol.test.ts index f2f2c8d..f0d59df 100644 --- a/tests/edge_protocol.test.ts +++ b/tests/edge_protocol.test.ts @@ -14,8 +14,8 @@ import legacyCapture from "./fixtures/gateway_frames_legacy_nulls.json"; * /ws/edge/v1 had silently drifted into completely different protocols * (different handshake, different telemetry shape, printer ids the gateway * could not know), and every hand-written example agreed with whichever side - * wrote it. Re-capture with the script in the proxy's examples/ when the - * protocol changes, rather than editing these by hand. + * wrote it. Re-capture with scripts/capture-gateway-frames.js when the protocol + * changes, rather than editing these by hand. */ function check(schema: Parameters[0], value: unknown) {