From 57af710b1ad4a327974c454062b0f56603083f2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iwo=20Strzebo=C5=84ski?= Date: Sat, 17 Oct 2026 13:59:56 +0000 Subject: [PATCH] Verify the edge token at the upgrade, not in the first frame `/ws/edge/v1` took its credential from a `token` field on the `hello` message. It now takes it from `Authorization: Bearer` on the upgrade request, verified before the socket exists, so an unauthenticated peer gets an HTTP 401 and no WebSocket: no entry in `connections`, no frame ever parsed, no state of any kind allocated for it. The old arrangement worked, and every message type refused to act without a verified farm. But "it cannot do anything" is a weaker guarantee than "it does not exist" - an anonymous peer still got a live socket and had its JSON run through `EdgeClientMessageSchema` and the message switch before proving anything, and it took a timeout to stop anyone holding sockets open for free. A `?token=` query parameter was never on the table: a bearer credential in a URL lands in access logs, proxy logs and error breadcrumbs, which is the thing the proxy's hand-written `Debug` for `PrinterCommand` already exists to prevent. The decision lives in `src/auth/edge_upgrade.ts`, a leaf module with no imports but the error types - the same reason as `dispatch/job_status.ts` and `stream.ts`. This is the authentication boundary for the entire gateway protocol, so it should be the last thing only reachable through a running database. It checks `scope` and not just the signature (the same secret could be made to sign something else one day), rejects a token whose farm claim is absent or not a string (that would otherwise produce a connection whose `farmId` is `undefined`, which reads as "every farm" to a lookup), and answers every failure with one message - which part of a forged token to fix next is not something to tell its holder. `conn.farmId` is therefore set from the verified token in `open`, and the `hello` handler no longer verifies anything. It still checks `message.farm_id` against the token's farm, so a gateway naming a farm its token was not minted for is refused rather than silently working against the wrong one. The handshake timer stays but changed meaning: it is no longer an auth timeout, just a bound on an authenticated socket that never identifies its gateway. Verified, not assumed. That a throwing `resolve` fails the upgrade is a framework behaviour the whole design rests on, so `tests/ edge_upgrade.test.ts` stands up a real Elysia app with a real JWT plugin and asserts a socket never opens without a valid edge token, and that a resolved value reaches `ws.data`. The real `continuum-proxy` was also run against this composition: valid token gives HTTP 101 with the farm taken from the token, forged token gives 401 at the handshake. The committed captures predate this and still contain `token: "dev-token"`. That validates - the schema is a liberal reader - and is harmless, because the field is now ignored and such a gateway is refused at the upgrade for having no header. A test says so. 95 tests (was 85), `bunx tsc --noEmit` clean. --- src/auth/edge_upgrade.ts | 61 ++++++++++++++++ src/routes/ws/edge.ts | 103 +++++++++++++++++++-------- tests/edge_protocol.test.ts | 34 +++++++++ tests/edge_upgrade.test.ts | 136 ++++++++++++++++++++++++++++++++++++ 4 files changed, 303 insertions(+), 31 deletions(-) create mode 100644 src/auth/edge_upgrade.ts create mode 100644 tests/edge_upgrade.test.ts diff --git a/src/auth/edge_upgrade.ts b/src/auth/edge_upgrade.ts new file mode 100644 index 0000000..f50703a --- /dev/null +++ b/src/auth/edge_upgrade.ts @@ -0,0 +1,61 @@ +import { UnauthorizedError, extractBearerToken } from "@continuum/ts-core"; + +/** + * Edge-token checking for the `/ws/edge/v1` upgrade, with no imports beyond + * the error types - the same reason `dispatch/job_status.ts` and `stream.ts` + * are leaf modules: `../db` opens a connection at import time, and this is + * the authentication boundary for the whole gateway protocol, so it is the + * last thing that should only be reachable through a running database. + */ + +/** + * Whatever verifies an edge JWT. Narrowed to what this needs rather than + * taking the `@elysiajs/jwt` decorator's type, so a test can pass a stub and + * exercise the real decision logic. + */ +export type EdgeTokenVerifier = ( + token: string, +) => Promise | false | null | undefined>; + +/** + * The farm a verified edge token belongs to, or an `UnauthorizedError`. + * + * Every failure gets one message. Which part of a forged token to fix next - + * signature, scope, expiry - is not something to tell whoever is holding + * one; the reason is logged instead. Same reasoning as `requireUser`. + * + * The farm is returned rather than the claims: it is the only thing the + * socket needs, and handing back the whole payload invites a later caller to + * start trusting some other claim without deciding to. + */ +export async function farmFromEdgeUpgrade( + authorization: string | null | undefined, + verify: EdgeTokenVerifier, +): Promise { + let token: string; + try { + token = extractBearerToken(authorization); + } catch { + throw new UnauthorizedError("Authorization header must be 'Bearer '"); + } + + const payload = await verify(token); + + // `scope` is checked, not just the signature: the same secret could be + // made to sign something else one day, and a token minted for another + // purpose must not open a farm's gateway socket. + if (!payload || payload.scope !== "edge") { + console.warn("[edge] upgrade rejected: token missing or not scoped to edge"); + throw new UnauthorizedError("Invalid or expired edge token"); + } + + // A token whose farm claim is absent or not a string would otherwise + // produce a connection whose `farmId` is `undefined`, which reads as + // "every farm" to anything doing a lookup. + if (typeof payload.farmId !== "string" || payload.farmId.length === 0) { + console.warn("[edge] upgrade rejected: edge token carries no farm"); + throw new UnauthorizedError("Invalid or expired edge token"); + } + + return payload.farmId; +} diff --git a/src/routes/ws/edge.ts b/src/routes/ws/edge.ts index d335d4a..2b4b391 100644 --- a/src/routes/ws/edge.ts +++ b/src/routes/ws/edge.ts @@ -12,6 +12,7 @@ import { printers } from "../../db/schema"; import { advanceJobsFrom } from "../../dispatch"; import { enqueueFrameIngestionBatch } from "../../queue"; import { edgeJwtSecret } from "../../auth/edge_secret"; +import { farmFromEdgeUpgrade } from "../../auth/edge_upgrade"; /** * Edge-gateway uplink. A continuum-proxy gateway connects here with a @@ -19,6 +20,37 @@ import { edgeJwtSecret } from "../../auth/edge_secret"; * for its whole printer fleet, and receives commands for individual printers * on the same socket. * + * ## Where the token goes, and why here + * + * `Authorization: Bearer ` on the upgrade request, verified before + * the socket exists. An unauthenticated peer therefore gets an HTTP 401 and + * no WebSocket at all: no entry in `connections`, no frame ever parsed, no + * state of any kind allocated for it. + * + * The two alternatives were both weaker, and one of them is what this used + * to do: + * + * - **In the first message** (a `token` field on `hello`, which is what this + * route did until now). It worked, but it meant an unauthenticated peer + * got a live socket, an entry in `connections`, and its JSON run through + * `EdgeClientMessageSchema` and this switch statement before proving + * anything. Every message type refused to act without a verified farm, so + * nothing could be *done* with it - but "it cannot do anything" is a + * weaker guarantee than "it does not exist", and it needed a timeout to + * stop anyone holding sockets open for free. + * - **A `?token=` query parameter.** A bearer credential in a URL ends up in + * access logs, proxy logs and error breadcrumbs. This codebase already + * goes to some trouble to avoid exactly that - `PrinterCommand` in the + * proxy has a hand-written `Debug` so a presigned URL prints as + * ``. `Authorization` is the header every log tool knows to + * redact; a query string is not. + * + * A browser cannot set headers on a WebSocket, so a future user-facing + * socket cannot copy this exactly. Its equivalent is the subprotocol field + * (`new WebSocket(url, ["bearer", token])`), which is also verified at the + * upgrade - the same property, through the one handshake field a browser + * does control. + * * The wire format is `schemas/continuum/v1/edge/` in continuum-schemas, * validated here through its TypeBox mirror in `@continuum/ts-types`. It is * snake_case, unlike this backend's HTTP routes: the other end is @@ -49,16 +81,16 @@ interface EdgeConn { */ streamBaseUrl?: string; /** - * Closes this socket if it never authenticates. Cleared by a successful - * `hello` and on close. + * Closes this socket if it never identifies itself. Cleared by a + * successful `hello` and on close. * - * The cost of carrying the token in the first message rather than the - * handshake is exactly this: there is a window in which an open socket - * belongs to nobody. Every other message type already refuses to act - * without a `farmId`, so such a socket can do nothing - but it does sit in - * `connections` forever, and that map is keyed by whoever can reach the - * port. A silent connection is not a slow one: a real gateway sends - * `hello` as its first frame. + * No longer an authentication timeout - the token is verified at the + * upgrade, so every socket here already belongs to a farm. What it bounds + * now is a protocol question: an authenticated socket that never sends + * `hello` has no `gateway_id`, so nothing can be logged about it usefully + * and it is holding a connection it is not using. Kept because a real + * gateway sends `hello` as its first frame, so silence means something is + * wrong rather than slow. */ handshakeTimer?: ReturnType; send: (message: EdgeServerMessage) => void; @@ -150,10 +182,10 @@ export function listUnclaimedPrinters(farmId: string): UnclaimedPrinter[] { const COMMAND_TIMEOUT_MS = 10_000; /** - * How long a socket may stay open without authenticating. Generous compared - * with the round trip it covers - the gateway sends `hello` immediately after - * the socket opens - because the only thing a tight bound would buy is - * dropping gateways on slow links. + * How long an authenticated socket may stay open without sending `hello`. + * Generous compared with the round trip it covers - a gateway sends `hello` + * immediately after the socket opens - because the only thing a tight bound + * would buy is dropping gateways on slow links. */ const HANDSHAKE_TIMEOUT_MS = 10_000; @@ -333,14 +365,30 @@ export function getStreamBaseUrl(farmId: string): string | undefined { export const edgeWsRoute = new Elysia() .use(jwt({ name: "edgeJwt", secret: edgeJwtSecret })) + /** + * Verifies the edge JWT on the upgrade request, before any socket exists. + * + * Throwing here fails the upgrade with an HTTP status, so a peer without a + * valid token never reaches `open` - which is the whole point of + * authenticating at the handshake rather than in the first frame. Modelled + * on `requireUser`'s scoped resolve, for the same reason: one verification + * per connection, in one place, ahead of every handler. + */ + .resolve({ as: "scoped" }, async ({ headers, edgeJwt }): Promise<{ edgeFarmId: string }> => ({ + edgeFarmId: await farmFromEdgeUpgrade(headers.authorization, (token) => edgeJwt.verify(token)), + })) .ws("/ws/edge/v1", { body: EdgeClientMessageSchema, open(ws) { // No farmId yet - set by `hello`, and every other message type refuses // to do anything until it is. + // The farm comes from the verified token, so it is known before the + // first frame. `gatewayId` still waits for `hello`: which gateway of a + // farm's this is, and where it serves camera streams, are things only + // the gateway can say. const conn: EdgeConn = { - farmId: "", + farmId: ws.data.edgeFarmId, gatewayId: "", sessionId: crypto.randomUUID(), lastSeenAt: Date.now(), @@ -350,7 +398,7 @@ export const edgeWsRoute = new Elysia() }; conn.handshakeTimer = setTimeout(() => { - if (conn.farmId) return; + if (conn.gatewayId) return; console.warn(`[edge] closing socket ${ws.id}: no hello within ${HANDSHAKE_TIMEOUT_MS}ms`); ws.close(); }, HANDSHAKE_TIMEOUT_MS); @@ -366,31 +414,24 @@ export const edgeWsRoute = new Elysia() switch (message.type) { case "hello": { - const payload = await ws.data.edgeJwt.verify(message.token); - if (!payload || payload.scope !== "edge") { - ws.send({ type: "hello_error", message: "invalid edge token" }); - ws.close(); - return; - } - - // The token decides the farm, never the message body: a gateway - // claiming a farm_id its token was not minted for is exactly what - // this check is here to stop. - const farmId = payload.farmId as string; + // Already authenticated at the upgrade, so there is no token to + // check here. The farm still comes from that token and never from + // the message body: a gateway claiming a farm_id its token was not + // minted for is exactly what the next check is here to stop. + const farmId = conn.farmId; if (message.farm_id && message.farm_id !== farmId) { ws.send({ type: "hello_error", message: "farm_id does not match the token's farm" }); ws.close(); return; } - // Authenticated, so it is no longer a socket belonging to nobody. - // Cleared only here, after the token verified and the farm matched - // - a rejected hello leaves the timer running, so a client that - // retries a bad token on one socket still gets dropped. + // Identified, so the timer has nothing left to catch. Cleared + // only after the farm matched - a rejected hello leaves it + // running, so a client that keeps sending mismatched farm ids on + // one socket still gets dropped. clearTimeout(conn.handshakeTimer); conn.handshakeTimer = undefined; - conn.farmId = farmId; conn.gatewayId = message.gateway_id; // `?? undefined`: absent and explicit null both mean "no streams". conn.streamBaseUrl = message.stream_base_url ?? undefined; diff --git a/tests/edge_protocol.test.ts b/tests/edge_protocol.test.ts index f853d58..4fba01d 100644 --- a/tests/edge_protocol.test.ts +++ b/tests/edge_protocol.test.ts @@ -100,6 +100,40 @@ describe("gateway -> backend frames", () => { // The old protocol's handshake, which no gateway ever sent. expect(Value.Check(EdgeClientMessageSchema, { type: "auth", token: "x" })).toBe(false); }); + + /** + * The hello carries no credential any more - the edge JWT goes in an + * `Authorization: Bearer` header on the upgrade, verified before the + * socket exists (see tests/edge_upgrade.test.ts). This asserts the schema + * does not require the `token` field it used to. + * + * Note the committed captures predate that change and still contain + * `token: "dev-token"`, which validates because this schema is a liberal + * reader. That is deliberate and harmless: the field is now ignored, and + * an old gateway sending it is refused at the upgrade for having no + * header, so there is no path by which a token in a frame authenticates + * anything. + */ + it("accepts a hello with no token at all", () => { + check(EdgeClientMessageSchema, { + type: "hello", + gateway_id: "dev-gw", + farm_id: "dev-farm", + }); + check(EdgeClientMessageSchema, { + type: "hello", + gateway_id: "dev-gw", + farm_id: "dev-farm", + stream_base_url: "http://gateway.local:1984", + }); + }); + + it("still requires the fields a hello cannot work without", () => { + // `farm_id` is checked against the token's own claim rather than + // trusted, which is only possible if it is actually sent. + expect(Value.Check(EdgeClientMessageSchema, { type: "hello", gateway_id: "g" })).toBe(false); + expect(Value.Check(EdgeClientMessageSchema, { type: "hello", farm_id: "f" })).toBe(false); + }); }); describe("backend -> gateway frames", () => { diff --git a/tests/edge_upgrade.test.ts b/tests/edge_upgrade.test.ts new file mode 100644 index 0000000..b9dee6e --- /dev/null +++ b/tests/edge_upgrade.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it } from "bun:test"; +import { Elysia } from "elysia"; +import jwt from "@elysiajs/jwt"; +import { UnauthorizedError } from "@continuum/ts-core"; +import { farmFromEdgeUpgrade } from "../src/auth/edge_upgrade"; + +/** + * The gateway protocol's authentication boundary. + * + * The edge JWT travels in `Authorization: Bearer` on the upgrade request and + * is verified before any socket exists. It used to be a `token` field on the + * `hello` message, which meant an unauthenticated peer got a live socket and + * a parsed frame before proving anything. These tests are what stops that + * coming back. + */ +describe("farmFromEdgeUpgrade", () => { + const verifyOk = async (token: string) => + token === "good" ? { scope: "edge", farmId: "farm-1" } : false; + + it("returns the farm from a valid edge token", async () => { + expect(await farmFromEdgeUpgrade("Bearer good", verifyOk)).toBe("farm-1"); + }); + + it("refuses a missing or malformed Authorization header", async () => { + for (const header of [undefined, null, "", "good", "Basic good", "Bearer"]) { + await expect(farmFromEdgeUpgrade(header, verifyOk)).rejects.toThrow(UnauthorizedError); + } + }); + + it("refuses a token the verifier rejects", async () => { + await expect(farmFromEdgeUpgrade("Bearer forged", verifyOk)).rejects.toThrow(UnauthorizedError); + }); + + /** + * A signature alone is not authorisation. The same secret could be made to + * sign something else one day, and such a token must not open a farm's + * gateway socket. + */ + it("refuses a correctly signed token minted for a different scope", async () => { + const verify = async () => ({ scope: "user", farmId: "farm-1" }); + await expect(farmFromEdgeUpgrade("Bearer good", verify)).rejects.toThrow(UnauthorizedError); + }); + + /** + * Without this, the connection's `farmId` would be `undefined`, which to + * anything doing a lookup reads as "every farm" rather than as an error. + */ + it("refuses a token with no usable farm claim", async () => { + for (const farmId of [undefined, null, "", 42, {}]) { + const verify = async () => ({ scope: "edge", farmId }) as Record; + await expect(farmFromEdgeUpgrade("Bearer good", verify)).rejects.toThrow(UnauthorizedError); + } + }); + + /** + * The messages must not say which part failed - that tells the holder of a + * forged token what to fix next. Asserted rather than assumed, because the + * natural way to write this code is one specific message per branch. + */ + it("says the same thing however the token fails", async () => { + const messages = new Set(); + for (const verify of [ + async () => false as const, + async () => ({ scope: "user", farmId: "farm-1" }), + async () => ({ scope: "edge" }) as Record, + ]) { + await farmFromEdgeUpgrade("Bearer x", verify).catch((err: Error) => + messages.add(err.message), + ); + } + expect(messages.size).toBe(1); + }); +}); + +/** + * That the check above actually gates the upgrade, which is the property the + * whole design rests on and is a framework behaviour rather than ours: a + * `resolve` that throws must fail the HTTP upgrade so `open` never runs. + * + * This builds its own Elysia app rather than importing `edgeWsRoute`, which + * pulls in `../../db` and so a live Postgres. What it proves is that this + * composition refuses a socket and that a resolved value reaches `ws.data`; + * what it cannot prove is that `edge.ts` is wired this way. `scripts/ + * e2e-edge.sh` runs the real route against a real database and a real + * gateway, and is the test for that. + */ +describe("a resolve that throws gates the websocket upgrade", () => { + const SECRET = "edge-upgrade-test-secret"; + + const app = new Elysia() + .use(jwt({ name: "edgeJwt", secret: SECRET })) + .get("/mint", ({ edgeJwt, query }) => + edgeJwt.sign({ scope: query.scope ?? "edge", farmId: "farm-1" }), + ) + .resolve({ as: "scoped" }, async ({ headers, edgeJwt }): Promise<{ edgeFarmId: string }> => ({ + edgeFarmId: await farmFromEdgeUpgrade(headers.authorization, (t) => edgeJwt.verify(t)), + })) + .ws("/ws/edge/v1", { + open(ws) { + ws.send(JSON.stringify({ farm: (ws.data as { edgeFarmId: string }).edgeFarmId })); + }, + }) + .listen(0); + + const base = `localhost:${app.server!.port}`; + + function attempt(headers: Record): Promise { + return new Promise((resolve) => { + const ws = new WebSocket(`ws://${base}/ws/edge/v1`, { headers } as never); + const done = (r: string) => { + try { + ws.close(); + } catch { + // Already closed by the server; nothing to do. + } + resolve(r); + }; + ws.onmessage = (event) => done(`opened:${event.data}`); + ws.onerror = () => done("refused"); + ws.onclose = () => done("refused"); + setTimeout(() => done("timed out"), 5_000); + }); + } + + it("opens for a valid edge token and exposes its farm", async () => { + const token = await (await fetch(`http://${base}/mint`)).text(); + expect(await attempt({ Authorization: `Bearer ${token}` })).toBe('opened:{"farm":"farm-1"}'); + }); + + it("never opens a socket without a valid edge token", async () => { + const wrongScope = await (await fetch(`http://${base}/mint?scope=user`)).text(); + expect(await attempt({})).toBe("refused"); + expect(await attempt({ Authorization: "Bearer not-a-jwt" })).toBe("refused"); + expect(await attempt({ Authorization: `Bearer ${wrongScope}` })).toBe("refused"); + }); +});