Raise alerts: failed or paused prints, and AI detections
Nothing subscribed to channel:print_alerts and nothing watched a printer fail, so an operator away from the screen found out when they came back to a ruined plate. Alerts are now rows in printer_alerts, listed at GET /alerts (open=true for what still needs a person) and cleared with POST /alerts/:id/acknowledge, which is operator-only and keeps the first acknowledger on the record. Two sources. Telemetry: a change into error is a failed print if the printer was printing or paused, otherwise a printer error; a print the printer paused itself (how an AMS fault looks) is announced too. Transitions, not levels - a printer sitting in error is one alert, not one per frame - and a connection's first sight of a printer raises nothing, so a gateway that reconnects does not re-announce known faults. AI: the worker's Valkey publications are stored when confidence is at least ALERT_MIN_CONFIDENCE (default 0.6); the worker deliberately does not threshold, so the policy lives here. A bad message costs one message, not the listener. e2e-authz.sh gains the scoping/ack cases; new e2e-alerts.sh drives a stand-in gateway over the real /ws/edge/v1 and the real Valkey channel (checked to fail when the derivation is broken).
This commit is contained in:
Executable
+216
@@ -0,0 +1,216 @@
|
||||
#!/usr/bin/env bash
|
||||
# End-to-end test of alerts: a stand-in gateway reports a printer going
|
||||
# from printing to failed over the real /ws/edge/v1, and an AI detection is
|
||||
# published on the real Valkey channel; both must become rows an operator can
|
||||
# list and acknowledge.
|
||||
#
|
||||
# This is the test that the unit tests cannot be. tests/authz.test.ts proves
|
||||
# tokens are verified and that the role ladder orders correctly; only this
|
||||
# proves that each route actually asks, that scoping is applied in the query
|
||||
# rather than after it, and that credentials stay out of responses - the three
|
||||
# things that were wrong when every route was unauthenticated.
|
||||
#
|
||||
# Four users, two organisations, no printers or gateways needed:
|
||||
# owner_a, operator_a, viewer_a in org A, which owns farm A and printer A
|
||||
# outsider_b in org B, which owns nothing
|
||||
#
|
||||
# Usage:
|
||||
# scripts/dev-env.sh up && scripts/dev-env.sh schema
|
||||
# scripts/e2e-authz.sh
|
||||
#
|
||||
# KEEP_DATA=1 leave the seeded organisations behind for inspection
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
PORT="${PORT:-3030}"
|
||||
AUTH_PORT="${AUTH_PORT:-3031}"
|
||||
BASE="http://localhost:$PORT"
|
||||
DATABASE_URL="${DATABASE_URL:-postgres://continuum:continuum@localhost:5432/continuum}"
|
||||
VALKEY_URL="${VALKEY_URL:-redis://localhost:6379}"
|
||||
EDGE_JWT_SECRET="${EDGE_JWT_SECRET:-e2e-edge-secret}"
|
||||
# Presigning a PUT is local HMAC with no network call, so POST /jobs works
|
||||
# offline - but the SDK refuses to sign with empty credentials.
|
||||
R2_ACCESS_KEY_ID="${R2_ACCESS_KEY_ID:-e2e}"
|
||||
R2_SECRET_ACCESS_KEY="${R2_SECRET_ACCESS_KEY:-e2e}"
|
||||
R2_ENDPOINT="${R2_ENDPOINT:-http://localhost:9000}"
|
||||
export DATABASE_URL VALKEY_URL EDGE_JWT_SECRET PORT
|
||||
export R2_ACCESS_KEY_ID R2_SECRET_ACCESS_KEY R2_ENDPOINT
|
||||
|
||||
WORK="$(mktemp -d)"
|
||||
BACKEND_PID=""
|
||||
AUTH_PID=""
|
||||
FAILURES=0
|
||||
|
||||
# `uuidgen` is not installed everywhere; the kernel's generator always is.
|
||||
new_uuid() {
|
||||
if [ -r /proc/sys/kernel/random/uuid ]; then
|
||||
cat /proc/sys/kernel/random/uuid
|
||||
else
|
||||
python3 -c 'import uuid; print(uuid.uuid4())'
|
||||
fi
|
||||
}
|
||||
|
||||
# Uses DATABASE_URL rather than a hardcoded localhost. The backend under test
|
||||
# reads that variable, so a psql pointed anywhere else seeds fixtures into a
|
||||
# different database than the one being exercised - and the failure message
|
||||
# below names DATABASE_URL, which would then be naming a URL nothing tried.
|
||||
psql_q() { psql "$DATABASE_URL" -tAq -c "$1"; }
|
||||
|
||||
cleanup() {
|
||||
local rc=$?
|
||||
# Never `kill $PID` with an unset PID: `kill 0` signals the whole process
|
||||
# group, which includes this script and whatever launched it.
|
||||
local pids=()
|
||||
[ -n "$BACKEND_PID" ] && pids+=("$BACKEND_PID")
|
||||
[ -n "$AUTH_PID" ] && pids+=("$AUTH_PID")
|
||||
if [ ${#pids[@]} -gt 0 ]; then
|
||||
kill "${pids[@]}" 2>/dev/null || true
|
||||
sleep 0.3
|
||||
kill -9 "${pids[@]}" 2>/dev/null || true
|
||||
fi
|
||||
if [ -z "${KEEP_DATA:-}" ]; then
|
||||
for org in "${ORG_A:-}" "${ORG_B:-}"; do
|
||||
[ -n "$org" ] && psql_q "DELETE FROM organisations WHERE id = '$org'" >/dev/null 2>&1 || true
|
||||
done
|
||||
fi
|
||||
if [ $rc -ne 0 ]; then
|
||||
echo
|
||||
echo "--- backend log (tail) ---"; tail -30 "$WORK/backend.log" 2>/dev/null || true
|
||||
fi
|
||||
rm -rf "$WORK"
|
||||
exit $rc
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
fail() { echo "FAIL: $*" >&2; exit 1; }
|
||||
ok() { echo " ok: $*"; }
|
||||
bad() { echo " FAILED: $*" >&2; FAILURES=$((FAILURES + 1)); }
|
||||
|
||||
# `req <method> <path> <token|-> [json body]` -> prints the status code,
|
||||
# leaves the response body in $WORK/last.json.
|
||||
req() {
|
||||
local method="$1" path="$2" token="$3" body="${4:-}"
|
||||
local args=(-s -o "$WORK/last.json" -w '%{http_code}' -X "$method" "$BASE$path")
|
||||
[ "$token" != "-" ] && args+=(-H "authorization: Bearer $token")
|
||||
[ -n "$body" ] && args+=(-H 'content-type: application/json' -d "$body")
|
||||
curl "${args[@]}"
|
||||
}
|
||||
|
||||
# `expect <code> <method> <path> <token> [body] -- <description>`
|
||||
expect() {
|
||||
local want="$1"; shift
|
||||
local method="$1" path="$2" token="$3"; shift 3
|
||||
local body=""
|
||||
if [ "${1:-}" != "--" ]; then body="$1"; shift; fi
|
||||
shift # the --
|
||||
local what="$*"
|
||||
local got; got="$(req "$method" "$path" "$token" "$body")"
|
||||
if [ "$got" = "$want" ]; then
|
||||
ok "$what ($got)"
|
||||
else
|
||||
bad "$what: expected $want, got $got - $(head -c 300 "$WORK/last.json")"
|
||||
fi
|
||||
}
|
||||
|
||||
body_has() { grep -q "$1" "$WORK/last.json" || bad "expected '$1' in: $(head -c 300 "$WORK/last.json")"; }
|
||||
body_lacks() { grep -q "$1" "$WORK/last.json" && bad "'$1' leaked into: $(head -c 300 "$WORK/last.json")" || true; }
|
||||
json() { python3 -c "import json,sys; d=json.load(sys.stdin); print($1)" < "$WORK/last.json"; }
|
||||
|
||||
# --- 0. preconditions ------------------------------------------------------
|
||||
|
||||
echo "==> checking the database is up and migrated"
|
||||
psql_q "SELECT 1" >/dev/null 2>&1 || fail "no database at $DATABASE_URL - run scripts/dev-env.sh up"
|
||||
psql_q "SELECT org_id FROM farms LIMIT 0" >/dev/null 2>&1 \
|
||||
|| fail "farms.org_id is missing - run scripts/dev-env.sh schema"
|
||||
psql_q "SELECT role FROM org_members LIMIT 0" >/dev/null 2>&1 \
|
||||
|| fail "org_members is missing - run scripts/dev-env.sh schema"
|
||||
ok "schema has the org layer"
|
||||
|
||||
# --- 1. the issuer and the backend ----------------------------------------
|
||||
|
||||
echo "==> starting the dev-auth JWKS issuer on :$AUTH_PORT"
|
||||
bun scripts/dev-auth.ts --port "$AUTH_PORT" > "$WORK/auth.log" 2>&1 &
|
||||
AUTH_PID=$!
|
||||
AUTH_URL="http://localhost:$AUTH_PORT"
|
||||
for _ in $(seq 1 40); do
|
||||
curl -sf "$AUTH_URL/auth/v1/.well-known/jwks.json" >/dev/null 2>&1 && break
|
||||
sleep 0.25
|
||||
done
|
||||
curl -sf "$AUTH_URL/auth/v1/.well-known/jwks.json" >/dev/null 2>&1 \
|
||||
|| fail "the dev-auth issuer did not come up: $(cat "$WORK/auth.log")"
|
||||
export SUPABASE_URL="$AUTH_URL"
|
||||
ok "issuer up, SUPABASE_URL=$SUPABASE_URL"
|
||||
|
||||
mint() { curl -sf "$AUTH_URL/token?sub=$1" | python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])'; }
|
||||
|
||||
|
||||
echo "==> starting continuum-backend on :$PORT"
|
||||
DISPATCH_ENABLED=false bun run src/index.ts > "$WORK/backend.log" 2>&1 &
|
||||
BACKEND_PID=$!
|
||||
for _ in $(seq 1 60); do
|
||||
curl -sf "$BASE/health" >/dev/null 2>&1 && break
|
||||
sleep 0.5
|
||||
done
|
||||
curl -sf "$BASE/health" >/dev/null 2>&1 || fail "backend did not become healthy: $(cat "$WORK/backend.log")"
|
||||
ok "backend healthy"
|
||||
|
||||
# --- 2. a farm with one claimed printer ---------------------------------
|
||||
|
||||
OWNER_ID="$(new_uuid)"; OWNER="$(mint "$OWNER_ID")"
|
||||
expect 201 POST /orgs "$OWNER" '{"name":"e2e alerts org"}' -- "create an org"
|
||||
ORG_A="$(json 'd["id"]')"
|
||||
expect 201 POST /farms "$OWNER" "{\"name\":\"e2e alerts farm\",\"orgId\":\"$ORG_A\"}" -- "create a farm"
|
||||
FARM_A="$(json 'd["id"]')"
|
||||
expect 201 POST /printers "$OWNER" \
|
||||
"{\"farmId\":\"$FARM_A\",\"name\":\"Fake P1\",\"vendor\":\"bambu\",\"gatewayPrinterId\":\"fake1\"}" -- "claim a printer"
|
||||
PRINTER_A="$(json 'd["id"]')"
|
||||
expect 200 POST "/farms/$FARM_A/edge-proxy-token" "$OWNER" -- "mint the gateway token"
|
||||
PROXY_TOKEN="$(json 'd["edgeProxyToken"]')"
|
||||
expect 200 POST /auth/edge-token - "{\"farmId\":\"$FARM_A\",\"edgeProxyToken\":\"$PROXY_TOKEN\"}" -- "trade it for an edge JWT"
|
||||
EDGE_JWT="$(json 'd["token"]')"
|
||||
[ -n "$EDGE_JWT" ] || fail "no edge token in the response"
|
||||
|
||||
# --- 3. telemetry -> alert ------------------------------------------------
|
||||
|
||||
echo "==> a print that fails raises exactly one alert"
|
||||
# The first frame is only an observation (nothing is known before it), so
|
||||
# IDLE,RUNNING,RUNNING,FAILED,FAILED has one transition worth raising.
|
||||
bun scripts/fake-gateway.ts "ws://localhost:$PORT/ws/edge/v1" "$EDGE_JWT" fake1 IDLE,RUNNING,RUNNING,FAILED,FAILED \
|
||||
|| fail "the fake gateway could not talk to the backend"
|
||||
N="$(psql_q "SELECT count(*) FROM printer_alerts WHERE farm_id='$FARM_A' AND kind='print_failed'")"
|
||||
[ "$N" = "1" ] && ok "one print_failed alert from a printing -> failed transition" || bad "expected 1 print_failed alert, found $N"
|
||||
TOTAL="$(psql_q "SELECT count(*) FROM printer_alerts WHERE farm_id='$FARM_A'")"
|
||||
[ "$TOTAL" = "1" ] && ok "and nothing else: staying failed and idle -> running are quiet" || bad "expected 1 alert in total, found $TOTAL"
|
||||
|
||||
echo "==> a reconnect does not re-announce what was already known"
|
||||
bun scripts/fake-gateway.ts "ws://localhost:$PORT/ws/edge/v1" "$EDGE_JWT" fake1 FAILED,FAILED || fail "reconnect failed"
|
||||
TOTAL="$(psql_q "SELECT count(*) FROM printer_alerts WHERE farm_id='$FARM_A'")"
|
||||
[ "$TOTAL" = "1" ] && ok "still one alert after reconnecting into the same state" || bad "reconnect raised alerts: $TOTAL"
|
||||
|
||||
expect 200 GET "/alerts?open=true&farmId=$FARM_A" "$OWNER" -- "the owner lists open alerts"
|
||||
body_has "the print failed"
|
||||
body_has "Fake P1"
|
||||
|
||||
# --- 4. AI detection -> alert --------------------------------------------
|
||||
|
||||
echo "==> an AI detection on the Valkey channel becomes a row (above the floor only)"
|
||||
publish() { valkey-cli -u "$VALKEY_URL" PUBLISH channel:print_alerts "$1" >/dev/null; }
|
||||
mk() { printf '{"printer_id":"%s","label":"spaghetti","confidence":%s,"captured_at":"2026-10-29T10:00:00Z","r2_object_key":"frames/e2e.jpg"}' "$PRINTER_A" "$1"; }
|
||||
publish "$(mk 0.95)"
|
||||
publish "$(mk 0.10)"
|
||||
publish 'this is not json'
|
||||
sleep 1
|
||||
AI="$(psql_q "SELECT count(*) FROM printer_alerts WHERE farm_id='$FARM_A' AND kind='ai_detection'")"
|
||||
[ "$AI" = "1" ] && ok "the 0.95 detection was recorded; the 0.10 one and the junk were not" || bad "expected 1 ai_detection alert, found $AI"
|
||||
CONF="$(psql_q "SELECT confidence FROM printer_alerts WHERE farm_id='$FARM_A' AND kind='ai_detection'")"
|
||||
[ "$CONF" = "0.95" ] && ok "confidence stored as published" || bad "confidence is '$CONF'"
|
||||
|
||||
echo
|
||||
if [ "$FAILURES" -eq 0 ]; then
|
||||
echo "PASS - telemetry transitions and AI detections both become alerts."
|
||||
else
|
||||
echo "FAIL - $FAILURES assertion(s) failed." >&2
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user