octoturge
b431ee4030
A job_event from a gateway moves a job that is still in flight (uploading, printing, paused) to printing, completed or failed, and is acknowledged whether or not it changed anything so the gateway stops sending it. A job the cloud has cancelled, completed or put back in the queue is left alone, and a gateway can only touch jobs of its own farm. If the database is unreachable it is not acknowledged, so it is sent again. scripts/e2e-vendors.sh gains a phase that takes the backend away for the whole of a fake print and checks the job completes on reconnect from the proxy's kept record.
172 lines
9.8 KiB
Bash
Executable File
172 lines
9.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# The non-Bambu vendors end to end, against FAKE printers: add each one through
|
|
# the API (so the provision frame, the real proxy's driver and its store are
|
|
# all exercised), then run a job through the real scheduler and watch it go
|
|
# queued -> uploading -> printing -> completed from the fake's own telemetry.
|
|
#
|
|
# What this proves is the plumbing around the drivers. It does NOT prove the
|
|
# drivers speak the real Klipper / OctoPrint / PrusaLink protocols: the fake
|
|
# (scripts/fake-http-printer.ts) is written from the same reading of the docs
|
|
# as the driver, so a shared misreading passes here and fails on hardware.
|
|
#
|
|
# Needs the dev stack (scripts/dev-env.sh up && schema) and the object store
|
|
# credentials in ../dev-secrets.sh (the job file goes through the real
|
|
# presigned URLs). No real printer is touched.
|
|
set -euo pipefail
|
|
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$REPO_ROOT"
|
|
[ -f "$REPO_ROOT/../dev-secrets.sh" ] && . "$REPO_ROOT/../dev-secrets.sh"
|
|
|
|
KINDS="${KINDS:-klipper octoprint prusa_link}"
|
|
PROXY_DIR="${PROXY_DIR:-$REPO_ROOT/../continuum-proxy}"
|
|
PORT="${PORT:-3062}"; AUTH_PORT="${AUTH_PORT:-3063}"; FAKE_PORT="${FAKE_PORT:-7141}"
|
|
BASE="http://localhost:$PORT"
|
|
DATABASE_URL="${DATABASE_URL:-postgres://continuum:continuum@localhost:5432/continuum}"
|
|
VALKEY_URL="${VALKEY_URL:-redis://localhost:6379}"
|
|
EDGE_JWT_SECRET="${EDGE_JWT_SECRET:-e2e-edge-secret}"
|
|
export DATABASE_URL VALKEY_URL EDGE_JWT_SECRET PORT R2_ACCESS_KEY_ID R2_SECRET_ACCESS_KEY R2_ENDPOINT
|
|
WORK="$(mktemp -d)"
|
|
BACKEND_PID=""; PROXY_PID=""; AUTH_PID=""; FAKE_PID=""; ORGS=""
|
|
fail() { echo "FAIL: $*" >&2; exit 1; }
|
|
ok() { echo " ok: $*"; }
|
|
psql_q() { psql "$DATABASE_URL" -tAq -c "$1"; }
|
|
new_uuid() { cat /proc/sys/kernel/random/uuid; }
|
|
json() { python3 -c "import json,sys; d=json.load(sys.stdin); print($1)"; }
|
|
|
|
stop() { for pid in "$@"; do [ -n "$pid" ] && kill "$pid" 2>/dev/null || true; done; sleep 0.5; for pid in "$@"; do [ -n "$pid" ] && kill -9 "$pid" 2>/dev/null || true; done; }
|
|
cleanup() {
|
|
local rc=$?
|
|
stop "$PROXY_PID" "$FAKE_PID" "$BACKEND_PID" "$AUTH_PID"
|
|
for org in $ORGS; do psql_q "DELETE FROM organisations WHERE id = '$org'" >/dev/null 2>&1 || true; done
|
|
if [ $rc -ne 0 ]; then
|
|
echo "--- backend log (tail) ---"; tail -25 "$WORK/backend.log" 2>/dev/null || true
|
|
echo "--- proxy log (tail) ---"; tail -30 "$WORK/proxy.log" 2>/dev/null || true
|
|
echo "--- fake printer log (tail) ---"; tail -15 "$WORK/fake.log" 2>/dev/null || true
|
|
fi
|
|
[ -n "${KEEP_LOGS:-}" ] && cp "$WORK"/*.log "$KEEP_LOGS"/ 2>/dev/null || true
|
|
rm -rf "$WORK"; exit $rc
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
[ -n "${R2_ENDPOINT:-}" ] || fail "no object store credentials - ../dev-secrets.sh"
|
|
( cd "$PROXY_DIR" && cargo build --quiet 2>&1 | tail -3 ) || fail "proxy did not build"
|
|
|
|
bun scripts/dev-auth.ts --port "$AUTH_PORT" > "$WORK/auth.log" 2>&1 &
|
|
AUTH_PID=$!
|
|
AUTH_URL="http://localhost:$AUTH_PORT"
|
|
for _ in $(seq 1 40); do curl -sf "$AUTH_URL/auth/v1/.well-known/jwks.json" >/dev/null 2>&1 && break; sleep 0.25; done
|
|
export SUPABASE_URL="$AUTH_URL"
|
|
USER_ID="$(new_uuid)"
|
|
TOKEN="$(curl -sf "$AUTH_URL/token?sub=$USER_ID" | json 'd["access_token"]')"
|
|
auth=(-H "authorization: Bearer $TOKEN")
|
|
|
|
start_backend() {
|
|
DISPATCH_ENABLED=true bun run src/index.ts >> "$WORK/backend.log" 2>&1 &
|
|
BACKEND_PID=$!
|
|
for _ in $(seq 1 60); do curl -sf "$BASE/health" >/dev/null 2>&1 && return 0; sleep 0.5; done
|
|
fail "backend did not become healthy"
|
|
}
|
|
start_backend
|
|
|
|
# run_kind <kind> <allow insecure downloads: true|false> [outage]
|
|
# With downloads refused the proxy accepts the start and then fails it, which
|
|
# is the late-failure path: the job must fail with the proxy's reason.
|
|
run_kind() {
|
|
local KIND="$1" ALLOW="$2" OUTAGE="${3:-}"
|
|
echo "==> $KIND (insecure local downloads: $ALLOW${OUTAGE:+, cloud outage mid-job})"
|
|
KEY="key-$(new_uuid | cut -c1-8)"
|
|
env ${OUTAGE:+RUN_S=14} bun scripts/fake-http-printer.ts "$KIND" "$FAKE_PORT" "$KEY" > "$WORK/fake.log" 2>&1 &
|
|
FAKE_PID=$!
|
|
for _ in $(seq 1 40); do grep -q listening "$WORK/fake.log" && break; sleep 0.25; done
|
|
|
|
ORG_ID="$(curl -sf "${auth[@]}" -H 'content-type: application/json' -d "{\"name\":\"e2e vendors $KIND\"}" "$BASE/orgs" | json 'd["id"]')"
|
|
ORGS="$ORGS $ORG_ID"
|
|
FARM_ID="$(curl -sf "${auth[@]}" -H 'content-type: application/json' -d "{\"name\":\"farm $KIND\",\"orgId\":\"$ORG_ID\"}" "$BASE/farms" | json 'd["id"]')"
|
|
GW_TOKEN="$(curl -sf "${auth[@]}" -X POST "$BASE/farms/$FARM_ID/edge-proxy-token" | json 'd["edgeProxyToken"]')"
|
|
|
|
env -C "$PROXY_DIR" \
|
|
CONTINUUM_ALLOW_INSECURE_LOCAL_DOWNLOADS="$ALLOW" CONTINUUM_GATEWAY_ID=e2e-vendors-gw CONTINUUM_FARM_ID="$FARM_ID" CONTINUUM_GATEWAY_TOKEN="$GW_TOKEN" \
|
|
CONTINUUM_UPLINK_URL="ws://localhost:$PORT/ws/edge/v1" CONTINUUM_GO2RTC_BIN=/nonexistent-go2rtc CONTINUUM_GO2RTC_CONFIG="$WORK/go2rtc.yaml" \
|
|
CONTINUUM_PRINTERS_PATH="$WORK/no-such-printers.toml" CONTINUUM_STORE_PATH="$WORK/store-$KIND/continuum.db" RUST_LOG=info \
|
|
"$PROXY_DIR/target/debug/continuum-proxy" > "$WORK/proxy.log" 2>&1 &
|
|
PROXY_PID=$!
|
|
# This farm's own connect line: the log is shared across the loop.
|
|
for _ in $(seq 1 60); do grep -q "connected for farm $FARM_ID" "$WORK/backend.log" && break; sleep 0.5; done
|
|
grep -q "connected for farm $FARM_ID" "$WORK/backend.log" || fail "the proxy never connected"
|
|
|
|
# Klipper needs no credential; the others take an API key.
|
|
CRED=""; [ "$KIND" != klipper ] && CRED=",\"apiKey\":\"$KEY\""
|
|
CODE="$(curl -s -o "$WORK/add.json" -w '%{http_code}' "${auth[@]}" -H 'content-type: application/json' \
|
|
-d "{\"name\":\"Fake $KIND\",\"vendor\":\"$KIND\",\"host\":\"127.0.0.1:$FAKE_PORT\"$CRED}" "$BASE/farms/$FARM_ID/printers")"
|
|
[ "$CODE" = 201 ] || fail "adding the $KIND printer returned $CODE: $(head -c 300 "$WORK/add.json")"
|
|
PRN_UUID="$(json 'd["id"]' < "$WORK/add.json")"
|
|
ok "added through the API and connected by the proxy"
|
|
grep -q "$KEY" "$WORK/proxy.log" "$WORK/backend.log" && fail "the API key is in a log"
|
|
ok "the API key is in neither log"
|
|
|
|
if [ "$KIND" != klipper ]; then
|
|
CODE="$(curl -s -o /dev/null -w '%{http_code}' "${auth[@]}" -H 'content-type: application/json' \
|
|
-d "{\"name\":\"No key\",\"vendor\":\"$KIND\",\"host\":\"127.0.0.1:$FAKE_PORT\"}" "$BASE/farms/$FARM_ID/printers")"
|
|
[ "$CODE" = 422 ] && ok "a missing API key is refused (422)" || fail "a keyless add returned $CODE"
|
|
fi
|
|
|
|
for _ in $(seq 1 30); do [ "$(psql_q "SELECT status FROM printers WHERE id='$PRN_UUID'")" = idle ] && break; sleep 1; done
|
|
[ "$(psql_q "SELECT status FROM printers WHERE id='$PRN_UUID'")" = idle ] || fail "the cloud never saw the printer idle"
|
|
ok "the cloud sees it idle (telemetry from the driver's poller)"
|
|
|
|
J="$(curl -sf "${auth[@]}" -H 'content-type: application/json' \
|
|
-d "{\"farmId\":\"$FARM_ID\",\"fileName\":\"e2e-part.gcode\",\"contentType\":\"application/octet-stream\"}" "$BASE/jobs")"
|
|
JOB_ID="$(printf '%s' "$J" | json 'd["job"]["id"]')"
|
|
OBJECT_KEY="$(printf '%s' "$J" | json 'd["job"]["r2ObjectKey"]')"
|
|
UPLOAD_URL="$(printf '%s' "$J" | json 'd["uploadUrl"]')"
|
|
printf 'G28\nG1 X10 Y10 F3000\nM104 S0\n' > "$WORK/part.gcode"
|
|
CODE="$(curl -s -o /dev/null -w '%{http_code}' -X PUT -H 'Content-Type: application/octet-stream' --data-binary @"$WORK/part.gcode" "$UPLOAD_URL")"
|
|
[ "$CODE" = 200 ] || fail "job file upload returned $CODE"
|
|
|
|
if [ -n "$OUTAGE" ]; then
|
|
# Take the cloud away the moment the job is dispatched - before any
|
|
# telemetry can have told it the print began - and bring it back only
|
|
# after the fake print is over. Telemetry alone would leave the job
|
|
# `uploading` (idle printer, never seen printing); only the gateway's
|
|
# own kept record can complete it.
|
|
for _ in $(seq 1 300); do [ "$(psql_q "SELECT status FROM print_jobs WHERE id='$JOB_ID'")" = uploading ] && break; sleep 0.1; done
|
|
[ "$(psql_q "SELECT status FROM print_jobs WHERE id='$JOB_ID'")" = uploading ] || fail "the job was not dispatched"
|
|
stop "$BACKEND_PID"; BACKEND_PID=""
|
|
echo " (cloud down; the fake print runs 14s without it)"
|
|
sleep 20
|
|
start_backend
|
|
fi
|
|
SEEN=""; START=$SECONDS; STATUS=""
|
|
while [ $((SECONDS - START)) -lt 60 ]; do
|
|
STATUS="$(psql_q "SELECT status FROM print_jobs WHERE id='$JOB_ID'")"
|
|
case " $SEEN " in *" $STATUS "*) ;; *) SEEN="$SEEN $STATUS" ;; esac
|
|
[ "$STATUS" = completed ] || [ "$STATUS" = failed ] && break
|
|
sleep 1
|
|
done
|
|
echo " job status history:$SEEN"
|
|
if [ -n "$OUTAGE" ]; then
|
|
[ "$STATUS" = completed ] || fail "after the outage the $KIND job is '$STATUS', not completed"
|
|
grep -Eq "gateway reported (started|finished)" "$WORK/backend.log" || fail "it completed, but not from the gateway's kept record"
|
|
ok "the cloud was away for the whole print; the proxy's kept record completed the job on reconnect"
|
|
elif [ "$ALLOW" = true ]; then
|
|
[ "$STATUS" = completed ] || fail "the $KIND job ended as '$STATUS'"
|
|
ok "$KIND: queued, scheduled, uploaded, printed and completed (fake printer)"
|
|
else
|
|
[ "$STATUS" = failed ] || fail "the refused download left the job '$STATUS', not failed"
|
|
REASON="$(psql_q "SELECT metadata->>'lastDispatchError' FROM print_jobs WHERE id='$JOB_ID'")"
|
|
case "$REASON" in *"non-HTTPS"*) ok "the job failed at once with the proxy's reason: $REASON" ;; *) fail "the job failed without the proxy's reason: '$REASON'" ;; esac
|
|
fi
|
|
|
|
bun -e "
|
|
import { s3 } from './src/s3';
|
|
import { DeleteObjectCommand } from '@aws-sdk/client-s3';
|
|
await s3.send(new DeleteObjectCommand({ Bucket: process.env.R2_BUCKET ?? 'continuum', Key: process.argv[1] }));
|
|
" "$OBJECT_KEY" >/dev/null 2>&1 || true
|
|
stop "$PROXY_PID" "$FAKE_PID"; PROXY_PID=""; FAKE_PID=""
|
|
}
|
|
for KIND in $KINDS; do run_kind "$KIND" true; done
|
|
run_kind klipper false
|
|
run_kind klipper true outage
|
|
echo "PASS - $KINDS: added from the API, scheduled and completed against fake printers; a refused download fails the job with its reason; a job finished while the cloud was away completes on reconnect."
|