octoturge
64a43593b2
A start_print is acknowledged as soon as the gateway accepts it, and the download and transfer that follow can still fail. That late result was only logged, so the job sat in uploading until the stall reaper gave up twenty minutes later. The dispatcher now remembers which job each start command belonged to. A late failure for it fails the job with the gateway's own reason in metadata.lastDispatchError - but only while the job is still uploading, so a job the printer has since started is left to its telemetry. scripts/e2e-vendors.sh gains a phase where the proxy refuses the download and the job must fail at once with that reason.
150 lines
8.5 KiB
Bash
Executable File
150 lines
8.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# The non-Bambu vendors end to end, against FAKE printers: add each one through
|
|
# the API (so the provision frame, the real proxy's driver and its store are
|
|
# all exercised), then run a job through the real scheduler and watch it go
|
|
# queued -> uploading -> printing -> completed from the fake's own telemetry.
|
|
#
|
|
# What this proves is the plumbing around the drivers. It does NOT prove the
|
|
# drivers speak the real Klipper / OctoPrint / PrusaLink protocols: the fake
|
|
# (scripts/fake-http-printer.ts) is written from the same reading of the docs
|
|
# as the driver, so a shared misreading passes here and fails on hardware.
|
|
#
|
|
# Needs the dev stack (scripts/dev-env.sh up && schema) and the object store
|
|
# credentials in ../dev-secrets.sh (the job file goes through the real
|
|
# presigned URLs). No real printer is touched.
|
|
set -euo pipefail
|
|
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$REPO_ROOT"
|
|
[ -f "$REPO_ROOT/../dev-secrets.sh" ] && . "$REPO_ROOT/../dev-secrets.sh"
|
|
|
|
KINDS="${KINDS:-klipper octoprint prusa_link}"
|
|
PROXY_DIR="${PROXY_DIR:-$REPO_ROOT/../continuum-proxy}"
|
|
PORT="${PORT:-3062}"; AUTH_PORT="${AUTH_PORT:-3063}"; FAKE_PORT="${FAKE_PORT:-7141}"
|
|
BASE="http://localhost:$PORT"
|
|
DATABASE_URL="${DATABASE_URL:-postgres://continuum:continuum@localhost:5432/continuum}"
|
|
VALKEY_URL="${VALKEY_URL:-redis://localhost:6379}"
|
|
EDGE_JWT_SECRET="${EDGE_JWT_SECRET:-e2e-edge-secret}"
|
|
export DATABASE_URL VALKEY_URL EDGE_JWT_SECRET PORT R2_ACCESS_KEY_ID R2_SECRET_ACCESS_KEY R2_ENDPOINT
|
|
WORK="$(mktemp -d)"
|
|
BACKEND_PID=""; PROXY_PID=""; AUTH_PID=""; FAKE_PID=""; ORGS=""
|
|
fail() { echo "FAIL: $*" >&2; exit 1; }
|
|
ok() { echo " ok: $*"; }
|
|
psql_q() { psql "$DATABASE_URL" -tAq -c "$1"; }
|
|
new_uuid() { cat /proc/sys/kernel/random/uuid; }
|
|
json() { python3 -c "import json,sys; d=json.load(sys.stdin); print($1)"; }
|
|
|
|
stop() { for pid in "$@"; do [ -n "$pid" ] && kill "$pid" 2>/dev/null || true; done; sleep 0.5; for pid in "$@"; do [ -n "$pid" ] && kill -9 "$pid" 2>/dev/null || true; done; }
|
|
cleanup() {
|
|
local rc=$?
|
|
stop "$PROXY_PID" "$FAKE_PID" "$BACKEND_PID" "$AUTH_PID"
|
|
for org in $ORGS; do psql_q "DELETE FROM organisations WHERE id = '$org'" >/dev/null 2>&1 || true; done
|
|
if [ $rc -ne 0 ]; then
|
|
echo "--- backend log (tail) ---"; tail -25 "$WORK/backend.log" 2>/dev/null || true
|
|
echo "--- proxy log (tail) ---"; tail -30 "$WORK/proxy.log" 2>/dev/null || true
|
|
echo "--- fake printer log (tail) ---"; tail -15 "$WORK/fake.log" 2>/dev/null || true
|
|
fi
|
|
rm -rf "$WORK"; exit $rc
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
[ -n "${R2_ENDPOINT:-}" ] || fail "no object store credentials - ../dev-secrets.sh"
|
|
( cd "$PROXY_DIR" && cargo build --quiet 2>&1 | tail -3 ) || fail "proxy did not build"
|
|
|
|
bun scripts/dev-auth.ts --port "$AUTH_PORT" > "$WORK/auth.log" 2>&1 &
|
|
AUTH_PID=$!
|
|
AUTH_URL="http://localhost:$AUTH_PORT"
|
|
for _ in $(seq 1 40); do curl -sf "$AUTH_URL/auth/v1/.well-known/jwks.json" >/dev/null 2>&1 && break; sleep 0.25; done
|
|
export SUPABASE_URL="$AUTH_URL"
|
|
USER_ID="$(new_uuid)"
|
|
TOKEN="$(curl -sf "$AUTH_URL/token?sub=$USER_ID" | json 'd["access_token"]')"
|
|
auth=(-H "authorization: Bearer $TOKEN")
|
|
|
|
DISPATCH_ENABLED=true bun run src/index.ts > "$WORK/backend.log" 2>&1 &
|
|
BACKEND_PID=$!
|
|
for _ in $(seq 1 60); do curl -sf "$BASE/health" >/dev/null 2>&1 && break; sleep 0.5; done
|
|
curl -sf "$BASE/health" >/dev/null || fail "backend did not become healthy"
|
|
|
|
# run_kind <kind> <allow insecure downloads: true|false>
|
|
# With downloads refused the proxy accepts the start and then fails it, which
|
|
# is the late-failure path: the job must fail with the proxy's reason.
|
|
run_kind() {
|
|
local KIND="$1" ALLOW="$2"
|
|
echo "==> $KIND (insecure local downloads: $ALLOW)"
|
|
KEY="key-$(new_uuid | cut -c1-8)"
|
|
bun scripts/fake-http-printer.ts "$KIND" "$FAKE_PORT" "$KEY" > "$WORK/fake.log" 2>&1 &
|
|
FAKE_PID=$!
|
|
for _ in $(seq 1 40); do grep -q listening "$WORK/fake.log" && break; sleep 0.25; done
|
|
|
|
ORG_ID="$(curl -sf "${auth[@]}" -H 'content-type: application/json' -d "{\"name\":\"e2e vendors $KIND\"}" "$BASE/orgs" | json 'd["id"]')"
|
|
ORGS="$ORGS $ORG_ID"
|
|
FARM_ID="$(curl -sf "${auth[@]}" -H 'content-type: application/json' -d "{\"name\":\"farm $KIND\",\"orgId\":\"$ORG_ID\"}" "$BASE/farms" | json 'd["id"]')"
|
|
GW_TOKEN="$(curl -sf "${auth[@]}" -X POST "$BASE/farms/$FARM_ID/edge-proxy-token" | json 'd["edgeProxyToken"]')"
|
|
|
|
env -C "$PROXY_DIR" \
|
|
CONTINUUM_ALLOW_INSECURE_LOCAL_DOWNLOADS="$ALLOW" CONTINUUM_GATEWAY_ID=e2e-vendors-gw CONTINUUM_FARM_ID="$FARM_ID" CONTINUUM_GATEWAY_TOKEN="$GW_TOKEN" \
|
|
CONTINUUM_UPLINK_URL="ws://localhost:$PORT/ws/edge/v1" CONTINUUM_GO2RTC_BIN=/nonexistent-go2rtc CONTINUUM_GO2RTC_CONFIG="$WORK/go2rtc.yaml" \
|
|
CONTINUUM_PRINTERS_PATH="$WORK/no-such-printers.toml" CONTINUUM_STORE_PATH="$WORK/store-$KIND/continuum.db" RUST_LOG=info \
|
|
"$PROXY_DIR/target/debug/continuum-proxy" > "$WORK/proxy.log" 2>&1 &
|
|
PROXY_PID=$!
|
|
# This farm's own connect line: the log is shared across the loop.
|
|
for _ in $(seq 1 60); do grep -q "connected for farm $FARM_ID" "$WORK/backend.log" && break; sleep 0.5; done
|
|
grep -q "connected for farm $FARM_ID" "$WORK/backend.log" || fail "the proxy never connected"
|
|
|
|
# Klipper needs no credential; the others take an API key.
|
|
CRED=""; [ "$KIND" != klipper ] && CRED=",\"apiKey\":\"$KEY\""
|
|
CODE="$(curl -s -o "$WORK/add.json" -w '%{http_code}' "${auth[@]}" -H 'content-type: application/json' \
|
|
-d "{\"name\":\"Fake $KIND\",\"vendor\":\"$KIND\",\"host\":\"127.0.0.1:$FAKE_PORT\"$CRED}" "$BASE/farms/$FARM_ID/printers")"
|
|
[ "$CODE" = 201 ] || fail "adding the $KIND printer returned $CODE: $(head -c 300 "$WORK/add.json")"
|
|
PRN_UUID="$(json 'd["id"]' < "$WORK/add.json")"
|
|
ok "added through the API and connected by the proxy"
|
|
grep -q "$KEY" "$WORK/proxy.log" "$WORK/backend.log" && fail "the API key is in a log"
|
|
ok "the API key is in neither log"
|
|
|
|
if [ "$KIND" != klipper ]; then
|
|
CODE="$(curl -s -o /dev/null -w '%{http_code}' "${auth[@]}" -H 'content-type: application/json' \
|
|
-d "{\"name\":\"No key\",\"vendor\":\"$KIND\",\"host\":\"127.0.0.1:$FAKE_PORT\"}" "$BASE/farms/$FARM_ID/printers")"
|
|
[ "$CODE" = 422 ] && ok "a missing API key is refused (422)" || fail "a keyless add returned $CODE"
|
|
fi
|
|
|
|
for _ in $(seq 1 30); do [ "$(psql_q "SELECT status FROM printers WHERE id='$PRN_UUID'")" = idle ] && break; sleep 1; done
|
|
[ "$(psql_q "SELECT status FROM printers WHERE id='$PRN_UUID'")" = idle ] || fail "the cloud never saw the printer idle"
|
|
ok "the cloud sees it idle (telemetry from the driver's poller)"
|
|
|
|
J="$(curl -sf "${auth[@]}" -H 'content-type: application/json' \
|
|
-d "{\"farmId\":\"$FARM_ID\",\"fileName\":\"e2e-part.gcode\",\"contentType\":\"application/octet-stream\"}" "$BASE/jobs")"
|
|
JOB_ID="$(printf '%s' "$J" | json 'd["job"]["id"]')"
|
|
OBJECT_KEY="$(printf '%s' "$J" | json 'd["job"]["r2ObjectKey"]')"
|
|
UPLOAD_URL="$(printf '%s' "$J" | json 'd["uploadUrl"]')"
|
|
printf 'G28\nG1 X10 Y10 F3000\nM104 S0\n' > "$WORK/part.gcode"
|
|
CODE="$(curl -s -o /dev/null -w '%{http_code}' -X PUT -H 'Content-Type: application/octet-stream' --data-binary @"$WORK/part.gcode" "$UPLOAD_URL")"
|
|
[ "$CODE" = 200 ] || fail "job file upload returned $CODE"
|
|
|
|
SEEN=""; START=$SECONDS; STATUS=""
|
|
while [ $((SECONDS - START)) -lt 60 ]; do
|
|
STATUS="$(psql_q "SELECT status FROM print_jobs WHERE id='$JOB_ID'")"
|
|
case " $SEEN " in *" $STATUS "*) ;; *) SEEN="$SEEN $STATUS" ;; esac
|
|
[ "$STATUS" = completed ] || [ "$STATUS" = failed ] && break
|
|
sleep 1
|
|
done
|
|
echo " job status history:$SEEN"
|
|
if [ "$ALLOW" = true ]; then
|
|
[ "$STATUS" = completed ] || fail "the $KIND job ended as '$STATUS'"
|
|
ok "$KIND: queued, scheduled, uploaded, printed and completed (fake printer)"
|
|
else
|
|
[ "$STATUS" = failed ] || fail "the refused download left the job '$STATUS', not failed"
|
|
REASON="$(psql_q "SELECT metadata->>'lastDispatchError' FROM print_jobs WHERE id='$JOB_ID'")"
|
|
case "$REASON" in *"could not start it"*"non-HTTPS"*) ok "the job failed at once with the proxy's reason: $REASON" ;; *) fail "the job failed without the proxy's reason: '$REASON'" ;; esac
|
|
fi
|
|
|
|
bun -e "
|
|
import { s3 } from './src/s3';
|
|
import { DeleteObjectCommand } from '@aws-sdk/client-s3';
|
|
await s3.send(new DeleteObjectCommand({ Bucket: process.env.R2_BUCKET ?? 'continuum', Key: process.argv[1] }));
|
|
" "$OBJECT_KEY" >/dev/null 2>&1 || true
|
|
stop "$PROXY_PID" "$FAKE_PID"; PROXY_PID=""; FAKE_PID=""
|
|
}
|
|
for KIND in $KINDS; do run_kind "$KIND" true; done
|
|
run_kind klipper false
|
|
echo "PASS - $KINDS: added from the API, scheduled and completed against fake printers; a refused download fails the job with its reason."
|