Continuum/continuum-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Apply job outcomes a gateway kept while the cloud was away

Browse Source 
A job_event from a gateway moves a job that is still in flight (uploading,
printing, paused) to printing, completed or failed, and is acknowledged
whether or not it changed anything so the gateway stops sending it. A job
the cloud has cancelled, completed or put back in the queue is left alone,
and a gateway can only touch jobs of its own farm. If the database is
unreachable it is not acknowledged, so it is sent again.

scripts/e2e-vendors.sh gains a phase that takes the backend away for the
whole of a fake print and checks the job completes on reconnect from the
proxy's kept record.
This commit is contained in:
octoturge
2026-10-30 08:05:05 +00:00
parent 64a43593b2
commit b431ee4030
5 changed files with 148 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/e2e-vendors.sh
+33 -11
View File
@@ -44,6 +44,7 @@ cleanup() {
echo "--- proxy log (tail) ---"; tail -30 "$WORK/proxy.log" 2>/dev/null || true
echo "--- fake printer log (tail) ---"; tail -15 "$WORK/fake.log" 2>/dev/null || true
fi
[ -n "${KEEP_LOGS:-}" ] && cp "$WORK"/*.log "$KEEP_LOGS"/ 2>/dev/null || true
rm -rf "$WORK"; exit $rc
}
trap cleanup EXIT
@@ -60,19 +61,22 @@ USER_ID="$(new_uuid)"
TOKEN="$(curl -sf "$AUTH_URL/token?sub=$USER_ID" | json 'd["access_token"]')"
auth=(-H "authorization: Bearer $TOKEN")
DISPATCH_ENABLED=true bun run src/index.ts > "$WORK/backend.log" 2>&1 &
BACKEND_PID=$!
for _ in $(seq 1 60); do curl -sf "$BASE/health" >/dev/null 2>&1 && break; sleep 0.5; done
curl -sf "$BASE/health" >/dev/null || fail "backend did not become healthy"
start_backend() {
DISPATCH_ENABLED=true bun run src/index.ts >> "$WORK/backend.log" 2>&1 &
BACKEND_PID=$!
for _ in $(seq 1 60); do curl -sf "$BASE/health" >/dev/null 2>&1 && return 0; sleep 0.5; done
fail "backend did not become healthy"
}
start_backend
# run_kind <kind> <allow insecure downloads: true|false>
# run_kind <kind> <allow insecure downloads: true|false> [outage]
# With downloads refused the proxy accepts the start and then fails it, which
# is the late-failure path: the job must fail with the proxy's reason.
run_kind() {
local KIND="$1" ALLOW="$2"
echo "==> $KIND (insecure local downloads: $ALLOW)"
local KIND="$1" ALLOW="$2" OUTAGE="${3:-}"
echo "==> $KIND (insecure local downloads: $ALLOW${OUTAGE:+, cloud outage mid-job})"
KEY="key-$(new_uuid | cut -c1-8)"
bun scripts/fake-http-printer.ts "$KIND" "$FAKE_PORT" "$KEY" > "$WORK/fake.log" 2>&1 &
env ${OUTAGE:+RUN_S=14} bun scripts/fake-http-printer.ts "$KIND" "$FAKE_PORT" "$KEY" > "$WORK/fake.log" 2>&1 &
FAKE_PID=$!
for _ in $(seq 1 40); do grep -q listening "$WORK/fake.log" && break; sleep 0.25; done
@@ -120,6 +124,19 @@ run_kind() {
CODE="$(curl -s -o /dev/null -w '%{http_code}' -X PUT -H 'Content-Type: application/octet-stream' --data-binary @"$WORK/part.gcode" "$UPLOAD_URL")"
[ "$CODE" = 200 ] || fail "job file upload returned $CODE"
if [ -n "$OUTAGE" ]; then
# Take the cloud away the moment the job is dispatched - before any
# telemetry can have told it the print began - and bring it back only
# after the fake print is over. Telemetry alone would leave the job
# `uploading` (idle printer, never seen printing); only the gateway's
# own kept record can complete it.
for _ in $(seq 1 300); do [ "$(psql_q "SELECT status FROM print_jobs WHERE id='$JOB_ID'")" = uploading ] && break; sleep 0.1; done
[ "$(psql_q "SELECT status FROM print_jobs WHERE id='$JOB_ID'")" = uploading ] || fail "the job was not dispatched"
stop "$BACKEND_PID"; BACKEND_PID=""
echo " (cloud down; the fake print runs 14s without it)"
sleep 20
start_backend
fi
SEEN=""; START=$SECONDS; STATUS=""
while [ $((SECONDS - START)) -lt 60 ]; do
STATUS="$(psql_q "SELECT status FROM print_jobs WHERE id='$JOB_ID'")"
@@ -128,13 +145,17 @@ run_kind() {
sleep 1
done
echo " job status history:$SEEN"
if [ "$ALLOW" = true ]; then
if [ -n "$OUTAGE" ]; then
[ "$STATUS" = completed ] || fail "after the outage the $KIND job is '$STATUS', not completed"
grep -Eq "gateway reported (started|finished)" "$WORK/backend.log" || fail "it completed, but not from the gateway's kept record"
ok "the cloud was away for the whole print; the proxy's kept record completed the job on reconnect"
elif [ "$ALLOW" = true ]; then
[ "$STATUS" = completed ] || fail "the $KIND job ended as '$STATUS'"
ok "$KIND: queued, scheduled, uploaded, printed and completed (fake printer)"
else
[ "$STATUS" = failed ] || fail "the refused download left the job '$STATUS', not failed"
REASON="$(psql_q "SELECT metadata->>'lastDispatchError' FROM print_jobs WHERE id='$JOB_ID'")"
case "$REASON" in *"could not start it"*"non-HTTPS"*) ok "the job failed at once with the proxy's reason: $REASON" ;; *) fail "the job failed without the proxy's reason: '$REASON'" ;; esac
case "$REASON" in *"non-HTTPS"*) ok "the job failed at once with the proxy's reason: $REASON" ;; *) fail "the job failed without the proxy's reason: '$REASON'" ;; esac
fi
bun -e "
@@ -146,4 +167,5 @@ await s3.send(new DeleteObjectCommand({ Bucket: process.env.R2_BUCKET ?? 'contin
}
for KIND in $KINDS; do run_kind "$KIND" true; done
run_kind klipper false
echo "PASS - $KINDS: added from the API, scheduled and completed against fake printers; a refused download fails the job with its reason."
run_kind klipper true outage
echo "PASS - $KINDS: added from the API, scheduled and completed against fake printers; a refused download fails the job with its reason; a job finished while the cloud was away completes on reconnect."