Fail a job when the gateway reports it could not start it

A start_print is acknowledged as soon as the gateway accepts it, and the
download and transfer that follow can still fail. That late result was only
logged, so the job sat in uploading until the stall reaper gave up twenty
minutes later.

The dispatcher now remembers which job each start command belonged to. A
late failure for it fails the job with the gateway's own reason in
metadata.lastDispatchError - but only while the job is still uploading, so a
job the printer has since started is left to its telemetry.

scripts/e2e-vendors.sh gains a phase where the proxy refuses the download
and the job must fail at once with that reason.

scripts/e2e-vendors.sh

@@ -65,8 +65,12 @@ BACKEND_PID=$!
 for _ in $(seq 1 60); do curl -sf "$BASE/health" >/dev/null 2>&1 && break; sleep 0.5; done
 curl -sf "$BASE/health" >/dev/null || fail "backend did not become healthy"
 
-for KIND in $KINDS; do
-  echo "==> $KIND"
+# run_kind <kind> <allow insecure downloads: true|false>
+# With downloads refused the proxy accepts the start and then fails it, which
+# is the late-failure path: the job must fail with the proxy's reason.
+run_kind() {
+  local KIND="$1" ALLOW="$2"
+  echo "==> $KIND (insecure local downloads: $ALLOW)"
   KEY="key-$(new_uuid | cut -c1-8)"
   bun scripts/fake-http-printer.ts "$KIND" "$FAKE_PORT" "$KEY" > "$WORK/fake.log" 2>&1 &
   FAKE_PID=$!
@@ -78,7 +82,7 @@ for KIND in $KINDS; do
   GW_TOKEN="$(curl -sf "${auth[@]}" -X POST "$BASE/farms/$FARM_ID/edge-proxy-token" | json 'd["edgeProxyToken"]')"
 
   env -C "$PROXY_DIR" \
-    CONTINUUM_ALLOW_INSECURE_LOCAL_DOWNLOADS=true CONTINUUM_GATEWAY_ID=e2e-vendors-gw CONTINUUM_FARM_ID="$FARM_ID" CONTINUUM_GATEWAY_TOKEN="$GW_TOKEN" \
+    CONTINUUM_ALLOW_INSECURE_LOCAL_DOWNLOADS="$ALLOW" CONTINUUM_GATEWAY_ID=e2e-vendors-gw CONTINUUM_FARM_ID="$FARM_ID" CONTINUUM_GATEWAY_TOKEN="$GW_TOKEN" \
     CONTINUUM_UPLINK_URL="ws://localhost:$PORT/ws/edge/v1" CONTINUUM_GO2RTC_BIN=/nonexistent-go2rtc CONTINUUM_GO2RTC_CONFIG="$WORK/go2rtc.yaml" \
     CONTINUUM_PRINTERS_PATH="$WORK/no-such-printers.toml" CONTINUUM_STORE_PATH="$WORK/store-$KIND/continuum.db" RUST_LOG=info \
     "$PROXY_DIR/target/debug/continuum-proxy" > "$WORK/proxy.log" 2>&1 &
@@ -117,15 +121,21 @@ for KIND in $KINDS; do
   [ "$CODE" = 200 ] || fail "job file upload returned $CODE"
 
   SEEN=""; START=$SECONDS; STATUS=""
-  while [ $((SECONDS - START)) -lt 90 ]; do
+  while [ $((SECONDS - START)) -lt 60 ]; do
     STATUS="$(psql_q "SELECT status FROM print_jobs WHERE id='$JOB_ID'")"
     case " $SEEN " in *" $STATUS "*) ;; *) SEEN="$SEEN $STATUS" ;; esac
     [ "$STATUS" = completed ] || [ "$STATUS" = failed ] && break
     sleep 1
   done
   echo "  job status history:$SEEN"
-  [ "$STATUS" = completed ] || fail "the $KIND job ended as '$STATUS'"
-  ok "$KIND: queued, scheduled, uploaded, printed and completed (fake printer)"
+  if [ "$ALLOW" = true ]; then
+    [ "$STATUS" = completed ] || fail "the $KIND job ended as '$STATUS'"
+    ok "$KIND: queued, scheduled, uploaded, printed and completed (fake printer)"
+  else
+    [ "$STATUS" = failed ] || fail "the refused download left the job '$STATUS', not failed"
+    REASON="$(psql_q "SELECT metadata->>'lastDispatchError' FROM print_jobs WHERE id='$JOB_ID'")"
+    case "$REASON" in *"could not start it"*"non-HTTPS"*) ok "the job failed at once with the proxy's reason: $REASON" ;; *) fail "the job failed without the proxy's reason: '$REASON'" ;; esac
+  fi
 
   bun -e "
 import { s3 } from './src/s3';
@@ -133,5 +143,7 @@ import { DeleteObjectCommand } from '@aws-sdk/client-s3';
 await s3.send(new DeleteObjectCommand({ Bucket: process.env.R2_BUCKET ?? 'continuum', Key: process.argv[1] }));
 " "$OBJECT_KEY" >/dev/null 2>&1 || true
   stop "$PROXY_PID" "$FAKE_PID"; PROXY_PID=""; FAKE_PID=""
-done
-echo "PASS - $KINDS: added from the API, scheduled and completed against fake printers."
+}
+for KIND in $KINDS; do run_kind "$KIND" true; done
+run_kind klipper false
+echo "PASS - $KINDS: added from the API, scheduled and completed against fake printers; a refused download fails the job with its reason."