Fail a job when the gateway reports it could not start it

A start_print is acknowledged as soon as the gateway accepts it, and the
download and transfer that follow can still fail. That late result was only
logged, so the job sat in uploading until the stall reaper gave up twenty
minutes later.

The dispatcher now remembers which job each start command belonged to. A
late failure for it fails the job with the gateway's own reason in
metadata.lastDispatchError - but only while the job is still uploading, so a
job the printer has since started is left to its telemetry.

scripts/e2e-vendors.sh gains a phase where the proxy refuses the download
and the job must fail at once with that reason.

src/dispatch/index.ts

@@ -1,9 +1,10 @@
-import { and, asc, eq, inArray, isNotNull, isNull, lt } from "drizzle-orm";
+import { and, asc, eq, inArray, isNotNull, isNull, lt, sql } from "drizzle-orm";
 import { db } from "../db";
 import { printers, printJobs } from "../db/schema";
 import { getDownloadUrl, headObject } from "../s3";
 import { isGatewayOnline, sendPrinterCommand } from "../routes/ws/edge";
 import { nextJobStatus } from "./job_status";
+import { rememberStart, takeStart } from "./late_failure";
 import { dispatchEnabled, dispatchSwitchWarning } from "./enabled";
 
 /**
@@ -217,6 +218,9 @@ export async function dispatchTick(): Promise<number> {
         ...(typeof job.metadata?.amsSlot === "number" ? { ams_slot: job.metadata.amsSlot } : {}),
       });
 
+      // Accepted or not, a late result for this command can name the job.
+      rememberStart(result.commandId, job.id);
+
       if (!result.ok) {
         // Requeued, not failed. The gateway refusing a start is usually
         // transient from the cloud's point of view - the printer was busier
@@ -249,6 +253,35 @@ export async function dispatchTick(): Promise<number> {
   return dispatched;
 }
 
+/**
+ * A gateway that had accepted a `start_print` reports that the download or
+ * transfer then failed. Fails the job with that reason - but only while it is
+ * still `uploading`: one that has since reached `printing` is being driven by
+ * the printer's own telemetry, and a late message must not overrule it.
+ *
+ * Failed rather than requeued, for the reason `reapStalledUploads` gives.
+ */
+export async function failJobFromGateway(commandId: string, error: string): Promise<void> {
+  const jobId = takeStart(commandId);
+  if (!jobId) return;
+
+  const failed = await db
+    .update(printJobs)
+    .set({
+      status: "failed",
+      completedAt: new Date(),
+      updatedAt: new Date(),
+      metadata: sql`coalesce(${printJobs.metadata}, '{}'::jsonb) || ${JSON.stringify({
+        lastDispatchError: `the gateway could not start it: ${error}`,
+        lastDispatchErrorAt: new Date().toISOString(),
+      })}::jsonb`,
+    })
+    .where(and(eq(printJobs.id, jobId), eq(printJobs.status, "uploading")))
+    .returning({ id: printJobs.id });
+
+  if (failed.length > 0) console.warn(`[dispatch] job ${jobId} failed: the gateway could not start it: ${error}`);
+}
+
 /**
  * Fails jobs that have been `uploading` too long.
  *
@@ -274,7 +307,7 @@ async function reapStalledUploads(): Promise<void> {
         lastDispatchError:
           `the gateway accepted the start but the printer never reported printing within ` +
           `${Math.round(UPLOADING_TIMEOUT_MS / 60_000)} minutes - check the gateway's log for ` +
-          `the download or transfer error, which has no path back to the cloud yet`,
+          `the download or transfer error (a failure it reports promptly fails the job sooner)`,
         lastDispatchErrorAt: new Date().toISOString(),
       },
     })