Apply job outcomes a gateway kept while the cloud was away

A job_event from a gateway moves a job that is still in flight (uploading,
printing, paused) to printing, completed or failed, and is acknowledged
whether or not it changed anything so the gateway stops sending it. A job
the cloud has cancelled, completed or put back in the queue is left alone,
and a gateway can only touch jobs of its own farm. If the database is
unreachable it is not acknowledged, so it is sent again.

scripts/e2e-vendors.sh gains a phase that takes the backend away for the
whole of a fake print and checks the job completes on reconnect from the
proxy's kept record.

src/dispatch/index.ts

@@ -3,7 +3,7 @@ import { db } from "../db";
 import { printers, printJobs } from "../db/schema";
 import { getDownloadUrl, headObject } from "../s3";
 import { isGatewayOnline, sendPrinterCommand } from "../routes/ws/edge";
-import { nextJobStatus } from "./job_status";
+import { jobStatusFromGatewayEvent, nextJobStatus } from "./job_status";
 import { rememberStart, takeStart } from "./late_failure";
 import { dispatchEnabled, dispatchSwitchWarning } from "./enabled";
 
@@ -282,6 +282,55 @@ export async function failJobFromGateway(commandId: string, error: string): Prom
   if (failed.length > 0) console.warn(`[dispatch] job ${jobId} failed: the gateway could not start it: ${error}`);
 }
 
+/**
+ * Applies an outcome a gateway kept for us. Throws only when the database
+ * could not be reached - the one case where the gateway must NOT be told it
+ * was handled, so that it sends it again. A job that is unknown, belongs to
+ * another farm, or is no longer in flight is not an error: there is nothing
+ * to change, and the gateway should stop sending it.
+ */
+export async function applyGatewayJobEvent(
+  farmId: string,
+  event: { job_id: string; event: "started" | "finished" | "failed"; detail?: string | null; at_ms: number },
+): Promise<"applied" | "ignored"> {
+  // The farm in the `where` is the authorisation: a gateway can only move
+  // jobs of the farm it is connected for.
+  const job = await db.query.printJobs.findFirst({
+    columns: { id: true, status: true },
+    where: and(eq(printJobs.id, event.job_id), eq(printJobs.farmId, farmId)),
+  });
+  if (!job) return "ignored";
+
+  const next = jobStatusFromGatewayEvent(job.status, event.event);
+  if (!next) return "ignored";
+
+  const at = new Date(event.at_ms);
+  const terminal = next === "completed" || next === "failed";
+  const updated = await db
+    .update(printJobs)
+    .set({
+      status: next,
+      updatedAt: new Date(),
+      ...(next === "completed" ? { progressPct: 100 } : {}),
+      ...(terminal ? { completedAt: at } : {}),
+      ...(next === "failed"
+        ? {
+            metadata: sql`coalesce(${printJobs.metadata}, '{}'::jsonb) || ${JSON.stringify({
+              lastDispatchError: event.detail ?? "the gateway reported the job failed",
+              lastDispatchErrorAt: at.toISOString(),
+            })}::jsonb`,
+          }
+        : {}),
+    })
+    // Re-checked in the write: the status may have moved since the read.
+    .where(and(eq(printJobs.id, job.id), eq(printJobs.status, job.status)))
+    .returning({ id: printJobs.id });
+
+  if (updated.length === 0) return "ignored";
+  console.log(`[dispatch] job ${job.id} ${job.status} -> ${next} (gateway reported ${event.event})`);
+  return "applied";
+}
+
 /**
  * Fails jobs that have been `uploading` too long.
  *