chore(komodo): park-lab posts review crops as booth-lab; collector lists its token

Own pseudonymous id and own secret (wash_review_token_booth_lab), never a real
booth's, so bench crops stay separable per booth on the collector and can be
labelled unusable. Fleet page row updated.

Claude-Session: https://claude.ai/code/session_01FWncR69HgGPuei1dLrW3cU

apps/collector/src/app.test.ts

@@ -151,3 +151,46 @@ describe("config", () => {
     expect(() => parseBoothTokens("nocolon")).toThrow(/bad pair/);
   });
 });
+
+describe("schema migration", () => {
+  it("opens a database created before the `kind` column and adds the missing columns, so ingest and stats work", async () => {
+    // art-docker-station, 2026-09-16: the volume's DB predated `kind`; CREATE TABLE IF NOT
+    // EXISTS left it alone, and /health, every ingest and the trainer's readiness failed
+    // with "no such column: kind". Replay: a file with the ORIGINAL column set.
+    const { default: Database } = await import("better-sqlite3");
+    const file = path.join(dir, "old.sqlite");
+    const old = new Database(file);
+    old.exec(`CREATE TABLE items (
+      id TEXT PRIMARY KEY, booth TEXT NOT NULL, order_ref TEXT NOT NULL, at TEXT NOT NULL,
+      service TEXT NOT NULL, vision_class TEXT NOT NULL, vision_confidence REAL NOT NULL,
+      image_width INTEGER NOT NULL, image_height INTEGER NOT NULL, plate_blurred INTEGER NOT NULL,
+      image_path TEXT NOT NULL, received_at TEXT NOT NULL, review_label TEXT, reviewed_at TEXT, reviewer TEXT)`);
+    old.prepare(
+      "INSERT INTO items (id, booth, order_ref, at, service, vision_class, vision_confidence, image_width, image_height, plate_blurred, image_path, received_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
+    ).run("legacy-1", "booth-7", "o-0", "2026-09-01T00:00:00.000Z", "Standard", "suv", 0.8, 100, 100, 1, "crops/legacy-1.jpg", "2026-09-01T00:00:00.000Z");
+    old.close();
+
+    const legacy = await buildCollector({ host: "127.0.0.1", port: 0, dataDir: dir, boothTokens: TOKENS, reviewer: REVIEWER, trainerUrl: null }, { dbFile: file });
+    await legacy.ready();
+    try {
+      const health = await legacy.inject({ method: "GET", url: "/health" });
+      expect(health.statusCode).toBe(200);
+      expect(health.json()).toMatchObject({ ok: true, booths: 1, pending: 1 });
+
+      const { body, type } = multipart({ meta: JSON.stringify(meta({ item: "item-new" })) }, JPEG);
+      const r = await legacy.inject({ method: "POST", url: "/ingest", headers: { authorization: `Bearer ${TOKENS.get("booth-7")!}`, "content-type": type }, payload: body });
+      expect(r.statusCode).toBe(201);
+
+      const stats = await legacy.inject({ method: "GET", url: "/api/stats", headers: { authorization: basic } });
+      expect(stats.statusCode).toBe(200);
+
+      // The legacy row reads back with the defaults the new columns carry.
+      const cols = new Database(file, { readonly: true }).prepare("PRAGMA table_info(items)").all() as { name: string }[];
+      expect(cols.map((c) => c.name)).toEqual(expect.arrayContaining(["kind", "operator_ref", "operator_classes", "vision_category_id", "downgraded"]));
+      const legacyRow = new Database(file, { readonly: true }).prepare("SELECT kind, operator_classes, downgraded FROM items WHERE id = 'legacy-1'").get() as Record<string, unknown>;
+      expect(legacyRow).toEqual({ kind: "wash", operator_classes: "[]", downgraded: 0 });
+    } finally {
+      await legacy.close();
+    }
+  });
+});

apps/collector/src/app.ts

@@ -253,7 +253,18 @@ export async function buildCollector(cfg: CollectorConfig, opts: { dbFile?: stri
     try {
       const get = async (p: string) => {
         const r = await fetch(trainer + p, { signal: AbortSignal.timeout(15_000) });
-        if (!r.ok) throw new Error(`${p} → HTTP ${r.status}`);
+        if (!r.ok) {
+          // Surface the trainer's own error text (its handlers answer 500 JSON), so the
+          // reviewer reads "no such column: kind", not just a status code.
+          const detail = await r.text().then((t) => {
+            try {
+              return String((JSON.parse(t) as { error?: unknown }).error ?? t);
+            } catch {
+              return t;
+            }
+          }, () => "");
+          throw new Error(`${p} → HTTP ${r.status}${detail ? `: ${detail.slice(0, 300)}` : ""}`);
+        }
         return r.json() as Promise<Record<string, unknown>>;
       };
       const [health, readiness, versions, jobs] = await Promise.all([get("/health"), get("/readiness"), get("/versions"), get("/jobs")]);

apps/collector/src/db.ts

@@ -71,6 +71,39 @@ export class CollectorDb {
       CREATE INDEX IF NOT EXISTS items_pending ON items (reviewed_at, received_at);
       CREATE INDEX IF NOT EXISTS items_booth ON items (booth, received_at);
     `);
+    this.#migrate();
+  }
+
+  /** Columns added after the first deploy, with the DDL that adds them to an EXISTING
+   *  table. `CREATE TABLE IF NOT EXISTS` above only shapes a NEW database; a volume that
+   *  was created by an earlier build keeps its old columns, and every query naming a new
+   *  one then fails ("no such column: kind" — art-docker-station, 2026-09-16: the
+   *  collector's /health, every ingest, and the trainer's readiness all broke on a DB
+   *  from before `kind`). Each entry must be addable to a populated table, i.e. nullable
+   *  or carrying a DEFAULT. Append here whenever a column joins the CREATE above. */
+  static readonly #ADDED_COLUMNS: ReadonlyArray<readonly [name: string, ddl: string]> = [
+    ["kind", "TEXT NOT NULL DEFAULT 'wash'"],
+    ["operator_ref", "TEXT NOT NULL DEFAULT ''"],
+    ["operator_category_id", "TEXT NOT NULL DEFAULT ''"],
+    ["operator_category_name", "TEXT NOT NULL DEFAULT ''"],
+    ["operator_classes", "TEXT NOT NULL DEFAULT '[]'"],
+    ["vision_category_id", "TEXT"],
+    ["downgraded", "INTEGER NOT NULL DEFAULT 0"],
+  ];
+
+  /** Bring an existing `items` table up to the current column set (idempotent). */
+  #migrate(): void {
+    const present = new Set(
+      (this.#db.prepare("PRAGMA table_info(items)").all() as { name: string }[]).map((c) => c.name),
+    );
+    for (const [name, ddl] of CollectorDb.#ADDED_COLUMNS) {
+      if (!present.has(name)) this.#db.exec(`ALTER TABLE items ADD COLUMN ${name} ${ddl}`);
+    }
+  }
+
+  /** The current column names of `items` (for tests and diagnostics). */
+  columns(): string[] {
+    return (this.#db.prepare("PRAGMA table_info(items)").all() as { name: string }[]).map((c) => c.name);
   }
 
   close(): void {

apps/trainer/tests/test_server.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import json
+import sqlite3
 import threading
 import urllib.error
 import urllib.request
@@ -109,3 +110,38 @@ def test_train_job_then_versions_and_report(api) -> None:  # type: ignore[no-unt
     code, job2 = call("POST", "/jobs", {"kind": "evaluate", "version": "vapi"})
     wait_idle(Handler.jobs)
     assert call("GET", f"/jobs/{job2['id']}")[1]["status"] == "done"
+
+
+def test_a_handler_error_is_a_500_json_reply_not_a_dropped_connection(tmp_path: Path) -> None:
+    """A collector DB from before the `kind` column (art-docker-station, 2026-09-16): readiness
+    raised sqlite3.OperationalError, the stdlib server printed the traceback and closed the
+    socket, and the collector could only say "trainer not reachable: fetch failed". The
+    handler must answer 500 JSON naming the error instead."""
+    old = tmp_path / "old"
+    old.mkdir()
+    con = sqlite3.connect(old / "collector.sqlite")
+    con.executescript(
+        "CREATE TABLE items (id TEXT PRIMARY KEY, booth TEXT NOT NULL, order_ref TEXT NOT NULL,"
+        " at TEXT NOT NULL, service TEXT NOT NULL, vision_class TEXT NOT NULL,"
+        " vision_confidence REAL NOT NULL, image_width INTEGER NOT NULL, image_height INTEGER NOT NULL,"
+        " plate_blurred INTEGER NOT NULL, image_path TEXT NOT NULL, received_at TEXT NOT NULL,"
+        " review_label TEXT, reviewed_at TEXT, reviewer TEXT)"
+    )
+    con.commit()
+    con.close()
+    Handler.jobs = Jobs(old, tmp_path / "out", "https://example.invalid/pkg", "tok")
+    httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+    threading.Thread(target=httpd.serve_forever, daemon=True).start()
+    try:
+        req = urllib.request.Request(f"http://127.0.0.1:{httpd.server_address[1]}/readiness")
+        with pytest.raises(urllib.error.HTTPError) as ei:
+            urllib.request.urlopen(req, timeout=10)
+        assert ei.value.code == 500
+        body = json.loads(ei.value.read())
+        assert "no such column: kind" in body["error"]
+        # /health does not touch the DB and still answers.
+        with urllib.request.urlopen(f"http://127.0.0.1:{httpd.server_address[1]}/health", timeout=10) as r:
+            assert r.status == 200
+    finally:
+        httpd.shutdown()
+        httpd.server_close()

apps/trainer/trainer/server.py

@@ -309,6 +309,26 @@ class Handler(BaseHTTPRequestHandler):
         self.wfile.write(raw)
 
     def do_GET(self) -> None:  # noqa: N802
+        self._guarded(self._get)
+
+    def do_POST(self) -> None:  # noqa: N802
+        self._guarded(self._post)
+
+    def _guarded(self, handler: Any) -> None:
+        """Run a handler; an unexpected exception becomes a 500 JSON reply instead of a
+        dropped connection (the stdlib server would print the traceback and close the
+        socket, which the collector could only report as "fetch failed" — 2026-09-16,
+        a collector DB from before the `kind` column)."""
+        try:
+            handler()
+        except Exception as exc:  # noqa: BLE001 — anything: the reply must be a reply
+            sys.stderr.write(f"[trainer.serve] {self.command} {self.path}: {type(exc).__name__}: {exc}\n")
+            try:
+                self._json(500, {"error": f"{type(exc).__name__}: {exc}"})
+            except Exception:  # noqa: BLE001 — headers already sent; nothing left to do
+                pass
+
+    def _get(self) -> None:
         path = self.path.split("?", 1)[0]
         j = self.jobs
         if path == "/health":
@@ -336,7 +356,7 @@ class Handler(BaseHTTPRequestHandler):
         else:
             self._json(404, {"error": "not found"})
 
-    def do_POST(self) -> None:  # noqa: N802
+    def _post(self) -> None:
         if self.path.split("?", 1)[0] != "/jobs":
             self._json(404, {"error": "not found"})
             return

komodo/resources.toml

@@ -54,6 +54,15 @@ COOKIE_SECURE=0
 # Venue modules this site is ENTITLED to (vendor decision; the site admin activates within
 # this set in Setup → Site). Unset = every registered module. See wiki/decisions/venue-modules.md.
 MODULES_ENTITLED=parking,validation
+# Car Wash review outbox (wiki/concepts/vision-review-outbox.md): the collector's ingest URL
+# on the Netbird overlay, this booth's PSEUDONYMOUS id (never the site name — the crops leave
+# the site), and its token — the SAME secret the wash-collector stack lists under that id.
+CARWASH_REVIEW_URL=http://docker-station.nb.infra:8090/ingest
+CARWASH_REVIEW_BOOTH_ID=booth-1
+CARWASH_REVIEW_TOKEN=[[wash_review_token_booth_buzi]]
+# Also send ENTRY reads as training material (gate view, no wash): 1 = every entry (storage
+# and bandwidth are not the limit; review what you have time for). N = one in N. 0 = off.
+CARWASH_REVIEW_ENTRY_SAMPLE=1
 VISION_ENABLED=1
 # Desktop app WS handshake: Origin is tauri://localhost (set explicitly by
 # platform-ws.ts, since the native WS plugin has no page context to auto-attach
@@ -114,8 +123,8 @@ BACKUP_KEY=[[park_2_backup_key]]
 # Stack — the LAB BENCH (not a booth): a spare Linux box with the field printer and
 # whatever device is under investigation, so a booth bug can be reproduced on the booth's
 # exact image before touching a real site. Same compose files + pinned TAG as the staging
-# booths. No review outbox (the lab is not a site — it must never feed the training pool
-# under a booth's identity). Its own secrets. See wiki/decisions/fleet-deployment-komodo.md.
+# booths. Review outbox under its OWN id (booth-lab) — never a
+# booth's identity, so bench crops stay separable on the collector. Its own secrets. See wiki/decisions/fleet-deployment-komodo.md.
 ##############################################################################
 
 [[stack]]
@@ -140,8 +149,13 @@ TAG=stage-88f9c53
 COOKIE_SECURE=0
 # Entitled to Car Wash too, so the wash-desk printer role and till can be exercised on the bench.
 MODULES_ENTITLED=parking,carwash
-# NO review outbox on the lab (CARWASH_REVIEW_URL/BOOTH_ID/TOKEN deliberately unset): the
-# collector's training pool is per-booth, and the bench is not a booth.
+# Review outbox ON for the bench too, under its OWN pseudonymous id (booth-lab) and its own
+# secret — never a real booth's — so lab crops are separable on the collector (per-booth stats,
+# the export's booth column) and the reviewer can label bench junk "unusable".
+CARWASH_REVIEW_URL=http://docker-station.nb.infra:8090/ingest
+CARWASH_REVIEW_BOOTH_ID=booth-lab
+CARWASH_REVIEW_TOKEN=[[wash_review_token_booth_lab]]
+CARWASH_REVIEW_ENTRY_SAMPLE=1
 VISION_ENABLED=1
 # Desktop app WS handshake: Origin is tauri://localhost (set explicitly by
 # platform-ws.ts, since the native WS plugin has no page context to auto-attach
@@ -176,7 +190,7 @@ registry_account = "komodo"
 environment = """
 REGISTRY=git.infra.msai.al/mca/parking_solution
 # Pinned like the booths: bump to the stage-<sha> that carries the collector.
-TAG=stage-2d9bb15
+TAG=stage-73bfc1d
 # The host's NETBIRD address (an IP: Docker port bindings take no hostname) — the ingest port
 # is published on the overlay only. Booths reach it by its Netbird DNS name.
 COLLECTOR_BIND=100.75.184.156
@@ -184,7 +198,7 @@ COLLECTOR_BIND=100.75.184.156
 # that booth's own stack as its CARWASH_REVIEW_TOKEN — one value, two consumers, nothing
 # to keep in sync, and rotating a booth touches one secret. The booth id is the booth's
 # pseudonymous CARWASH_REVIEW_BOOTH_ID, never a site name. Add a pair per booth.
-COLLECTOR_BOOTH_TOKENS=booth-2:[[wash_review_token_booth_2]]
+COLLECTOR_BOOTH_TOKENS=booth-2:[[wash_review_token_booth_2]],booth-1:[[wash_review_token_booth_buzi]],booth-lab:[[wash_review_token_booth_lab]]
 # Phase-B trainer (the `trainer` service beside the collector; the Training section of /review
 # is its UI). Only `publish` needs this: a Gitea token with package:write for the model's generic
 # package. Uncomment when the first model is to be published.

wiki/concepts/vision-review-outbox.md

@@ -154,6 +154,26 @@ its own repo the day it needs its own cadence. Deploy the collector BEFORE a boo
 package kind it does not know (a 422 is abandoned, not retried). The export neutralises cells
 that start like a spreadsheet formula (category/service names are booth-supplied text).
 
+> **Incident 2026-09-16 — the collector's DB predated the `kind` column; nothing worked for 9
+> days and nothing said so.** The reviewer opened /review: *Training — trainer not reachable:
+> fetch failed*. On the host: collector `stage-2d9bb15` **unhealthy** (`/health` → 500 *no such
+> column: kind*), trainer healthy but every `/readiness` a Python traceback; the volume's
+> `collector.sqlite` (created 2026-09-07 by the previous build, **0 items**) had the original
+> column set. `CREATE TABLE IF NOT EXISTS` shapes only a NEW database — an existing volume keeps
+> its old columns, so every query naming `kind` failed: the collector's health, **every ingest**
+> (booths would have got 500s and kept retrying — the log shows none ever arrived, a separate
+> question), and the trainer's readiness. The trainer's stdlib server printed the traceback and
+> dropped the socket, which the collector could only render as "fetch failed".
+>
+> Fixes (same day): `CollectorDb` now **migrates on open** — `PRAGMA table_info` vs a list of the
+> columns added since the first deploy, `ALTER TABLE … ADD COLUMN` for each missing one (all
+> nullable or defaulted; **append to that list whenever a column joins the CREATE**); the trainer's
+> handlers are guarded — an unexpected exception is a **500 JSON** naming the error, never a
+> dropped connection; the collector's status proxy surfaces the trainer's error text. Rule going
+> forward: the collector owns the schema; the trainer only reads; a deploy that changes the table
+> must be accompanied by a migration entry, and the Training section is the first place a
+> schema/DB mismatch shows — read its error text before suspecting the network.
+
 **Status (2026-09-07).** Live: the collector runs on `art-docker-station` and park-2 is wired to
 it (`stage-dbbb051` on both stacks, every entry sampled). The review screen at
 `http://docker-station.nb.infra:8090/review` is filling; no labels reviewed yet.

wiki/decisions/bodytype-classifier-training.md

@@ -173,6 +173,13 @@ The CLI is still there for debugging, inside the running container:
 - **Reviewing is the bottleneck**: the Training section shows labels per class against the
   minimum and keeps Train disabled until two classes clear it.
 
+**"Training — trainer not reachable: fetch failed" (2026-09-16).** Not a network problem: the
+trainer answered `/health` but its `/readiness` crashed on the collector's DB (a volume from before
+the `kind` column) and the stdlib server dropped the socket without a reply. Since the fix the
+trainer answers **500 JSON with the error** and the collector shows that text; a genuine network
+failure still reads "fetch failed" / ECONNREFUSED. See [[vision-review-outbox]] §Incident 2026-09-16.
+
+
 ## Packaging rule (same as the vision service)
 
 Core deps are light (numpy, opencv-headless, onnxruntime): `inspect`, `evaluate`, the data and

wiki/decisions/fleet-deployment-komodo.md

@@ -198,7 +198,7 @@ Second `[[stack]]` in `komodo/resources.toml`: **`park-lab`** (server = the lab
 
 | Stack | compose branch | image tag | secrets |
 | --- | --- | --- | --- |
-| park-lab | `dev` → **`stage` (2026-09-09)** | ~~moving `dev`~~ → **pinned to the booth's `stage-<sha>` under reproduction** (2026-09-09: the lab re-joined the fleet to reproduce the park-buzi printer cover-open bug, so it must run the booth's exact image; **no review outbox** — the bench is not a booth and must never feed the collector under a booth id) | `park_lab_*` |
+| park-lab | `dev` → **`stage` (2026-09-09)** | ~~moving `dev`~~ → **pinned to the booth's `stage-<sha>` under reproduction** (2026-09-09: the lab re-joined the fleet to reproduce the park-buzi printer cover-open bug, so it must run the booth's exact image; review outbox under its **own** id `booth-lab` since 2026-09-16 — never a booth's, so bench crops stay separable per booth on the collector) | `park_lab_*` |
 | park-buzi | `stage` | pinned `stage-<sha>` | `park_buzi_*` |
 
 The three knobs are independent per stack — the ResourceSync's own branch only governs where the

wiki/log.md

@@ -3308,3 +3308,17 @@ Live after the fix: degraded "cover open, paper out, printer off-line"; closed
 "we are good using the network with this printer." Also: park-lab Periphery was "not loaded"
 after a reboot — the unit had never been enabled; `systemctl --user enable --now periphery`
 recorded as a §7a gotcha on [[appliance-provisioning]]. Pages: [[k200l-printer]].
+
+## [2026-09-16] fix | Collector DB schema migration; trainer handlers answer 500 JSON; the "trainer not reachable" incident
+User: "Training — trainer not reachable: fetch failed". Read-only look at art-docker-station over
+SSH: collector stage-2d9bb15 unhealthy (`/health` 500 "no such column: kind"), trainer healthy
+but `/readiness` tracebacks on the same column; the volume's collector.sqlite (2026-09-07, 0
+items, original columns) predates `kind` — CREATE TABLE IF NOT EXISTS never migrates an existing
+table. No `/ingest` request in the container's 9-day log at all (park-2 either not sending or not
+reaching the host — to check on the booth). Built: `CollectorDb.#migrate()` (PRAGMA table_info vs
+the list of columns added since the first deploy → ALTER TABLE ADD COLUMN; test replays the old
+schema: health, ingest, stats, legacy row reads back with defaults); trainer `Handler._guarded`
+(any exception → 500 JSON naming it; test: readiness on an old-schema DB → 500 "no such column:
+kind", /health still 200); the collector's training proxy includes the trainer's error text. Pages:
+[[vision-review-outbox]] (incident + rule), [[bodytype-classifier-training]] (what the message
+means). Deploy: nothing manual — the new collector migrates on start.