diff --git a/apps/collector/src/app.test.ts b/apps/collector/src/app.test.ts index 7fe416d..ecaeb71 100644 --- a/apps/collector/src/app.test.ts +++ b/apps/collector/src/app.test.ts @@ -151,3 +151,46 @@ describe("config", () => { expect(() => parseBoothTokens("nocolon")).toThrow(/bad pair/); }); }); + +describe("schema migration", () => { + it("opens a database created before the `kind` column and adds the missing columns, so ingest and stats work", async () => { + // art-docker-station, 2026-09-16: the volume's DB predated `kind`; CREATE TABLE IF NOT + // EXISTS left it alone, and /health, every ingest and the trainer's readiness failed + // with "no such column: kind". Replay: a file with the ORIGINAL column set. + const { default: Database } = await import("better-sqlite3"); + const file = path.join(dir, "old.sqlite"); + const old = new Database(file); + old.exec(`CREATE TABLE items ( + id TEXT PRIMARY KEY, booth TEXT NOT NULL, order_ref TEXT NOT NULL, at TEXT NOT NULL, + service TEXT NOT NULL, vision_class TEXT NOT NULL, vision_confidence REAL NOT NULL, + image_width INTEGER NOT NULL, image_height INTEGER NOT NULL, plate_blurred INTEGER NOT NULL, + image_path TEXT NOT NULL, received_at TEXT NOT NULL, review_label TEXT, reviewed_at TEXT, reviewer TEXT)`); + old.prepare( + "INSERT INTO items (id, booth, order_ref, at, service, vision_class, vision_confidence, image_width, image_height, plate_blurred, image_path, received_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + ).run("legacy-1", "booth-7", "o-0", "2026-09-01T00:00:00.000Z", "Standard", "suv", 0.8, 100, 100, 1, "crops/legacy-1.jpg", "2026-09-01T00:00:00.000Z"); + old.close(); + + const legacy = await buildCollector({ host: "127.0.0.1", port: 0, dataDir: dir, boothTokens: TOKENS, reviewer: REVIEWER, trainerUrl: null }, { dbFile: file }); + await legacy.ready(); + try { + const health = await legacy.inject({ method: "GET", url: "/health" }); + expect(health.statusCode).toBe(200); + expect(health.json()).toMatchObject({ ok: true, booths: 1, pending: 1 }); + + const { body, type } = multipart({ meta: JSON.stringify(meta({ item: "item-new" })) }, JPEG); + const r = await legacy.inject({ method: "POST", url: "/ingest", headers: { authorization: `Bearer ${TOKENS.get("booth-7")!}`, "content-type": type }, payload: body }); + expect(r.statusCode).toBe(201); + + const stats = await legacy.inject({ method: "GET", url: "/api/stats", headers: { authorization: basic } }); + expect(stats.statusCode).toBe(200); + + // The legacy row reads back with the defaults the new columns carry. + const cols = new Database(file, { readonly: true }).prepare("PRAGMA table_info(items)").all() as { name: string }[]; + expect(cols.map((c) => c.name)).toEqual(expect.arrayContaining(["kind", "operator_ref", "operator_classes", "vision_category_id", "downgraded"])); + const legacyRow = new Database(file, { readonly: true }).prepare("SELECT kind, operator_classes, downgraded FROM items WHERE id = 'legacy-1'").get() as Record; + expect(legacyRow).toEqual({ kind: "wash", operator_classes: "[]", downgraded: 0 }); + } finally { + await legacy.close(); + } + }); +}); diff --git a/apps/collector/src/app.ts b/apps/collector/src/app.ts index 73dc736..844efe4 100644 --- a/apps/collector/src/app.ts +++ b/apps/collector/src/app.ts @@ -253,7 +253,18 @@ export async function buildCollector(cfg: CollectorConfig, opts: { dbFile?: stri try { const get = async (p: string) => { const r = await fetch(trainer + p, { signal: AbortSignal.timeout(15_000) }); - if (!r.ok) throw new Error(`${p} → HTTP ${r.status}`); + if (!r.ok) { + // Surface the trainer's own error text (its handlers answer 500 JSON), so the + // reviewer reads "no such column: kind", not just a status code. + const detail = await r.text().then((t) => { + try { + return String((JSON.parse(t) as { error?: unknown }).error ?? t); + } catch { + return t; + } + }, () => ""); + throw new Error(`${p} → HTTP ${r.status}${detail ? `: ${detail.slice(0, 300)}` : ""}`); + } return r.json() as Promise>; }; const [health, readiness, versions, jobs] = await Promise.all([get("/health"), get("/readiness"), get("/versions"), get("/jobs")]); diff --git a/apps/collector/src/db.ts b/apps/collector/src/db.ts index abaf8db..b6ab2f0 100644 --- a/apps/collector/src/db.ts +++ b/apps/collector/src/db.ts @@ -71,6 +71,39 @@ export class CollectorDb { CREATE INDEX IF NOT EXISTS items_pending ON items (reviewed_at, received_at); CREATE INDEX IF NOT EXISTS items_booth ON items (booth, received_at); `); + this.#migrate(); + } + + /** Columns added after the first deploy, with the DDL that adds them to an EXISTING + * table. `CREATE TABLE IF NOT EXISTS` above only shapes a NEW database; a volume that + * was created by an earlier build keeps its old columns, and every query naming a new + * one then fails ("no such column: kind" — art-docker-station, 2026-09-16: the + * collector's /health, every ingest, and the trainer's readiness all broke on a DB + * from before `kind`). Each entry must be addable to a populated table, i.e. nullable + * or carrying a DEFAULT. Append here whenever a column joins the CREATE above. */ + static readonly #ADDED_COLUMNS: ReadonlyArray = [ + ["kind", "TEXT NOT NULL DEFAULT 'wash'"], + ["operator_ref", "TEXT NOT NULL DEFAULT ''"], + ["operator_category_id", "TEXT NOT NULL DEFAULT ''"], + ["operator_category_name", "TEXT NOT NULL DEFAULT ''"], + ["operator_classes", "TEXT NOT NULL DEFAULT '[]'"], + ["vision_category_id", "TEXT"], + ["downgraded", "INTEGER NOT NULL DEFAULT 0"], + ]; + + /** Bring an existing `items` table up to the current column set (idempotent). */ + #migrate(): void { + const present = new Set( + (this.#db.prepare("PRAGMA table_info(items)").all() as { name: string }[]).map((c) => c.name), + ); + for (const [name, ddl] of CollectorDb.#ADDED_COLUMNS) { + if (!present.has(name)) this.#db.exec(`ALTER TABLE items ADD COLUMN ${name} ${ddl}`); + } + } + + /** The current column names of `items` (for tests and diagnostics). */ + columns(): string[] { + return (this.#db.prepare("PRAGMA table_info(items)").all() as { name: string }[]).map((c) => c.name); } close(): void { diff --git a/apps/trainer/tests/test_server.py b/apps/trainer/tests/test_server.py index 75bfa60..33e5131 100644 --- a/apps/trainer/tests/test_server.py +++ b/apps/trainer/tests/test_server.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import sqlite3 import threading import urllib.error import urllib.request @@ -109,3 +110,38 @@ def test_train_job_then_versions_and_report(api) -> None: # type: ignore[no-unt code, job2 = call("POST", "/jobs", {"kind": "evaluate", "version": "vapi"}) wait_idle(Handler.jobs) assert call("GET", f"/jobs/{job2['id']}")[1]["status"] == "done" + + +def test_a_handler_error_is_a_500_json_reply_not_a_dropped_connection(tmp_path: Path) -> None: + """A collector DB from before the `kind` column (art-docker-station, 2026-09-16): readiness + raised sqlite3.OperationalError, the stdlib server printed the traceback and closed the + socket, and the collector could only say "trainer not reachable: fetch failed". The + handler must answer 500 JSON naming the error instead.""" + old = tmp_path / "old" + old.mkdir() + con = sqlite3.connect(old / "collector.sqlite") + con.executescript( + "CREATE TABLE items (id TEXT PRIMARY KEY, booth TEXT NOT NULL, order_ref TEXT NOT NULL," + " at TEXT NOT NULL, service TEXT NOT NULL, vision_class TEXT NOT NULL," + " vision_confidence REAL NOT NULL, image_width INTEGER NOT NULL, image_height INTEGER NOT NULL," + " plate_blurred INTEGER NOT NULL, image_path TEXT NOT NULL, received_at TEXT NOT NULL," + " review_label TEXT, reviewed_at TEXT, reviewer TEXT)" + ) + con.commit() + con.close() + Handler.jobs = Jobs(old, tmp_path / "out", "https://example.invalid/pkg", "tok") + httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + try: + req = urllib.request.Request(f"http://127.0.0.1:{httpd.server_address[1]}/readiness") + with pytest.raises(urllib.error.HTTPError) as ei: + urllib.request.urlopen(req, timeout=10) + assert ei.value.code == 500 + body = json.loads(ei.value.read()) + assert "no such column: kind" in body["error"] + # /health does not touch the DB and still answers. + with urllib.request.urlopen(f"http://127.0.0.1:{httpd.server_address[1]}/health", timeout=10) as r: + assert r.status == 200 + finally: + httpd.shutdown() + httpd.server_close() diff --git a/apps/trainer/trainer/server.py b/apps/trainer/trainer/server.py index ecead02..31d468f 100644 --- a/apps/trainer/trainer/server.py +++ b/apps/trainer/trainer/server.py @@ -309,6 +309,26 @@ class Handler(BaseHTTPRequestHandler): self.wfile.write(raw) def do_GET(self) -> None: # noqa: N802 + self._guarded(self._get) + + def do_POST(self) -> None: # noqa: N802 + self._guarded(self._post) + + def _guarded(self, handler: Any) -> None: + """Run a handler; an unexpected exception becomes a 500 JSON reply instead of a + dropped connection (the stdlib server would print the traceback and close the + socket, which the collector could only report as "fetch failed" — 2026-09-16, + a collector DB from before the `kind` column).""" + try: + handler() + except Exception as exc: # noqa: BLE001 — anything: the reply must be a reply + sys.stderr.write(f"[trainer.serve] {self.command} {self.path}: {type(exc).__name__}: {exc}\n") + try: + self._json(500, {"error": f"{type(exc).__name__}: {exc}"}) + except Exception: # noqa: BLE001 — headers already sent; nothing left to do + pass + + def _get(self) -> None: path = self.path.split("?", 1)[0] j = self.jobs if path == "/health": @@ -336,7 +356,7 @@ class Handler(BaseHTTPRequestHandler): else: self._json(404, {"error": "not found"}) - def do_POST(self) -> None: # noqa: N802 + def _post(self) -> None: if self.path.split("?", 1)[0] != "/jobs": self._json(404, {"error": "not found"}) return diff --git a/wiki/concepts/vision-review-outbox.md b/wiki/concepts/vision-review-outbox.md index 3dc15ef..8d5906e 100644 --- a/wiki/concepts/vision-review-outbox.md +++ b/wiki/concepts/vision-review-outbox.md @@ -154,6 +154,26 @@ its own repo the day it needs its own cadence. Deploy the collector BEFORE a boo package kind it does not know (a 422 is abandoned, not retried). The export neutralises cells that start like a spreadsheet formula (category/service names are booth-supplied text). +> **Incident 2026-09-16 — the collector's DB predated the `kind` column; nothing worked for 9 +> days and nothing said so.** The reviewer opened /review: *Training — trainer not reachable: +> fetch failed*. On the host: collector `stage-2d9bb15` **unhealthy** (`/health` → 500 *no such +> column: kind*), trainer healthy but every `/readiness` a Python traceback; the volume's +> `collector.sqlite` (created 2026-09-07 by the previous build, **0 items**) had the original +> column set. `CREATE TABLE IF NOT EXISTS` shapes only a NEW database — an existing volume keeps +> its old columns, so every query naming `kind` failed: the collector's health, **every ingest** +> (booths would have got 500s and kept retrying — the log shows none ever arrived, a separate +> question), and the trainer's readiness. The trainer's stdlib server printed the traceback and +> dropped the socket, which the collector could only render as "fetch failed". +> +> Fixes (same day): `CollectorDb` now **migrates on open** — `PRAGMA table_info` vs a list of the +> columns added since the first deploy, `ALTER TABLE … ADD COLUMN` for each missing one (all +> nullable or defaulted; **append to that list whenever a column joins the CREATE**); the trainer's +> handlers are guarded — an unexpected exception is a **500 JSON** naming the error, never a +> dropped connection; the collector's status proxy surfaces the trainer's error text. Rule going +> forward: the collector owns the schema; the trainer only reads; a deploy that changes the table +> must be accompanied by a migration entry, and the Training section is the first place a +> schema/DB mismatch shows — read its error text before suspecting the network. + **Status (2026-09-07).** Live: the collector runs on `art-docker-station` and park-2 is wired to it (`stage-dbbb051` on both stacks, every entry sampled). The review screen at `http://docker-station.nb.infra:8090/review` is filling; no labels reviewed yet. diff --git a/wiki/decisions/bodytype-classifier-training.md b/wiki/decisions/bodytype-classifier-training.md index bb4d812..499b265 100644 --- a/wiki/decisions/bodytype-classifier-training.md +++ b/wiki/decisions/bodytype-classifier-training.md @@ -173,6 +173,13 @@ The CLI is still there for debugging, inside the running container: - **Reviewing is the bottleneck**: the Training section shows labels per class against the minimum and keeps Train disabled until two classes clear it. +**"Training — trainer not reachable: fetch failed" (2026-09-16).** Not a network problem: the +trainer answered `/health` but its `/readiness` crashed on the collector's DB (a volume from before +the `kind` column) and the stdlib server dropped the socket without a reply. Since the fix the +trainer answers **500 JSON with the error** and the collector shows that text; a genuine network +failure still reads "fetch failed" / ECONNREFUSED. See [[vision-review-outbox]] §Incident 2026-09-16. + + ## Packaging rule (same as the vision service) Core deps are light (numpy, opencv-headless, onnxruntime): `inspect`, `evaluate`, the data and diff --git a/wiki/log.md b/wiki/log.md index 088b8d7..446f279 100644 --- a/wiki/log.md +++ b/wiki/log.md @@ -3308,3 +3308,17 @@ Live after the fix: degraded "cover open, paper out, printer off-line"; closed "we are good using the network with this printer." Also: park-lab Periphery was "not loaded" after a reboot — the unit had never been enabled; `systemctl --user enable --now periphery` recorded as a §7a gotcha on [[appliance-provisioning]]. Pages: [[k200l-printer]]. + +## [2026-09-16] fix | Collector DB schema migration; trainer handlers answer 500 JSON; the "trainer not reachable" incident +User: "Training — trainer not reachable: fetch failed". Read-only look at art-docker-station over +SSH: collector stage-2d9bb15 unhealthy (`/health` 500 "no such column: kind"), trainer healthy +but `/readiness` tracebacks on the same column; the volume's collector.sqlite (2026-09-07, 0 +items, original columns) predates `kind` — CREATE TABLE IF NOT EXISTS never migrates an existing +table. No `/ingest` request in the container's 9-day log at all (park-2 either not sending or not +reaching the host — to check on the booth). Built: `CollectorDb.#migrate()` (PRAGMA table_info vs +the list of columns added since the first deploy → ALTER TABLE ADD COLUMN; test replays the old +schema: health, ingest, stats, legacy row reads back with defaults); trainer `Handler._guarded` +(any exception → 500 JSON naming it; test: readiness on an old-schema DB → 500 "no such column: +kind", /health still 200); the collector's training proxy includes the trainer's error text. Pages: +[[vision-review-outbox]] (incident + rule), [[bodytype-classifier-training]] (what the message +means). Deploy: nothing manual — the new collector migrates on start.