From 540b333b06cfa3c62d16658bfb5afb62cc892b81 Mon Sep 17 00:00:00 2001 From: Julian Cuni Date: Fri, 19 Jun 2026 16:29:10 +0200 Subject: [PATCH] feat(vision): persist every recognized plate + snapshot as telemetry (non-blocking) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers "is a recognized plate saved?" — now yes, for both transient and subscriber, as an ANPR audit trail independent of whether it matched anything. VisionReader now stores the snapshot bytes in `snapshots` keyed by identity=PLATE — the same identity the flow signs its anomaly/event with — so GET /api/snapshots/by-identity/:plate (the booth event-detail modal's snapshot strip) shows the car's photo against that anomaly with no UI changes. It also records an unsigned device_events{kind:"read"} breadcrumb (plate, confidence, region, model, snapshotId, and the dispatch outcome) as a queryable recognition log. Switched from emitRead to calling ReadDispatcher.dispatch directly (like qr-reader) to capture that outcome. Non-blocking: a refused read (no session / unpaid / unknown plate) just returns rejected — no barrier hold — and is logged with its snapshot for investigation. Plate stays advisory (exit demands payment; subscription matches only a bound plate). Verified e2e: a recognized AL plate with no open session signed exit.refused.noSession (identity=plate), stored a 555KB snapshot under that plate, recorded the read breadcrumb (accepted:false, reason "no open session"), and by-identity returned the image — the refused read is fully investigable with its picture. Build + lint green. Claude-Session: https://claude.ai/code/session_01Xcm6ikLgGoCxxHrxtjkk5V --- apps/server/src/server.ts | 2 +- apps/server/src/vision-reader.ts | 148 ++++++++++++++++++++++++--- wiki/entities/opencv-anpr-service.md | 28 +++-- wiki/log.md | 4 + 4 files changed, 157 insertions(+), 25 deletions(-) diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 8b6f64b..7c816fc 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -177,7 +177,7 @@ export async function buildServer(opts: BuildOptions = {}): Promise visionReader.start()); app.addHook("onClose", async () => visionReader.stop()); diff --git a/apps/server/src/vision-reader.ts b/apps/server/src/vision-reader.ts index b3cbd90..390a11f 100644 --- a/apps/server/src/vision-reader.ts +++ b/apps/server/src/vision-reader.ts @@ -1,18 +1,37 @@ -import { devices, eq, type Db, type DeviceRow } from "@parking/db"; +import { randomUUID } from "node:crypto"; +import { + deviceEvents as deviceEventsTable, + devices, + eq, + snapshots, + type Db, + type DeviceRow, +} from "@parking/db"; import { registry, type CameraDevice } from "@parking/devices"; import type { FastifyBaseLogger } from "fastify"; -import { deviceEvents } from "./device-events.js"; -import { directionOf } from "./device-resolve.js"; +import { directionOf, type FlowDirection } from "./device-resolve.js"; +import type { ReadDispatcher } from "./read-dispatch.js"; import type { VisionClient } from "./vision-client.js"; // VisionReader — turns an ANPR camera into a virtual plate READER. It polls each // opt-in camera, sends a snapshot to the VisionClient (apps/vision), and on a CONFIDENT -// plate emits a `DeviceReadEvent{kind:"plate"}` onto the read bus — the SAME path a -// physical reader's scan takes, so the subscription/exit flows consume it unchanged. +// plate dispatches a `DeviceReadEvent{kind:"plate"}` through the SAME ReadDispatcher a +// physical reader's scan uses, so the subscription/exit flows consume it unchanged. +// +// EVERY confident read is PERSISTED for investigation (the ANPR audit trail): +// - the SNAPSHOT bytes are stored in `snapshots` keyed by `identity = plate` — the +// SAME identity the flow signs its anomaly/event with — so the booth event-detail +// modal's snapshot strip (GET /api/snapshots/by-identity/:identity) shows the car's +// photo against that anomaly with NO extra wiring. This is what makes a refused +// plate read investigable ("which car was this?"). +// - a `device_events` breadcrumb (kind:"read") records plate/confidence/region + +// the dispatch OUTCOME (accepted + reason), so there's a queryable log of every +// recognition and whether it matched, independent of the signed ledger. // // Per the fitness assessment (wiki/entities/opencv-anpr-service.md): a plate read is an -// ADVISORY identity + evidence, never the sole authority to open a paid barrier. The -// guards that keep it advisory live below the recognition, in the flows it feeds: +// ADVISORY identity + evidence, never the sole authority to open a paid barrier, and it +// NEVER BLOCKS — recognition runs alongside the flow; a refused read is logged with its +// snapshot, not a barrier hold. The guards that keep it advisory live in the flows: // - the EXIT flow still requires a covering `payment` (a plate can't bypass it); // - the SUBSCRIPTION flow only matches a plate BOUND to a subscription (subscriptionPlates). // So a recognized plate that owes money is refused exactly like a scanned ticket would be. @@ -22,7 +41,7 @@ import type { VisionClient } from "./vision-client.js"; // default). The VisionClient itself is also opt-in (VISION_ENABLED) and fail-soft. // - DEBOUNCE: a parked car sits in frame across many polls; the same plate from the // same camera is NOT re-emitted within `dedupeMs` (avoids a storm of identical reads). -// - LOW-CONFIDENCE reads are dropped (not emitted) — a shaky read must not act as an +// - LOW-CONFIDENCE reads are dropped (not dispatched) — a shaky read must not act as an // identity; the camera keeps polling until a confident frame (or the car leaves). const POLL_MS = Number(process.env.VISION_POLL_MS ?? 2000); @@ -36,6 +55,7 @@ interface CameraConfig { export class VisionReader { readonly #db: Db; readonly #vision: VisionClient; + readonly #dispatcher: ReadDispatcher; readonly #logger: FastifyBaseLogger; readonly #pollMs: number; readonly #dedupeMs: number; @@ -45,9 +65,17 @@ export class VisionReader { /** Last emitted plate + time per camera, for debounce. */ readonly #lastEmit = new Map(); - constructor(db: Db, vision: VisionClient, logger: FastifyBaseLogger, pollMs = POLL_MS, dedupeMs = DEDUPE_MS) { + constructor( + db: Db, + vision: VisionClient, + dispatcher: ReadDispatcher, + logger: FastifyBaseLogger, + pollMs = POLL_MS, + dedupeMs = DEDUPE_MS, + ) { this.#db = db; this.#vision = vision; + this.#dispatcher = dispatcher; this.#logger = logger; this.#pollMs = pollMs; this.#dedupeMs = dedupeMs; @@ -104,7 +132,7 @@ export class VisionReader { const direction = dir === "exit" ? "exit" : "entry"; // "both" → entry context const shot = await camera.captureSnapshot({ direction }); const result = await this.#vision.analyze(shot.bytes, shot.contentType); - // Fail-soft: null (disabled/unreachable/timeout) or no plate ⇒ nothing to emit. + // Fail-soft: null (disabled/unreachable/timeout) or no plate ⇒ nothing to do. if (!result || !result.plate) return; // Advisory gate: a low-confidence read is NOT an identity — drop it. if (result.lowConfidence) { @@ -116,16 +144,36 @@ export class VisionReader { if (this.#isDuplicate(row.id, plate)) return; // same car still in frame this.#lastEmit.set(row.id, { value: plate, at: Date.now() }); - // Emit onto the read bus — the SAME event a physical plate reader would send, so - // the ReadDispatcher routes it to the subscription/exit flow unchanged. - deviceEvents.emitRead({ + // PERSIST the snapshot keyed by `identity = plate` — the same identity the flow + // will sign its anomaly/event with — so the image is investigable against it. + const snapshotId = this.#storeSnapshot(row.id, direction, plate, shot.bytes, shot.contentType); + + this.#logger.info(`vision plate '${plate}' (${result.plate.confidence.toFixed(3)}) from camera ${row.id}`); + + // Dispatch the read through the SAME path a physical reader uses (like qr-reader), + // capturing the OUTCOME. A refused read does NOT block — it just returns rejected; + // we log it (with its snapshot already stored) for investigation. + const read = { driverId: row.driverId, deviceId: row.id, value: plate, - kind: "plate", + kind: "plate" as const, at: new Date().toISOString(), - }); - this.#logger.info(`vision plate '${plate}' (${result.plate.confidence.toFixed(3)}) from camera ${row.id}`); + }; + let outcome: { accepted: boolean; direction?: string; reason?: string }; + try { + outcome = await this.#dispatcher.dispatch(read); + } catch (err) { + outcome = { accepted: false, reason: (err as Error).message }; + } + + // Breadcrumb: a queryable record of the recognition + what the flow did with it. + this.#recordReadEvent(row, direction, plate, result, snapshotId, outcome); + if (!outcome.accepted) { + this.#logger.info( + `vision plate '${plate}' not accepted (${outcome.reason ?? "rejected"}) — logged with snapshot ${snapshotId ?? "none"}`, + ); + } } catch (err) { // Never let a camera/recognition error break the poll loop. this.#logger.warn(`vision reader ${row.id} failed: ${(err as Error).message}`); @@ -134,6 +182,74 @@ export class VisionReader { } } + /** Store the recognition snapshot keyed by `identity = plate`. Returns the snapshot + * id, or null on failure (persistence is best-effort — never blocks the flow). */ + #storeSnapshot( + cameraId: string, + direction: FlowDirection, + plate: string, + bytes: Buffer, + contentType: string, + ): string | null { + try { + const id = randomUUID(); + this.#db + .insert(snapshots) + .values({ + id, + direction, + deviceId: cameraId, + identity: plate, + contentType, + bytes, + capturedAt: new Date().toISOString(), + }) + .run(); + return id; + } catch (err) { + this.#logger.error(`vision snapshot store failed (${plate}): ${(err as Error).message}`); + return null; + } + } + + /** Record an unsigned `read` device_event: the ANPR audit trail (plate, confidence, + * region, the stored snapshot id, and the flow's outcome). Best-effort telemetry. */ + #recordReadEvent( + row: DeviceRow, + direction: FlowDirection, + plate: string, + result: { plate: { confidence: number; region?: string | null } | null; modelVersion: string }, + snapshotId: string | null, + outcome: { accepted: boolean; direction?: string; reason?: string }, + ): void { + try { + this.#db + .insert(deviceEventsTable) + .values({ + id: randomUUID(), + deviceId: row.id, + category: "camera", + kind: "read", + detail: { + driverId: row.driverId, + plate, + confidence: result.plate?.confidence, + region: result.plate?.region ?? null, + modelVersion: result.modelVersion, + direction, + snapshotId, + accepted: outcome.accepted, + outcomeDirection: outcome.direction, + reason: outcome.reason, + }, + occurredAt: new Date().toISOString(), + }) + .run(); + } catch (err) { + this.#logger.error(`vision read-event insert failed (${plate}): ${(err as Error).message}`); + } + } + /** Debounce: true if this same plate was emitted from this camera within dedupeMs. */ #isDuplicate(cameraId: string, plate: string): boolean { const last = this.#lastEmit.get(cameraId); diff --git a/wiki/entities/opencv-anpr-service.md b/wiki/entities/opencv-anpr-service.md index 8b0d50f..f12693f 100644 --- a/wiki/entities/opencv-anpr-service.md +++ b/wiki/entities/opencv-anpr-service.md @@ -173,14 +173,26 @@ now exists: **opt-in** (`VISION_ENABLED`, default off), **fail-soft** (any error end-to-end against the live service (Node → `AA558EE` 0.999, `region=Albania`). (2) ✅ **DONE — trigger wiring (`apps/server/src/vision-reader.ts`).** A **`VisionReader`** polls each **opt-in** camera (`config.anpr === true`, off by default) every `VISION_POLL_MS`, captures a snapshot → -`VisionClient.analyze` → on a **confident** plate emits a `DeviceReadEvent{kind:"plate"}` onto the -**same read bus a physical reader uses** (`deviceEvents.emitRead`), so the `ReadDispatcher` routes it -to the subscription/exit flow **unchanged**. Guards: low-confidence reads are dropped (not an -identity); a **debounce** (`VISION_DEDUPE_MS`) stops the same plate re-firing while a car sits in -frame; an in-flight guard prevents overlapping recognizes; idle when vision is off or no camera opts -in. Verified end-to-end (live service → reader → one `AA558EE` read on the bus; debounce held it to 1 -emit over 7 polls). Plate stays **advisory** — the exit flow still demands a `payment`, the -subscription flow only matches a **bound** plate. (3) **field-accuracy** unknown — re-benchmark/tune +`VisionClient.analyze` → on a **confident** plate dispatches a `DeviceReadEvent{kind:"plate"}` through +the **same `ReadDispatcher` a physical reader uses** (called directly to capture the outcome, like +`qr-reader.ts`), so the subscription/exit flow consumes it **unchanged**. Guards: low-confidence reads +are dropped (not an identity); a **debounce** (`VISION_DEDUPE_MS`) stops the same plate re-firing while +a car sits in frame; an in-flight guard prevents overlapping recognizes; idle when vision is off or no +camera opts in. Plate stays **advisory + non-blocking** — the exit flow still demands a `payment`, the +subscription flow only matches a **bound** plate, and a refused read never holds a barrier. + +**Every confident read is PERSISTED (the ANPR audit trail, so a read is investigable):** the +**snapshot bytes** are stored in `snapshots` keyed by **`identity = plate`** — the SAME identity the +flow signs its anomaly/event with — so `GET /api/snapshots/by-identity/:plate` (the booth event-detail +modal's snapshot strip) shows the car's photo **against that anomaly with no extra wiring**; plus an +unsigned **`device_events{kind:"read"}`** breadcrumb records plate / confidence / region / model / +`snapshotId` / the **dispatch outcome** (`accepted` + `reason`) — a queryable log of every recognition +and whether it matched, separate from the signed ledger. *Verified end-to-end:* a recognized AL plate +with no open session was non-blocking → signed a `exit.refused.noSession` anomaly (identity=plate), +stored a 555 KB snapshot under that plate, recorded the read breadcrumb with +`accepted:false, reason:"…no open session…"`, and `by-identity` returned the image — i.e. the refused +read is fully investigable with its picture. Debounce held a re-seen plate to 1 emit over 7 polls. +(3) **field-accuracy** unknown — re-benchmark/tune the threshold on real on-site captures (angle/night/dirt). (4) the **weight-provenance** check (open). **Bottom line: consume it as a gated advisory identity source feeding the existing `kind:"plate"` path — not as sole authority — and Job 2 is still required for the anti-spoofing value.** The diff --git a/wiki/log.md b/wiki/log.md index b6518fe..588a5a3 100644 --- a/wiki/log.md +++ b/wiki/log.md @@ -932,3 +932,7 @@ Scaffolded the Node-side adapter to the host vision microservice per the fitness ## [2026-06-19] feat | VisionReader — wire ANPR into the read bus (apps/server/src/vision-reader.ts) Wired the vision service into the entry/exit flows via the READ BUS. VisionReader polls each OPT-IN camera (config.anpr===true, off by default) every VISION_POLL_MS, captures a snapshot → VisionClient.analyze → on a CONFIDENT plate calls deviceEvents.emitRead({kind:"plate", value:PLATE, deviceId, driverId}) — the SAME event a physical plate reader emits, so the existing ReadDispatcher routes it to the subscription/exit flow UNCHANGED (no flow rewrite). The plate stays ADVISORY by construction: the exit flow still demands a covering payment (a plate can't bypass it), the subscription flow only matches a BOUND plate (subscriptionPlates). Guards: low-confidence reads DROPPED (a shaky read isn't an identity); DEBOUNCE (VISION_DEDUPE_MS, default 15s) so a parked car in frame doesn't re-fire the same plate; per-camera in-flight guard; idle when VISION_ENABLED off or no camera opts in; #recognizeOn is public for a future on-demand trigger (loop edge / API). Direction from directionOf (both→entry context). Constructed in server.ts, start on onReady / stop on onClose. VERIFIED END-TO-END: in-memory anpr camera returning the AL plate image + live fast_alpr service → VisionReader emitted exactly one {kind:"plate",value:"AA558EE"} read onto the bus; debounce held it to 1 emit over 7 polls. Build+lint green. Updated [[opencv-anpr-service]] (trigger-wiring gap + per-camera opt-in marked done). Remaining: SetupWizard anpr toggle, field tuning, weight-provenance, Job 2. + +## [2026-06-19] feat | Persist every recognized plate + snapshot (ANPR audit trail, non-blocking) + +VisionReader now PERSISTS every confident plate read so a recognition is investigable — and switched from emitRead to calling ReadDispatcher.dispatch directly (like qr-reader) to capture the OUTCOME. On a confident plate it: (1) stores the SNAPSHOT bytes in `snapshots` keyed by identity=PLATE — the SAME identity the flow signs its anomaly/event with — so GET /api/snapshots/by-identity/:plate (the booth event-detail modal's SnapshotStrip, which already uses e.identity) shows the car's photo against that anomaly with ZERO UI changes; (2) records an unsigned device_events{kind:"read"} breadcrumb with plate/confidence/region/modelVersion/snapshotId + the dispatch outcome (accepted + reason) = a queryable ANPR log independent of the signed ledger; (3) dispatches the read — NON-BLOCKING: a refused read just returns rejected (no barrier hold), logged with its snapshot for investigation. Plate stays advisory (exit demands payment; subscription matches only a bound plate). VERIFIED e2e: a recognized AL plate (AA558EE) with no open session → signed exit.refused.noSession anomaly (identity=plate), stored a 555KB snapshot under that plate, read-breadcrumb accepted:false reason:"no open session", and by-identity returned the image (status 200, 1 snapshot) → the refused read is investigable with its picture. Build+lint green. VisionReader constructor now takes the ReadDispatcher (wired in server.ts). Answers "is a recognized plate saved?" — now YES for both transient + subscriber, as telemetry + evidence image, regardless of match. Updated [[opencv-anpr-service]].