feat(snapshot): re-encode captures + disk-pressure retention
Camera snapshots were stored RAW — the camera's full-res JPEG straight into the BLOB, no resize/recompress. Measured on the dev DB: 300 snapshots = 81.7 MB = ~72% of the 114 MB SQLite file (the big ones 2688×1520 / ~600 KB, Hikvision main stream). They dominated the appliance's single backed-up DB file. Re-encode on capture (snapshot.ts): - Downscale each frame to SNAPSHOT_MAX_EDGE (1280px long edge) + recompress at SNAPSHOT_JPEG_QUALITY (80) via sharp (libvips, Apache-2.0) before storage — ~6-10× smaller (verified 2688×1520 → 1280×724, ~8×), plate still readable, clean image/jpeg (drops the camera's charset cruft). STORAGE-ONLY: recognition keeps the ORIGINAL full-res bytes (downscaling hurts OCR). Fail-soft — a re-encode error stores the original, never drops the snapshot or blocks the (already-open) path. sharp lives in apps/server (owns the capture path), where bcrypt already establishes the native-dep pattern. Disk-pressure retention (snapshot-retention.ts) — a SAFETY VALVE, not the daily mechanism (the re-encode does that). Daily check reads the DB filesystem used% (statfs on db.$client.name); no-op unless ≥ SNAPSHOT_DISK_HIGH_PCT (70). Over the mark: delete the OLDEST until an estimated SNAPSHOT_DISK_FREE_TARGET_PCT (10%) of disk is freed — never below SNAPSHOT_MIN_KEEP (500) — then VACUUM once to return space to the OS. A DELETE only frees SQLite pages (disk doesn't drop until VACUUM), so the loop is driven by estimated freed bytes (SUM(length(bytes))), not a live disk re-read; the prune owns the DB-locking VACUUM, run daily off-peak. diskUsage is injectable for tests. None of this touches the signed ledger — snapshots are unsigned/advisory, referenced only by id. Tests: encodeForStorage (downscale / clean-type / no-enlarge / fail-soft) + pruneSnapshots (no-op below mark / delete-oldest-to-target + VACUUM / MIN_KEEP floor / skip-VACUUM-when-empty). All four snapshot env knobs documented in the komodo env reference. Full workspace build/lint/test green; the prune smoke-verified on a scratch DB copy (file shrank after VACUUM). Existing ~81.7 MB of raw snapshots are unchanged (a one-off re-encode backfill is a separate optional follow-up). Updated entry-exit-points + technology-stack wiki. Claude-Session: https://claude.ai/code/session_01Xcm6ikLgGoCxxHrxtjkk5V
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import sharp from "sharp";
|
||||
import { deviceEvents as deviceEventsTable, snapshots, type Db } from "@parking/db";
|
||||
import { registry, type CameraDevice, type Snapshot } from "@parking/devices";
|
||||
import type { FastifyBaseLogger } from "fastify";
|
||||
@@ -27,6 +28,43 @@ import type { VisionClient } from "./vision-client.js";
|
||||
// fire-and-forget: it never blocks the open and never changes the entry/exit decision —
|
||||
// it's a record ("session X entered on plate AA558EE"). No polling; recognition only
|
||||
// happens on a real entry/exit. See wiki/entities/opencv-anpr-service.md.
|
||||
//
|
||||
// STORAGE RE-ENCODE (2026-06-28). Cameras serve full-res JPEGs (a Hikvision main stream is
|
||||
// 2688×1520 / ~600 KB); stored raw, snapshots dominated the appliance DB (~72%). Each frame
|
||||
// is now downscaled (long edge ≤ SNAPSHOT_MAX_EDGE) + re-compressed (q SNAPSHOT_JPEG_QUALITY)
|
||||
// BEFORE storage — ~6–10× smaller, plate still clearly readable. RECOGNITION runs on the
|
||||
// ORIGINAL full-res bytes (downscaling hurts OCR); the re-encode is storage-only. Fail-soft:
|
||||
// a re-encode error stores the original, never drops the snapshot or blocks the open.
|
||||
|
||||
/** Long-edge cap (px) + JPEG quality for the STORED snapshot. Env-overridable per appliance. */
|
||||
const SNAP_MAX_EDGE = Number(process.env.SNAPSHOT_MAX_EDGE ?? 1280);
|
||||
const SNAP_QUALITY = Number(process.env.SNAPSHOT_JPEG_QUALITY ?? 80);
|
||||
|
||||
/** Strip a camera's `; charset=...` cruft from a content type (a JPEG is binary). */
|
||||
function cleanType(ct: string): string {
|
||||
const base = ct.split(";")[0]?.trim();
|
||||
return base || "image/jpeg";
|
||||
}
|
||||
|
||||
/** Downscale + re-encode a captured frame for STORAGE (evidence, not OCR). Caps the long edge
|
||||
* and re-compresses to JPEG. Fail-soft: any error (e.g. a non-image body) returns the original
|
||||
* bytes with a cleaned content type, so a snapshot is never lost. */
|
||||
export async function encodeForStorage(
|
||||
shot: Snapshot,
|
||||
logger: FastifyBaseLogger,
|
||||
): Promise<{ bytes: Buffer; contentType: string }> {
|
||||
try {
|
||||
const out = await sharp(shot.bytes, { failOn: "none" })
|
||||
.rotate() // honor EXIF orientation before we drop the metadata
|
||||
.resize({ width: SNAP_MAX_EDGE, height: SNAP_MAX_EDGE, fit: "inside", withoutEnlargement: true })
|
||||
.jpeg({ quality: SNAP_QUALITY, mozjpeg: true })
|
||||
.toBuffer();
|
||||
return { bytes: out, contentType: "image/jpeg" };
|
||||
} catch (err) {
|
||||
logger.warn(`snapshot re-encode failed, storing original: ${(err as Error).message}`);
|
||||
return { bytes: shot.bytes, contentType: cleanType(shot.contentType) };
|
||||
}
|
||||
}
|
||||
|
||||
interface SnapshotJob {
|
||||
readonly db: Db;
|
||||
@@ -67,14 +105,17 @@ export function snapshotAsync(job: SnapshotJob): Promise<string[]> {
|
||||
// same vehicle, reuse it instead of a 2nd concurrent GET (which 503s).
|
||||
const shot = await captureSnapshotShared(row.id, camera, { direction });
|
||||
const id: string = randomUUID();
|
||||
// Re-encode for STORAGE only (downscale + recompress). Recognition below still
|
||||
// uses the original full-res `shot`.
|
||||
const stored = await encodeForStorage(shot, logger);
|
||||
db.insert(snapshots)
|
||||
.values({
|
||||
id,
|
||||
direction,
|
||||
deviceId: row.id,
|
||||
identity,
|
||||
contentType: shot.contentType,
|
||||
bytes: shot.bytes,
|
||||
contentType: stored.contentType,
|
||||
bytes: stored.bytes,
|
||||
capturedAt: shot.capturedAt,
|
||||
})
|
||||
.run();
|
||||
|
||||
Reference in New Issue
Block a user