fix(button-light): back off failed setAux sends — kill the ENETUNREACH hot loop
An unreachable controller rejects the UDP send instantly, and #pump's failure re-pump retried inline: a tight loop logging hundreds of identical errors per minute (park-buzi, 2026-07-07). Failed sends now arm a 1s→30s exponential retry (reset on success); desiredOn keeps tracking the truth table meanwhile and the armed retry converges to it. Logging is rate-limited: first failure of a streak in full, then one summary/minute, one info line on recovery. #finalOff waives the backoff so the last-gasp OFF on drop/shutdown still gets an immediate try. Claude-Session: https://claude.ai/code/session_01Xcm6ikLgGoCxxHrxtjkk5V
This commit is contained in:
@@ -192,11 +192,78 @@ describe("ButtonLightController truth table", () => {
|
||||
// First write (initial off) throws — must be swallowed.
|
||||
expect(() => ctl.start()).not.toThrow();
|
||||
await flush();
|
||||
// Subsequent writes work; driving to solid still converges to ON.
|
||||
// The failure arms a backoff (1s) rather than retrying inline; desired-state
|
||||
// changes during the window just update the target the retry will assert.
|
||||
lane(true);
|
||||
radar(true);
|
||||
await flush();
|
||||
expect(ctl.confirmedOf(CONTROLLER)).toBeNull(); // still backing off
|
||||
await vi.advanceTimersByTimeAsync(1000); // retry fires; aux is healthy again
|
||||
expect(ctl.confirmedOf(CONTROLLER)).toBe(true); // converged to solid ON
|
||||
ctl.stop();
|
||||
});
|
||||
|
||||
it("an unreachable controller backs off (1s→30s), not a hot retry loop", async () => {
|
||||
let attempts = 0;
|
||||
const aux: AuxOutputDevice = {
|
||||
async setAux() {
|
||||
attempts += 1;
|
||||
throw new Error("send ENETUNREACH 10.0.10.5:60000");
|
||||
},
|
||||
};
|
||||
const errors: string[] = [];
|
||||
const logger = silentLogger();
|
||||
(logger as { error: (msg: string) => void }).error = (msg) => errors.push(msg);
|
||||
const ctl = new ButtonLightController(db, logger, () => aux);
|
||||
ctl.start(); // initial OFF write → attempt 1 fails at t=0
|
||||
await flush();
|
||||
expect(attempts).toBe(1); // the old code hot-looped here
|
||||
|
||||
// Failures at t≈0,1,3,7,15,31 (doubling, capped 30s) → 6 attempts in the first
|
||||
// minute instead of thousands.
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
expect(attempts).toBeGreaterThanOrEqual(5);
|
||||
expect(attempts).toBeLessThanOrEqual(7);
|
||||
|
||||
// Only the FIRST failure was logged so far; the next log is a ≥60s summary.
|
||||
expect(errors).toHaveLength(1);
|
||||
await vi.advanceTimersByTimeAsync(35_000); // t≈95s → the t=61s attempt logged a summary
|
||||
expect(errors.length).toBe(2);
|
||||
expect(errors[1]).toContain("still failing");
|
||||
ctl.stop();
|
||||
});
|
||||
|
||||
it("logs a single recovery line and resets the backoff after success", async () => {
|
||||
let failing = true;
|
||||
let attempts = 0;
|
||||
const aux: AuxOutputDevice = {
|
||||
async setAux() {
|
||||
attempts += 1;
|
||||
if (failing) throw new Error("send ENETUNREACH 10.0.10.5:60000");
|
||||
},
|
||||
};
|
||||
const infos: string[] = [];
|
||||
const logger = silentLogger();
|
||||
(logger as { info: (msg: string) => void }).info = (msg) => infos.push(msg);
|
||||
const ctl = new ButtonLightController(db, logger, () => aux);
|
||||
ctl.start();
|
||||
await flush();
|
||||
await vi.advanceTimersByTimeAsync(3_000); // attempts at t=0,1,3 all fail
|
||||
const failed = attempts;
|
||||
expect(failed).toBeGreaterThanOrEqual(3);
|
||||
|
||||
failing = false; // controller reachable again
|
||||
await vi.advanceTimersByTimeAsync(8_000); // next armed retry succeeds
|
||||
expect(ctl.confirmedOf(CONTROLLER)).toBe(false); // OFF asserted on the device
|
||||
expect(infos.filter((m) => m.includes("recovered"))).toHaveLength(1);
|
||||
|
||||
// Backoff reset: a fresh state change sends immediately (no lingering retryAt).
|
||||
const before = attempts;
|
||||
lane(true);
|
||||
radar(true);
|
||||
await flush();
|
||||
expect(ctl.confirmedOf(CONTROLLER)).toBe(true);
|
||||
expect(attempts).toBe(before + 1);
|
||||
ctl.stop();
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user