Implement Phase 1 task 1.8 (Redis Streams publisher + main wiring)

- Bounded in-memory queue (default 10000); overflow throws PublishOverflowError
  so the framing layer skips ACK and the device retransmits.
- Background worker drains via XADD with MAXLEN ~ approximate trimming.
- JSON serialization with sentinel encoding for bigint/Buffer/Date; correctly
  handles Buffer.prototype.toJSON firing before the replacer.
- AdapterContext.publish(position, codec) with codec-label closure at dispatch
  in adapters/teltonika/index.ts; zero changes to the three codec parsers.
- connectRedis with retry-on-startup; main.ts wires the full pipeline.
- installGracefulShutdown stubbed (full hardening in task 1.12).
- 19 new tests (17 unit + 2 Docker-conditional integration). Total 81 passing.
This commit is contained in:
2026-04-30 16:37:51 +02:00
parent 381287bacc
commit c33c7a4f6b
12 changed files with 1896 additions and 55 deletions
+93 -34
View File
@@ -1,17 +1,19 @@
import type { Redis } from 'ioredis';
import type * as net from 'node:net';
import { loadConfig } from './config/load.js';
import type { Config } from './config/load.js';
import { createLogger } from './observability/logger.js';
import { makePublisher } from './core/publish.js';
import { createPublisher, connectRedis } from './core/publish.js';
import { startServer } from './core/server.js';
import { createTeltonikaAdapter } from './adapters/teltonika/index.js';
import { AllowAllAuthority } from './adapters/teltonika/device-authority.js';
import { CodecRegistry } from './adapters/teltonika/codec/registry.js';
import type { Metrics } from './core/types.js';
// -------------------------------------------------------------------------
// Startup: validate config (fail fast on bad env), build logger, boot server
// -------------------------------------------------------------------------
let config;
let config: Config;
try {
config = loadConfig();
} catch (err) {
@@ -29,45 +31,102 @@ const logger = createLogger({
logger.info('tcp-ingestion starting');
// Placeholder metrics implementation — replaced in task 1.10
// Placeholder metrics implementation — replaced in task 1.10.
// Using the Metrics interface from types.ts (no prom-client yet).
const metrics: Metrics = {
inc: (name, labels) => logger.debug({ metric: name, labels }, 'metric inc'),
observe: (name, value, labels) => logger.debug({ metric: name, value, labels }, 'metric observe'),
};
const publisher = makePublisher(logger);
// -------------------------------------------------------------------------
// Wire up the pipeline
// -------------------------------------------------------------------------
// Codec registry — empty until tasks 1.51.7 register handlers
const codecRegistry = new CodecRegistry();
async function main(): Promise<void> {
// 1. Connect Redis with exponential-backoff retry (3 attempts, up to 5s backoff)
const redis = await connectRedis(config.REDIS_URL, logger);
const teltonikaAdapter = createTeltonikaAdapter({
port: config.TELTONIKA_PORT,
deviceAuthority: new AllowAllAuthority(),
strictDeviceAuth: config.STRICT_DEVICE_AUTH,
codecRegistry,
});
// 2. Build the publisher (bounded queue + XADD worker)
const publisher = createPublisher(redis, config, logger, metrics);
const ctx = {
publish: publisher,
logger,
metrics,
};
const server = startServer(config.TELTONIKA_PORT, teltonikaAdapter, ctx);
// Graceful shutdown
function shutdown(signal: string): void {
logger.info({ signal }, 'shutdown signal received; closing server');
server.close(() => {
logger.info('server closed; exiting');
process.exit(0);
// 3. Build the Teltonika adapter (all three Phase 1 codecs registered via defaultRegistry)
const teltonikaAdapter = createTeltonikaAdapter({
port: config.TELTONIKA_PORT,
deviceAuthority: new AllowAllAuthority(),
strictDeviceAuth: config.STRICT_DEVICE_AUTH,
// No explicit codecRegistry — createTeltonikaAdapter builds defaultRegistry
// with codec8Handler, codec8eHandler, codec16Handler pre-registered.
});
// Force exit after 10s if connections are still open
setTimeout(() => {
logger.warn('forced exit after timeout');
process.exit(1);
}, 10_000).unref();
// 4. Start TCP server — publisher.publish is the AdapterContext.publish impl
const server = startServer(config.TELTONIKA_PORT, teltonikaAdapter, {
publish: publisher.publish,
logger,
metrics,
});
// 5. Install graceful shutdown (stub — full hardening in task 1.12)
installGracefulShutdown({ server, redis, publisher, logger });
logger.info({ port: config.TELTONIKA_PORT }, 'tcp-ingestion ready');
}
process.on('SIGTERM', () => shutdown('SIGTERM'));
process.on('SIGINT', () => shutdown('SIGINT'));
// -------------------------------------------------------------------------
// Graceful shutdown stub — task 1.12 finalizes this
// -------------------------------------------------------------------------
type ShutdownDeps = {
readonly server: net.Server;
readonly redis: Redis;
readonly publisher: { drain(timeoutMs: number): Promise<void> };
readonly logger: ReturnType<typeof createLogger>;
};
function installGracefulShutdown(deps: ShutdownDeps): void {
const { server, redis, publisher, logger: log } = deps;
let shuttingDown = false;
function shutdown(signal: string): void {
if (shuttingDown) return;
shuttingDown = true;
log.info({ signal }, 'shutdown signal received');
// Stop accepting new connections
server.close(() => {
log.info('TCP server closed');
});
// Drain publisher queue then disconnect Redis
publisher
.drain(10_000)
.then(() => redis.quit())
.then(() => {
log.info('graceful shutdown complete');
process.exit(0);
})
.catch((err) => {
log.error({ err }, 'error during shutdown');
process.exit(1);
});
// Force exit after 15s if graceful path stalls
setTimeout(() => {
log.warn('forced exit after shutdown timeout');
process.exit(1);
}, 15_000).unref();
}
process.on('SIGTERM', () => shutdown('SIGTERM'));
process.on('SIGINT', () => shutdown('SIGINT'));
}
// -------------------------------------------------------------------------
// Entry point
// -------------------------------------------------------------------------
main().catch((err) => {
process.stderr.write(`Fatal startup error: ${err instanceof Error ? err.message : String(err)}\n`);
process.exit(1);
});