Health Checks and Graceful Shutdown

// src/health.js
import { Router } from 'express';

function withTimeout(promise, ms) {
  let timer;
  const deadline = new Promise((_, reject) => {
    timer = setTimeout(() => reject(new Error('timeout')), ms);
  });
  return Promise.race([promise, deadline]).finally(() => clearTimeout(timer));
}

/**
 * @param {object} options
 * @param {{ shuttingDown: boolean }} options.state mutable state shared with the shutdown code
 * @param {Record<string, () => Promise<unknown>>} [options.checks] hard dependencies for readiness
 * @param {number} [options.checkTimeoutMs] deadline for each check
 */
export function createHealthRouter({ state, checks = {}, checkTimeoutMs = 1000 }) {
  const router = Router();

  router.use((req, res, next) => {
    res.set('Cache-Control', 'no-store');
    next();
  });

  // Liveness: the event loop answers. Never look at dependencies here.
  router.get('/livez', (req, res) => {
    res.json({ status: 'ok' });
  });

  // Readiness: not shutting down, and every hard dependency answers in time.
  router.get('/readyz', async (req, res) => {
    if (state.shuttingDown) {
      res.status(503).json({ status: 'shutting-down' });
      return;
    }

    const entries = await Promise.all(
      Object.entries(checks).map(async ([name, check]) => {
        try {
          await withTimeout(check(), checkTimeoutMs);
          return [name, 'up'];
        } catch {
          return [name, 'down'];
        }
      }),
    );

    const dependencies = Object.fromEntries(entries);
    const ready = entries.every(([, status]) => status === 'up');
    res.status(ready ? 200 : 503).json({ status: ready ? 'ready' : 'not-ready', dependencies });
  });

  return router;
}

// src/lifecycle.js
import { setTimeout as sleep } from 'node:timers/promises';

/**
 * Shared state read by the readiness endpoint and the drain middleware.
 */
export function createLifecycleState() {
  return { shuttingDown: false };
}

/**
 * While the server is draining, ask clients to close their keep-alive sockets.
 */
export function drainMiddleware(state) {
  return (req, res, next) => {
    if (state.shuttingDown) {
      res.set('Connection', 'close');
    }
    next();
  };
}

function closeServer(server) {
  return new Promise((resolve, reject) => {
    server.close((error) => (error ? reject(error) : resolve()));
    // Node.js 18 does not close idle connections in close(); do it explicitly.
    server.closeIdleConnections();
  });
}

/**
 * @param {object} options
 * @param {import('node:http').Server} options.server
 * @param {{ shuttingDown: boolean }} options.state
 * @param {Array<{ name: string, close: () => Promise<unknown> | unknown }>} [options.closers]
 * @param {number} [options.readinessDelayMs] keep serving while the platform removes the instance
 * @param {number} [options.drainTimeoutMs] after this, in-flight requests are cut off
 * @param {number} [options.hardExitMs] after this, the process exits whatever is pending
 */
export function createShutdown({
  server,
  state,
  closers = [],
  readinessDelayMs = 5000,
  drainTimeoutMs = 25000,
  hardExitMs = 35000,
  logger = console,
}) {
  let started = false;

  return async function shutdown(reason, exitCode = 0) {
    if (started) {
      logger.warn(`${reason} received while shutting down, exiting immediately`);
      process.exit(1);
    }
    started = true;
    logger.info(`${reason} received: draining`);

    // Last resort: never outlive the platform's grace period.
    setTimeout(() => {
      logger.error('Hard exit: shutdown did not finish in time');
      process.exit(1);
    }, hardExitMs).unref();

    let code = exitCode;

    // 1. Not ready any more; 2. keep serving while the load balancer notices.
    state.shuttingDown = true;
    await sleep(readinessDelayMs);

    // 3 and 4. Stop accepting connections and wait for in-flight requests...
    const drainDeadline = setTimeout(() => {
      logger.warn('Drain timeout reached: closing remaining connections');
      code = code || 1;
      server.closeAllConnections(); // 5. ...but only until the deadline.
    }, drainTimeoutMs);

    try {
      await closeServer(server);
    } catch (error) {
      logger.error('Error while closing the server', error);
      code = code || 1;
    } finally {
      clearTimeout(drainDeadline);
    }

    // 6. Close resources in reverse order of creation.
    for (const closer of [...closers].reverse()) {
      try {
        await closer.close();
        logger.info(`Closed ${closer.name}`);
      } catch (error) {
        logger.error(`Error while closing ${closer.name}`, error);
        code = code || 1;
      }
    }

    // 7. Exit explicitly: a stray handle must not keep a finished process alive.
    process.exit(code);
  };
}

/**
 * Register the process-level handlers once.
 */
export function registerShutdownHandlers(shutdown, logger = console) {
  for (const signal of ['SIGTERM', 'SIGINT']) {
    process.on(signal, () => {
      void shutdown(signal);
    });
  }

  // An uncaught exception leaves the process in an unknown state: log it, stop quickly with a failure code.
  process.on('uncaughtException', (error) => {
    logger.error('Uncaught exception', error);
    void shutdown('uncaughtException', 1);
  });
}

// src/app.js
import express from 'express';
import { createHealthRouter } from './health.js';
import { drainMiddleware } from './lifecycle.js';

export function createApp({ state, checks }) {
  const app = express();
  app.disable('x-powered-by');

  app.use(drainMiddleware(state));
  app.use(createHealthRouter({ state, checks }));

  app.get('/books', (req, res) => {
    res.json([{ id: 1, title: 'Example book' }]);
  });

  return app;
}

// src/server.js
import mongoose from 'mongoose';
import { createApp } from './app.js';
import { createLifecycleState, createShutdown, registerShutdownHandlers } from './lifecycle.js';

const port = Number(process.env.PORT ?? 3000);
const mongoUri = process.env.MONGODB_URI ?? 'mongodb://127.0.0.1:27017/bookshelf';

const state = createLifecycleState();

await mongoose.connect(mongoUri, { serverSelectionTimeoutMS: 5000 });

const app = createApp({
  state,
  checks: {
    mongodb: () => mongoose.connection.db.command({ ping: 1 }),
  },
});

const server = app.listen(port, (error) => {
  // Express 5 passes listen errors (EADDRINUSE and others) to this callback.
  if (error) {
    console.error('Cannot listen', error);
    process.exit(1);
  }
  console.log(`Bookshelf API listening on port ${port}`);
});

// Stay above the idle timeout of the load balancer in front of the service.
server.keepAliveTimeout = 65_000;
server.headersTimeout = 66_000;

const shutdown = createShutdown({
  server,
  state,
  closers: [
    // Registered in creation order; closed in reverse order.
    { name: 'mongodb', close: () => mongoose.connection.close() },
  ],
  readinessDelayMs: Number(process.env.SHUTDOWN_READINESS_DELAY_MS ?? 5000),
  drainTimeoutMs: Number(process.env.SHUTDOWN_DRAIN_TIMEOUT_MS ?? 25000),
  hardExitMs: Number(process.env.SHUTDOWN_HARD_EXIT_MS ?? 35000),
});

registerShutdownHandlers(shutdown);

# Dockerfile (excerpt)
FROM node:24-slim
WORKDIR /app
ENV NODE_ENV=production
COPY package.json package-lock.json ./
RUN npm ci --omit=dev
COPY src ./src
USER node
EXPOSE 3000

# The slim image has no curl: use Node.js itself for the container health check.
HEALTHCHECK --interval=15s --timeout=3s --start-period=20s --retries=3 \
  CMD node -e "fetch('http://127.0.0.1:3000/livez').then((r) => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))"

# Exec form: node is PID 1 and receives SIGTERM directly.
CMD ["node", "src/server.js"]

# compose.yaml (excerpt)
services:
  api:
    build: .
    init: true            # tini reaps child processes and forwards signals
    stop_grace_period: 40s
    environment:
      MONGODB_URI: mongodb://mongo:27017/bookshelf
      SHUTDOWN_DRAIN_TIMEOUT_MS: "25000"
      SHUTDOWN_HARD_EXIT_MS: "35000"
    healthcheck:
      test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3000/livez').then((r) => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))"]
      interval: 15s
      timeout: 3s
      retries: 3

# deployment.yaml (excerpt)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: bookshelf-api
spec:
  replicas: 3
  selector:
    matchLabels:
      app: bookshelf-api
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 0
      maxSurge: 1
  template:
    metadata:
      labels:
        app: bookshelf-api
    spec:
      terminationGracePeriodSeconds: 45
      containers:
        - name: api
          image: registry.example.com/bookshelf-api:1.0.0
          ports:
            - name: http
              containerPort: 3000
          env:
            - name: SHUTDOWN_READINESS_DELAY_MS
              value: "0"            # the preStop sleep already waits for endpoint removal
            - name: SHUTDOWN_DRAIN_TIMEOUT_MS
              value: "25000"
            - name: SHUTDOWN_HARD_EXIT_MS
              value: "35000"
          startupProbe:
            httpGet:
              path: /livez
              port: http
            periodSeconds: 2
            failureThreshold: 30
          livenessProbe:
            httpGet:
              path: /livez
              port: http
            periodSeconds: 10
            timeoutSeconds: 2
            failureThreshold: 3
          readinessProbe:
            httpGet:
              path: /readyz
              port: http
            periodSeconds: 5
            timeoutSeconds: 2
            failureThreshold: 2
          lifecycle:
            preStop:
              sleep:
                seconds: 5

// src/server-terminus.js
import { createTerminus } from '@godaddy/terminus';
import mongoose from 'mongoose';
import { createServer } from 'node:http';
import express from 'express';

const app = express();
app.get('/books', (req, res) => {
  res.json([]);
});

const server = createServer(app);

await mongoose.connect(process.env.MONGODB_URI ?? 'mongodb://127.0.0.1:27017/bookshelf');

createTerminus(server, {
  signal: 'SIGTERM',
  signals: ['SIGINT'],
  healthChecks: {
    '/readyz': async () => {
      await mongoose.connection.db.command({ ping: 1 });
    },
    '/livez': async () => {},
  },
  timeout: 25_000,
  // Give the load balancer time to notice that the instance is going away.
  beforeShutdown: () => new Promise((resolve) => setTimeout(resolve, 5000)),
  onSignal: async () => {
    await mongoose.connection.close();
  },
  onShutdown: async () => {
    console.log('Cleanup finished, server is shutting down');
  },
});

server.listen(Number(process.env.PORT ?? 3000));

// test/health.test.js
import assert from 'node:assert/strict';
import { after, before, describe, test } from 'node:test';
import express from 'express';
import { createHealthRouter } from '../src/health.js';

function listen(app) {
  return new Promise((resolve, reject) => {
    const server = app.listen(0, '127.0.0.1', (error) => (error ? reject(error) : resolve(server)));
  });
}

describe('health endpoints', () => {
  const state = { shuttingDown: false };
  let databaseUp = true;
  let server;
  let base;

  before(async () => {
    const app = express();
    app.use(
      createHealthRouter({
        state,
        checks: {
          database: async () => {
            if (!databaseUp) throw new Error('down');
          },
        },
      }),
    );
    server = await listen(app);
    base = `http://127.0.0.1:${server.address().port}`;
  });

  after(() => {
    server.close();
    // fetch keeps connections alive; close them so the test process can exit on every Node.js version.
    server.closeAllConnections();
  });

  test('liveness is 200 and does not depend on dependencies', async () => {
    databaseUp = false;
    const response = await fetch(`${base}/livez`);
    assert.equal(response.status, 200);
    assert.equal(response.headers.get('cache-control'), 'no-store');
    databaseUp = true;
  });

  test('readiness is 200 when the dependencies are up', async () => {
    const response = await fetch(`${base}/readyz`);
    assert.equal(response.status, 200);
    assert.deepEqual(await response.json(), { status: 'ready', dependencies: { database: 'up' } });
  });

  test('readiness is 503 when a dependency is down', async () => {
    databaseUp = false;
    const response = await fetch(`${base}/readyz`);
    assert.equal(response.status, 503);
    assert.equal((await response.json()).dependencies.database, 'down');
    databaseUp = true;
  });

  test('readiness is 503 while shutting down', async () => {
    state.shuttingDown = true;
    const response = await fetch(`${base}/readyz`);
    assert.equal(response.status, 503);
    state.shuttingDown = false;
  });
});

// test/shutdown.test.js
import assert from 'node:assert/strict';
import { spawn } from 'node:child_process';
import { once } from 'node:events';
import { test } from 'node:test';
import { setTimeout as sleep } from 'node:timers/promises';

test('SIGTERM drains the in-flight request and exits with 0', async () => {
  const child = spawn(process.execPath, ['test/fixtures/slow-server.js'], {
    env: { ...process.env, PORT: '3456', SHUTDOWN_READINESS_DELAY_MS: '100' },
    stdio: ['ignore', 'pipe', 'inherit'],
  });
  await once(child.stdout, 'data'); // the fixture prints a line once it is listening

  const inFlight = fetch('http://127.0.0.1:3456/slow'); // the fixture answers after about one second
  await sleep(200);
  child.kill('SIGTERM');

  const response = await inFlight;
  assert.equal(response.status, 200);

  const [code] = await once(child, 'exit');
  assert.equal(code, 0);
});