Health Checks & Observability

npm install @nestjs/terminus
# only for HttpHealthIndicator:
npm install @nestjs/axios axios

import { Module } from '@nestjs/common';
import { ConfigModule, ConfigService } from '@nestjs/config';
import { TerminusModule } from '@nestjs/terminus';
import { BookCatalogHealthIndicator } from './book-catalog.health.js';
import { CacheHealthIndicator } from './cache.health.js';
import { ExternalController } from './external.controller.js';
import { HealthState } from './health.state.js';
import { LivenessController } from './liveness.controller.js';
import { ReadinessController } from './readiness.controller.js';
import { SearchIndexHealthIndicator } from './search-index.health.js';
import { StartupController } from './startup.controller.js';

@Module({
  imports: [
    ConfigModule,
    TerminusModule.forRootAsync({
      imports: [ConfigModule],
      inject: [ConfigService],
      useFactory: (config: ConfigService) => ({
        errorLogStyle: 'json',
        // Longer than the readiness probe period; see "Graceful shutdown".
        gracefulShutdownTimeoutMs: config.get<number>('SHUTDOWN_DELAY_MS', 5000),
      }),
    }),
  ],
  controllers: [LivenessController, ReadinessController, StartupController, ExternalController],
  providers: [
    HealthState,
    BookCatalogHealthIndicator,
    SearchIndexHealthIndicator,
    CacheHealthIndicator,
  ],
  exports: [HealthState],
})
export class HealthModule {}

import { Controller, Get } from '@nestjs/common';
import { HealthCheck, HealthCheckService } from '@nestjs/terminus';

// Liveness: "is this process able to make progress?"  No dependencies in here:
// a failing database must never make Kubernetes restart the pods.
@Controller('health/live')
export class LivenessController {
  constructor(private readonly health: HealthCheckService) {}

  @Get()
  @HealthCheck()
  live() {
    return this.health.check([]);
  }
}

curl -i http://localhost:3000/health/live

HTTP/1.1 200 OK
Cache-Control: no-cache, no-store, must-revalidate
Content-Type: application/json; charset=utf-8

{ "status": "ok", "info": {}, "error": {}, "details": {} }

import { Controller, Get } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Transport } from '@nestjs/microservices';
import {
  HealthCheck,
  HealthCheckService,
  HttpHealthIndicator,
  MicroserviceHealthIndicator,
} from '@nestjs/terminus';

// Built-in indicators that talk to other processes.
@Controller('health/external')
export class ExternalController {
  constructor(
    private readonly health: HealthCheckService,
    private readonly http: HttpHealthIndicator,
    private readonly microservice: MicroserviceHealthIndicator,
    private readonly config: ConfigService,
  ) {}

  @Get()
  @HealthCheck()
  check() {
    const upstream = this.config.getOrThrow<string>('UPSTREAM_URL');
    return this.health.check([
      // 2xx/3xx means up
      () => this.http.pingCheck('upstream', `${upstream}/ping`),
      // you decide what "healthy" means: here only 204 counts
      () =>
        this.http.responseCheck(
          'upstream-204',
          `${upstream}/no-content`,
          (res) => res.status === 204,
        ),
      // a ClientProxy is created, sent a ping and closed again
      () =>
        this.microservice.pingCheck('notifications', {
          transport: Transport.TCP,
          options: {
            host: this.config.get('NOTIFICATIONS_HOST', '127.0.0.1'),
            port: this.config.getOrThrow<number>('NOTIFICATIONS_PORT'),
          },
          timeout: 500,
        }),
    ]);
  }
}

{
  "status": "error",
  "info": {
    "upstream": { "status": "up" },
    "notifications": { "responseTime": 10, "status": "up" }
  },
  "error": {
    "upstream-204": { "status": "down" }
  },
  "details": {
    "upstream": { "status": "up" },
    "notifications": { "responseTime": 10, "status": "up" },
    "upstream-204": { "status": "down" }
  }
}

() => this.http.pingCheck('legacy-api', `${upstream}/status`, { validateStatus: (status) => status < 500 }),

import { Controller, Get, Module } from '@nestjs/common';
import { InjectDataSource, TypeOrmModule } from '@nestjs/typeorm';
import { HealthCheck, HealthCheckService, TerminusModule, TypeOrmHealthIndicator, MongooseHealthIndicator, DiskHealthIndicator } from '@nestjs/terminus';
import type { DataSource } from 'typeorm';

// Two TypeORM connections: pass each DataSource through the `connection` option.
@Controller('health/databases')
export class DatabasesHealthController {
  constructor(
    private readonly health: HealthCheckService,
    private readonly db: TypeOrmHealthIndicator,
    @InjectDataSource('reporting') private readonly reporting: DataSource,
    @InjectDataSource() private readonly primary: DataSource,
  ) {}

  @Get()
  @HealthCheck()
  check() {
    return this.health.check([
      () => this.db.pingCheck('primary', { connection: this.primary }).withTimeout(1000),
      () => this.db.pingCheck('reporting', { connection: this.reporting }).withTimeout(1000),
    ]);
  }
}

@Module({
  imports: [
    TerminusModule,
    TypeOrmModule.forRoot({ type: 'better-sqlite3', database: ':memory:' }),
    TypeOrmModule.forRoot({ name: 'reporting', type: 'better-sqlite3', database: ':memory:' }),
  ],
  controllers: [DatabasesHealthController],
})
export class DatabasesHealthModule {}

// Needs a MongoDB connection to answer: Mongoose and fixed-size disk checks.
@Controller('health/mongo')
export class MongoHealthController {
  constructor(
    private readonly health: HealthCheckService,
    private readonly mongo: MongooseHealthIndicator,
    private readonly disk: DiskHealthIndicator,
  ) {}

  @Get()
  @HealthCheck()
  check() {
    return this.health.check([
      () => this.mongo.pingCheck('activity-feed').withTimeout(1000),
      // absolute threshold: down when more than 10 GiB are used on /var/lib/data
      () => this.disk.checkStorage('data-volume', { path: '/var/lib/data', threshold: 10 * 1024 ** 3 }),
    ]);
  }
}

import { Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { HealthIndicatorService } from '@nestjs/terminus';

// attempt(): run an operation; it resolves -> up, it throws -> down.
// The AbortSignal is aborted when withTimeout() expires, so pass it on.
@Injectable()
export class BookCatalogHealthIndicator {
  constructor(
    private readonly healthIndicatorService: HealthIndicatorService,
    private readonly config: ConfigService,
  ) {}

  isHealthy(key: string) {
    const url = this.config.getOrThrow<string>('CATALOG_URL');
    return this.healthIndicatorService
      .check(key)
      .attempt(async ({ signal }) => {
        const response = await fetch(`${url}/books/count`, { signal });
        if (!response.ok) {
          throw new Error(`catalog answered ${response.status}`);
        }
        const { count } = (await response.json()) as { count: number };
        return { books: count }; // appended to the indicator result
      })
      .withTimeout(1000);
  }
}

import { Injectable } from '@nestjs/common';
import { HealthIndicatorService } from '@nestjs/terminus';

export interface SearchIndexStats {
  documents: number;
  staleDocuments: number;
}

export const SEARCH_INDEX = Symbol('SEARCH_INDEX');

// up()/down(): you decide the state yourself; nothing is thrown.
@Injectable()
export class SearchIndexHealthIndicator {
  constructor(private readonly healthIndicatorService: HealthIndicatorService) {}

  async isHealthy(key: string, stats: () => Promise<SearchIndexStats>) {
    const indicator = this.healthIndicatorService.check(key);
    const { documents, staleDocuments } = await stats();

    if (documents === 0) {
      return indicator.down('search index is empty'); // string -> { message }
    }
    if (staleDocuments > 100) {
      return indicator.down({ staleDocuments }); // object -> merged into the result
    }
    return indicator.up({ documents });
  }
}

import { Controller, Get } from '@nestjs/common';
import {
  DiskHealthIndicator,
  HealthCheck,
  HealthCheckService,
  MemoryHealthIndicator,
  TypeOrmHealthIndicator,
} from '@nestjs/terminus';
import { BookCatalogHealthIndicator } from './book-catalog.health.js';
import { CacheHealthIndicator } from './cache.health.js';
import { HealthState } from './health.state.js';
import { SearchIndexHealthIndicator } from './search-index.health.js';

// Readiness: "should this instance receive traffic right now?"
@Controller('health/ready')
export class ReadinessController {
  constructor(
    private readonly health: HealthCheckService,
    private readonly db: TypeOrmHealthIndicator,
    private readonly memory: MemoryHealthIndicator,
    private readonly disk: DiskHealthIndicator,
    private readonly catalog: BookCatalogHealthIndicator,
    private readonly search: SearchIndexHealthIndicator,
    private readonly cache: CacheHealthIndicator,
    private readonly state: HealthState,
  ) {}

  @Get()
  @HealthCheck()
  ready() {
    return this.health.check([
      // an attempt can be passed directly or wrapped in a function
      () => this.db.pingCheck('database').withTimeout(1500).cacheFor(5000),
      () => this.catalog.isHealthy('catalog'),
      () =>
        this.search.isHealthy('search', async () => ({
          documents: this.state.searchDocuments,
          staleDocuments: this.state.staleDocuments,
        })),
      () => this.cache.isHealthy('cache', async () => this.state.cacheConnected),
      () => this.memory.checkHeap('memory_heap', 512 * 1024 * 1024),
      () => this.disk.checkStorage('storage', { path: '/', thresholdPercent: 0.98 }),
    ]);
  }
}

{
  "status": "ok",
  "error": {},
  "details": {
    "database": { "responseTime": 5, "status": "up" },
    "catalog": { "books": 3, "responseTime": 9, "status": "up" },
    "search": { "documents": 42, "status": "up" },
    "cache": { "status": "up" },
    "memory_heap": { "status": "up" },
    "storage": { "status": "up" }
  }
}

import { Controller, Get } from '@nestjs/common';
import { HealthCheck, HealthCheckService, HealthIndicatorService } from '@nestjs/terminus';
import { HealthState } from './health.state.js';

// Startup probe: "has the application finished warming up?"
@Controller('health/startup')
export class StartupController {
  constructor(
    private readonly health: HealthCheckService,
    private readonly indicators: HealthIndicatorService,
    private readonly state: HealthState,
  ) {}

  @Get()
  @HealthCheck()
  startup() {
    return this.health.check([
      async () => {
        const indicator = this.indicators.check('warmup');
        return this.state.warmedUp ? indicator.up() : indicator.down('still warming up');
      },
    ]);
  }
}

apiVersion: apps/v1
kind: Deployment
metadata:
  name: bookshelf-api
spec:
  replicas: 3
  selector:
    matchLabels:
      app: bookshelf-api
  template:
    metadata:
      labels:
        app: bookshelf-api
    spec:
      terminationGracePeriodSeconds: 30
      containers:
        - name: api
          image: registry.example.com/bookshelf-api:1.0.0
          ports:
            - containerPort: 3000
          env:
            - name: SHUTDOWN_DELAY_MS
              value: "12000"
            - name: OTEL_SERVICE_NAME
              value: bookshelf-api
            - name: OTEL_EXPORTER_OTLP_ENDPOINT
              value: http://otel-collector:4318
          startupProbe:
            httpGet: { path: /health/startup, port: 3000 }
            periodSeconds: 2
            failureThreshold: 30
          livenessProbe:
            httpGet: { path: /health/live, port: 3000 }
            periodSeconds: 10
            timeoutSeconds: 2
            failureThreshold: 3
          readinessProbe:
            httpGet: { path: /health/ready, port: 3000 }
            periodSeconds: 5
            timeoutSeconds: 3
            failureThreshold: 2

const app = await NestFactory.create(AppModule);
app.enableShutdownHooks(); // without this, Terminus never sees SIGTERM
await app.listen(3000);

before signal : 200 ok
after SIGTERM (+300 ms): 503 shutting_down
process exited after 1509 ms (code=null, signal=SIGTERM)
LOG Received termination signal SIGTERM
LOG Awaiting 1500ms before shutdown
LOG Timeout reached, shutting down now

// A custom logger for Terminus only. It extends ConsoleLogger, so you override
// just the parts you need. Transient scope gives Terminus its own instance.
@Injectable({ scope: Scope.TRANSIENT })
export class TerminusLogger extends ConsoleLogger {
  error(message: any, stack?: string, context?: string): void;
  error(message: any, ...optionalParams: any[]): void;
  error(message: unknown, ...rest: unknown[]): void {
    // e.g. forward to your structured logger; here we just tag the line
    super.error(`[health] ${String(message)}`, ...(rest as []));
  }
}

@Module({
  imports: [TerminusModule.forRoot({ logger: TerminusLogger, errorLogStyle: 'pretty' })],
})
export class PrettyHealthModule {}

@Module({ imports: [TerminusModule.forRoot({ logger: false })] })
export class QuietHealthModule {}

ERROR [HealthCheckService] [health] Health Check has failed!

❌ index
message: index is empty
status: down

== SDK imported after express:
   1   name: 'GET'                      <- only the outgoing client span
== SDK started with --import:
   1   name: 'GET /ping'                <- server span with the route
   2   name: 'request handler - /ping'
   1   name: 'middleware - patched'
   1   name: 'GET'

// Loaded with `node --import ./dist/instrumentation.js dist/main.js`
// so that it runs BEFORE any application module is evaluated.
import { register } from 'node:module';
import { NodeSDK } from '@opentelemetry/sdk-node';
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';

// ESM only: let OpenTelemetry patch modules that are loaded with `import`.
register('@opentelemetry/instrumentation/hook.mjs', import.meta.url);

const sdk = new NodeSDK({
  // Exporter, service name, sampler... come from OTEL_* environment variables
  // (OTEL_SERVICE_NAME, OTEL_TRACES_EXPORTER, OTEL_EXPORTER_OTLP_ENDPOINT, ...).
  instrumentations: [getNodeAutoInstrumentations()],
});
sdk.start();

// Flush pending spans on SIGTERM; Nest's own shutdown hooks run in parallel.
process.once('SIGTERM', () => {
  sdk.shutdown().catch((error: unknown) => console.error('OTel shutdown failed', error));
});

nest build
node --import ./dist/instrumentation.js dist/main.js

OTEL_SERVICE_NAME=bookshelf-api OTEL_TRACES_EXPORTER=console \
  node --import ./dist/instrumentation.js dist/main.js

// node --require ./dist/instrumentation.js dist/main.js
import { NodeSDK } from '@opentelemetry/sdk-node';
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';

const sdk = new NodeSDK({ instrumentations: [getNodeAutoInstrumentations()] });
sdk.start();

// node --import ./dist/instrumentation-otlp.js dist/main.js
import { register } from 'node:module';
import { NodeSDK } from '@opentelemetry/sdk-node';
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base';
import { resourceFromAttributes } from '@opentelemetry/resources';
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';

register('@opentelemetry/instrumentation/hook.mjs', import.meta.url);

const sdk = new NodeSDK({
  resource: resourceFromAttributes({
    [ATTR_SERVICE_NAME]: process.env.OTEL_SERVICE_NAME ?? 'bookshelf-api',
    [ATTR_SERVICE_VERSION]: process.env.npm_package_version ?? '0.0.0',
  }),
  // No url: the exporter reads OTEL_EXPORTER_OTLP_ENDPOINT (and appends /v1/traces).
  spanProcessors: [new BatchSpanProcessor(new OTLPTraceExporter())],
  // Metrics are pulled by Prometheus from http://localhost:9464/metrics.
  metricReaders: [new PrometheusExporter({ port: Number(process.env.OTEL_PROMETHEUS_PORT ?? 9464) })],
  instrumentations: [
    getNodeAutoInstrumentations({
      // Very chatty and rarely useful for an HTTP API:
      '@opentelemetry/instrumentation-fs': { enabled: false },
      '@opentelemetry/instrumentation-net': { enabled: false },
      '@opentelemetry/instrumentation-dns': { enabled: false },
      // Probes and scrapes are the most frequent and least interesting requests.
      '@opentelemetry/instrumentation-http': {
        ignoreIncomingRequestHook: (request) => /^\/(health|metrics)/.test(request.url ?? ''),
      },
    }),
  ],
});
sdk.start();

process.once('SIGTERM', () => {
  sdk.shutdown().catch((error: unknown) => console.error('OTel shutdown failed', error));
});

[@opentelemetry/instrumentation-http]      GET /books/:id   (server span, one per request)
[@opentelemetry/instrumentation-express]   request handler - /books/:id, middleware - jsonParser, ...
[@opentelemetry/instrumentation-router]    request handler - /books/:id, middleware - patched
[@opentelemetry/instrumentation-undici]    GET              (client span of the test's fetch)
[bookshelf-api]                            books.findOne    (manual span, ERROR status for the 404)
[bookshelf-api.nest]                       BooksController.findOne   (interceptor span)

import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nestjs/common';
import { context, SpanStatusCode, trace } from '@opentelemetry/api';
import { Observable, tap } from 'rxjs';

const tracer = trace.getTracer('bookshelf-api.nest');

// A Nest-aware span per route handler: what @opentelemetry/instrumentation-nestjs-core
// provided before it stopped at @nestjs/core 11 (its supported range is <12).
@Injectable()
export class HandlerSpanInterceptor implements NestInterceptor {
  intercept(executionContext: ExecutionContext, next: CallHandler): Observable<unknown> {
    const name = `${executionContext.getClass().name}.${executionContext.getHandler().name}`;
    const span = tracer.startSpan(name, { attributes: { 'nestjs.type': 'handler' } });
    const withSpan = trace.setSpan(context.active(), span);

    return new Observable((subscriber) =>
      // The handler runs on subscription: subscribe inside the span's context.
      context.with(withSpan, () =>
        next
          .handle()
          .pipe(
            tap({
              error: (error: Error) => {
                span.recordException(error);
                span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
              },
              finalize: () => span.end(),
            }),
          )
          .subscribe(subscriber),
      ),
    );
  }
}

import { register } from 'node:module';
import { NodeSDK } from '@opentelemetry/sdk-node';
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
import { FastifyOtelInstrumentation } from '@fastify/otel';
import { InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base';

register('@opentelemetry/instrumentation/hook.mjs', import.meta.url);

const exporter = new InMemorySpanExporter();
(globalThis as { __spans?: InMemorySpanExporter }).__spans = exporter;

new NodeSDK({
  spanProcessors: [new SimpleSpanProcessor(exporter)],
  instrumentations: [
    getNodeAutoInstrumentations({ '@opentelemetry/instrumentation-fs': { enabled: false } }),
    // Registers its plugin on every Fastify instance created after this line.
    new FastifyOtelInstrumentation({ registerOnInitialization: true }),
  ],
}).start();

// With an explicit adapter the application options are the THIRD argument.
const app = await NestFactory.create<NestFastifyApplication>(AppModule, new FastifyAdapter(), { logger: false });
await app.listen(0, '127.0.0.1');

import { Injectable, NotFoundException } from '@nestjs/common';
import { metrics, SpanStatusCode, trace } from '@opentelemetry/api';

// Instrumentation scope: the name identifies YOUR code in the backend.
const tracer = trace.getTracer('bookshelf-api');
const meter = metrics.getMeter('bookshelf-api');

const booksViewed = meter.createCounter('bookshelf.books.viewed', {
  description: 'Number of book detail views',
});
const lookupDuration = meter.createHistogram('bookshelf.books.lookup.duration', {
  description: 'Duration of the catalog lookup',
  unit: 'ms',
});

const catalog = new Map([['42', { id: '42', title: 'Dune' }]]);

@Injectable()
export class BooksService {
  findOne(id: string) {
    // startActiveSpan makes the span the active one inside the callback,
    // so nested spans and logs are linked to it automatically.
    return tracer.startActiveSpan('books.findOne', async (span) => {
      const startedAt = performance.now();
      try {
        span.setAttribute('book.id', id);
        const book = catalog.get(id);
        if (!book) {
          throw new NotFoundException(`Book ${id} not found`);
        }
        booksViewed.add(1, { 'book.found': true });
        return book;
      } catch (error) {
        span.recordException(error as Error);
        span.setStatus({ code: SpanStatusCode.ERROR, message: (error as Error).message });
        booksViewed.add(1, { 'book.found': false });
        throw error;
      } finally {
        lookupDuration.record(performance.now() - startedAt);
        span.end(); // always end the span, or it is never exported
      }
    });
  }
}

bookshelf_books_viewed_total{book_found="true",otel_scope_name="bookshelf-api"} 1
bookshelf_books_viewed_total{book_found="false",otel_scope_name="bookshelf-api"} 1
bookshelf_books_lookup_duration_count{otel_scope_name="bookshelf-api"} 2
bookshelf_books_lookup_duration_sum{otel_scope_name="bookshelf-api"} 0.52

import { ConsoleLogger } from '@nestjs/common';
import { trace } from '@opentelemetry/api';

// Adds trace_id/span_id of the active span to every line, so logs and traces
// can be joined in the backend.
export class OtelConsoleLogger extends ConsoleLogger {
  protected formatMessage(
    logLevel: Parameters<ConsoleLogger['formatMessage']>[0],
    message: unknown,
    pidMessage: string,
    formattedLogLevel: string,
    contextMessage: string,
    timestampDiff: string,
  ): string {
    const spanContext = trace.getActiveSpan()?.spanContext();
    const suffix = spanContext ? ` trace_id=${spanContext.traceId} span_id=${spanContext.spanId}` : '';
    return (
      super.formatMessage(logLevel, message, pidMessage, formattedLogLevel, contextMessage, timestampDiff).trimEnd() +
      suffix +
      '\n'
    );
  }
}

LOG [BooksController] Looking up book 42 trace_id=5ddddd6d505d790674f174f208be8e58 span_id=23951b468e5da102

// For tests and local verification: spans stay in the process.
import { register } from 'node:module';
import { NodeSDK } from '@opentelemetry/sdk-node';
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
import { InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base';

register('@opentelemetry/instrumentation/hook.mjs', import.meta.url);

export const exporter = new InMemorySpanExporter();

const sdk = new NodeSDK({
  spanProcessors: [new SimpleSpanProcessor(exporter)],
  instrumentations: [getNodeAutoInstrumentations({ '@opentelemetry/instrumentation-fs': { enabled: false } })],
});
sdk.start();

// Expose the exporter to code that runs later (the --import module is evaluated once).
(globalThis as { __spans?: InMemorySpanExporter }).__spans = exporter;

@opentelemetry/instrumentation-net  | tcp.connect
bookshelf-api                       | books.findOne
bookshelf-api.nest                  | BooksController.findOne
@opentelemetry/instrumentation-http | GET
@opentelemetry/instrumentation-undici | GET

npm install @prometheus-io/client

import { Injectable } from '@nestjs/common';
import { Counter, Histogram, Registry, collectDefaultMetrics } from '@prometheus-io/client';

@Injectable()
export class MetricsService {
  // A dedicated registry instead of the global one: no cross-test leakage.
  readonly registry = new Registry();

  readonly httpDuration = new Histogram({
    name: 'http_server_request_duration_seconds',
    help: 'Duration of HTTP requests handled by Nest controllers',
    labelNames: ['method', 'route', 'status_code'] as const,
    buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
    registers: [this.registry],
  });

  readonly booksCreated = new Counter({
    name: 'bookshelf_books_created_total',
    help: 'Books created through the API',
    registers: [this.registry],
  });

  constructor() {
    // process_cpu_*, nodejs_eventloop_lag_*, nodejs_heap_size_*, ...
    collectDefaultMetrics({ register: this.registry });
  }
}

import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nestjs/common';
import { Observable } from 'rxjs';
import { MetricsService } from './metrics.service.js';

// Works on Express and Fastify: only the way to read the matched route differs.
@Injectable()
export class HttpMetricsInterceptor implements NestInterceptor {
  constructor(private readonly metrics: MetricsService) {}

  intercept(context: ExecutionContext, next: CallHandler): Observable<unknown> {
    if (context.getType() !== 'http') {
      return next.handle();
    }
    const http = context.switchToHttp();
    const request = http.getRequest();
    const response = http.getResponse();
    const raw = response.raw ?? response; // Fastify reply -> Node ServerResponse
    const end = this.metrics.httpDuration.startTimer();

    // 'finish' fires after exception filters ran, so the status code is final.
    raw.once('finish', () => {
      end({
        method: request.method,
        // Route PATTERN, never the URL: /books/42 must not become a label value.
        route: request.route?.path ?? request.routeOptions?.url ?? 'unmatched',
        status_code: String(raw.statusCode),
      });
    });
    return next.handle();
  }
}

import { Controller, Get, Header } from '@nestjs/common';
import { MetricsService } from './metrics.service.js';

@Controller('metrics')
export class MetricsController {
  constructor(private readonly metrics: MetricsService) {}

  @Get()
  @Header('Content-Type', 'text/plain; version=0.0.4; charset=utf-8')
  scrape(): Promise<string> {
    return this.metrics.registry.metrics();
  }
}

import { Global, Module } from '@nestjs/common';
import { APP_INTERCEPTOR } from '@nestjs/core';
import { HttpMetricsInterceptor } from './http-metrics.interceptor.js';
import { MetricsController } from './metrics.controller.js';
import { MetricsService } from './metrics.service.js';

@Global()
@Module({
  controllers: [MetricsController],
  providers: [MetricsService, { provide: APP_INTERCEPTOR, useClass: HttpMetricsInterceptor }],
  exports: [MetricsService],
})
export class MetricsModule {}

# TYPE http_server_request_duration_seconds histogram
http_server_request_duration_seconds_count{method="GET",route="/books/:id",status_code="200"} 2
http_server_request_duration_seconds_count{method="GET",route="/books/:id",status_code="404"} 1
http_server_request_duration_seconds_count{method="POST",route="/books",status_code="201"} 1
# TYPE bookshelf_books_created_total counter
bookshelf_books_created_total 1
# TYPE nodejs_eventloop_lag_seconds gauge

import { Controller, Get, Inject, Module } from '@nestjs/common';
import {
  InjectMetric,
  PrometheusModule,
  makeCounterProvider,
  makeHistogramProvider,
} from '@willsoto/nestjs-prometheus';
import type { Counter, Histogram } from 'prom-client';

@Controller('books')
export class WillsotoBooksController {
  constructor(
    @InjectMetric('bookshelf_books_viewed_total') private readonly viewed: Counter<string>,
    @InjectMetric('bookshelf_lookup_seconds') private readonly lookup: Histogram<string>,
  ) {}

  @Get()
  list() {
    const end = this.lookup.startTimer();
    this.viewed.inc();
    end();
    return [];
  }
}

@Module({
  imports: [
    PrometheusModule.register({
      path: '/metrics',
      defaultMetrics: { enabled: true },
      defaultLabels: { app: 'bookshelf-api' },
    }),
  ],
  controllers: [WillsotoBooksController],
  providers: [
    makeCounterProvider({ name: 'bookshelf_books_viewed_total', help: 'Book list views' }),
    makeHistogramProvider({ name: 'bookshelf_lookup_seconds', help: 'Catalog lookup time', buckets: [0.01, 0.1, 1] }),
  ],
})
export class WillsotoModule {}

global:
  scrape_interval: 15s

scrape_configs:
  - job_name: bookshelf-api
    metrics_path: /metrics
    static_configs:
      - targets: ['bookshelf-api:3000']
  - job_name: otel-collector
    static_configs:
      - targets: ['otel-collector:8889']

# requests per second by route
sum by (route) (rate(http_server_request_duration_seconds_count[5m]))

# share of 5xx answers
sum(rate(http_server_request_duration_seconds_count{status_code=~"5.."}[5m]))
  / sum(rate(http_server_request_duration_seconds_count[5m]))

# 95th percentile latency per route
histogram_quantile(0.95,
  sum by (le, route) (rate(http_server_request_duration_seconds_bucket[5m])))

# event loop lag (default metric): is the process CPU-bound?
nodejs_eventloop_lag_seconds

npm install @nestjs/observe

import { Module } from '@nestjs/common';
import { ConfigModule, ConfigService } from '@nestjs/config';
import { createObserveModule } from '@nestjs/observe';
import { BooksController } from './books.controller.js';
import { BooksService } from './books.service.js';

// createObserveModule() returns a matched pair bound to the same options:
// the module for `imports` and the hook for NestFactory.create({ instrument }).
export const { ObserveModule, ObserveInstrument } = createObserveModule({
  // Options that must be known before the application is created live here.
  traceIdKey: 'traceId',
  attachTraceIdToLogs: true,
  sourceContext: { linesOfContext: 5, maxFrames: 5 },
});

@Module({
  imports: [
    ConfigModule.forRoot({ isGlobal: true }),
    ObserveModule.forRootAsync({
      imports: [ConfigModule],
      inject: [ConfigService],
      useFactory: (config: ConfigService) => ({
        appKey: config.getOrThrow<string>('OBSERVE_APP_KEY'),
        appSecret: config.getOrThrow<string>('OBSERVE_APP_SECRET'),
        serviceId: config.get<string>('SERVICE_ID', 'bookshelf-api'),
        serviceVersion: config.get<string>('GIT_SHA'),
        // Keep health probes and metric scrapes out of the telemetry (and the bill).
        http: { ignore: ['/health', '/health/live', '/health/ready', '/metrics'] },
        tracesSampleRate: 1.0,
        flushInterval: 1000, // ms; values below 1000 are raised to 1000
      }),
    }),
  ],
  controllers: [BooksController],
  providers: [BooksService],
})
export class AppModule {}

import { NestFactory } from '@nestjs/core';
import { AppModule, ObserveInstrument } from './app.module.js';

const app = await NestFactory.create(AppModule, { instrument: ObserveInstrument });
app.enableShutdownHooks();

// The adapter is the 2nd argument, so the application options (and `instrument`) are the 3rd.
const app = await NestFactory.create<NestFastifyApplication>(AppModule, new FastifyAdapter(), {
  instrument: ObserveInstrument,
});

import { Injectable, NotFoundException } from '@nestjs/common';
import { TracerService } from '@nestjs/observe';

// Typing the request-scoped store makes setAttribute()/getAttribute() keys checked.
type RequestStore = { lastBook: string }; // a type alias: an interface lacks the index signature the constraint wants

@Injectable()
export class BooksService {
  private readonly books = new Map([['42', { id: '42', title: 'Dune' }]]);

  constructor(private readonly tracer: TracerService<RequestStore>) {}

  async findOne(id: string) {
    this.tracer.counter('bookshelf.books.lookups', { description: 'Number of book lookups' }).increment();

    return this.tracer.createSpan('books.findOne', async (span) => {
      span.addTags({ bookId: id });
      const book = this.books.get(id);
      if (!book) {
        // Handled here, so it would not be captured automatically if we recovered from it
        await this.tracer.captureError(new Error(`Book ${id} not found`), { bookId: id });
        throw new NotFoundException(`Book ${id} not found`);
      }
      this.tracer.setAttribute('lastBook', book.title);
      return book;
    });
  }

  traceId() {
    return this.tracer.currentTraceId();
  }
}

LOG [BooksController] Looking up book 42 (trace 01a0fda1-e689-7660-ae84-efa78cd263d7)   Trace ID: 01a0fda1-e689-7660-ae84-efa78cd263d7