Load Testing and Capacity Planning

import { WebSocket } from 'k6/websockets';
import { Trend, Counter } from 'k6/metrics';
import { check } from 'k6';

const ackRtt = new Trend('chatterbox_ack_rtt', true);
const fanout = new Trend('chatterbox_fanout_latency', true);
const refused = new Counter('chatterbox_handshake_failed');

const TARGET = __ENV.TARGET || 'ws://host.docker.internal:18801/ws';
const ROOMS = Number(__ENV.ROOMS || 5);
const HOLD_MS = Number(__ENV.HOLD_MS || 20000);

export const options = {
  scenarios: {
    chat: {
      executor: 'ramping-vus',
      startVUs: 0,
      stages: [
        { duration: '10s', target: Number(__ENV.VUS || 50) },
        { duration: '20s', target: Number(__ENV.VUS || 50) },
        { duration: '5s', target: 0 },
      ],
      gracefulRampDown: '30s',
    },
  },
  thresholds: {
    chatterbox_ack_rtt: ['p(95)<250'],
    chatterbox_fanout_latency: ['p(99)<500'],
    chatterbox_handshake_failed: ['count==0'],
    checks: ['rate>0.99'],
  },
};

let n = 0;
const id = () => `${__VU}-${__ITER}-${++n}`.padEnd(26, '0');

export default function () {
  const room = `load-${__VU % ROOMS}`;
  const pending = new Map(); // envelope id -> send time
  const ws = new WebSocket(TARGET, ['chatterbox.v1']);
  let timer;

  const send = (type, roomName, payload, extra = {}) => {
    const envId = id();
    pending.set(envId, Date.now());
    ws.send(JSON.stringify({ v: 1, type, id: envId, room: roomName, ts: 0, payload, ...extra }));
    return envId;
  };

  ws.addEventListener('open', () => {
    send('hello', null, { client: 'k6/1.0' });
  });

  ws.addEventListener('message', (e) => {
    const m = JSON.parse(e.data);
    if (m.type === 'hello') {
      send('join', room, { since: null });
    } else if (m.type === 'ack') {
      const sent = pending.get(m.payload.ref);
      if (sent) {
        ackRtt.add(Date.now() - sent);
        pending.delete(m.payload.ref);
      }
      check(m, { 'ack ok': (x) => x.payload.ok === true });
      if (!timer && m.room === room && m.payload.result.members) {
        // joined: start the chat loop with a random 1.5-2.5 s think time
        timer = setInterval(() => {
          send('message', room, { text: `t=${Date.now()};vu=${__VU}`, replyTo: null, attachments: [] });
        }, 1500 + Math.random() * 1000);
      }
    } else if (m.type === 'message') {
      const t = Number(/t=(\d+)/.exec(m.payload.text)[1]);
      fanout.add(Date.now() - t);
    } else if (m.type === 'error') {
      check(m, { 'no protocol error': () => false });
    }
  });

  ws.addEventListener('error', () => refused.add(1));
  ws.addEventListener('close', () => clearInterval(timer));

  setTimeout(() => ws.close(1000), HOLD_MS);
}

docker run --rm --add-host=host.docker.internal:host-gateway \
  -v "$PWD":/scripts:ro \
  -e VUS=500 -e ROOMS=50 -e HOLD_MS=40000 \
  grafana/k6 run /scripts/chatterbox-load.js

chatterbox_ack_rtt............: avg=1.18ms min=0s  med=1ms  max=5ms  p(90)=2ms  p(95)=2ms
chatterbox_fanout_latency.....: avg=1.25ms min=0s  med=1ms  max=5ms  p(90)=2ms  p(95)=2ms
ws_connecting.................: avg=1.93ms min=795µs med=1.77ms max=4.72ms
ws_msgs_received..............: 1958   39.1/s
ws_msgs_sent..................: 518    10.4/s
ws_sessions...................: 40     0.8/s

export const options = {
  scenarios: {
    arrivals: {
      executor: 'ramping-arrival-rate',   // open model: new connections start at a set rate
      startRate: 2,
      timeUnit: '1s',
      stages: [
        { target: 20, duration: '10s' },
        { target: 20, duration: '10s' },
      ],
      preAllocatedVUs: 100,
      maxVUs: 300,                        // each live connection holds one VU
    },
  },
  // thresholds as in the previous script
};

import { WebSocket } from 'k6/websockets';
import { Trend } from 'k6/metrics';

const ackRtt = new Trend('socketio_ack_rtt', true);
const BASE = __ENV.TARGET || 'ws://host.docker.internal:18801';

export const options = { vus: 5, duration: '15s' };

export default function () {
  // Engine.IO v4 over a raw WebSocket: no long-polling, so no sticky-session need on the first request
  const ws = new WebSocket(`${BASE}/socket.io/?EIO=4&transport=websocket`);
  let sentAt = 0;

  ws.addEventListener('message', (e) => {
    const frame = e.data;
    if (frame.startsWith('0')) {
      ws.send('40');                       // Socket.IO CONNECT to the main namespace
    } else if (frame.startsWith('2')) {
      ws.send('3');                        // Engine.IO ping -> pong
    } else if (frame.startsWith('40')) {
      ws.send('421["join","load-1"]');     // EVENT with ack id 1
    } else if (frame.startsWith('431')) {  // ACK for id 1
      sentAt = Date.now();
      ws.send('422["message","load-1","hello"]');
    } else if (frame.startsWith('432')) {
      ackRtt.add(Date.now() - sentAt);
    }
  });
  setTimeout(() => ws.close(1000), 10000);
}

<!doctype html>
<meta charset="utf-8">
<title>Huddle loopback stand-in</title>
<video id="remote" autoplay muted playsinline></video>
<script>
  // Stand-in for a real Huddle room page: one local peer pair publishes the fake camera and receives it.
  // A real test points the harness at the application's own URL instead and reads its getStats().
  let receiver;
  async function startCall() {
    const sender = new RTCPeerConnection();
    receiver = new RTCPeerConnection();
    sender.onicecandidate = (e) => e.candidate && receiver.addIceCandidate(e.candidate);
    receiver.onicecandidate = (e) => e.candidate && sender.addIceCandidate(e.candidate);
    receiver.ontrack = (e) => { document.getElementById('remote').srcObject = e.streams[0]; };
    const media = await navigator.mediaDevices.getUserMedia({ video: { width: 640, height: 360 }, audio: true });
    media.getTracks().forEach((t) => sender.addTrack(t, media));
    await sender.setLocalDescription();
    await receiver.setRemoteDescription(sender.localDescription);
    await receiver.setLocalDescription();
    await sender.setRemoteDescription(receiver.localDescription);
  }
  async function getHuddleStats() {
    const out = { framesDecoded: 0, packetsLost: 0, packetsReceived: 0, jitterMs: 0, bytesReceived: 0 };
    for (const r of (await receiver.getStats()).values()) {
      if (r.type === 'inbound-rtp' && r.kind === 'video') {
        out.framesDecoded = r.framesDecoded;
        out.packetsLost = r.packetsLost;
        out.packetsReceived = r.packetsReceived;
        out.jitterMs = Math.round((r.jitter ?? 0) * 1000);
        out.bytesReceived = r.bytesReceived;
      }
    }
    return out;
  }
</script>

import { chromium } from 'playwright-core';
import { createServer } from 'node:http';
import { readFile } from 'node:fs/promises';

const CLIENTS = Number(process.env.CLIENTS ?? 4);
const SECONDS = Number(process.env.SECONDS ?? 10);
const PORT = Number(process.env.PORT ?? 18803);
const PAGE = process.env.PAGE_URL ?? `http://localhost:${PORT}/loopback.html`;

// Serve the stand-in page (unused when PAGE_URL points at a real application).
const server = createServer(async (req, res) => {
  res.setHeader('content-type', 'text/html');
  res.end(await readFile(new URL('./loopback.html', import.meta.url)));
}).listen(PORT);

const browser = await chromium.launch({
  executablePath: process.env.CHROME_PATH,           // or channel: 'chrome'
  headless: true,
  args: [
    '--use-fake-device-for-media-stream',            // synthetic camera and microphone
    '--use-fake-ui-for-media-stream',                // auto-accept the permission prompt
    '--autoplay-policy=no-user-gesture-required',
  ],
});

const pages = [];
for (let i = 0; i < CLIENTS; i++) {
  const context = await browser.newContext();        // one context per simulated participant
  const page = await context.newPage();
  page.on('pageerror', (e) => console.error(`client ${i}:`, e.message));
  await page.goto(PAGE);
  await page.evaluate(() => startCall());
  pages.push(page);
}

await new Promise((r) => setTimeout(r, SECONDS * 1000));

const rows = [];
for (const [i, page] of pages.entries()) {
  rows.push({ client: i, ...(await page.evaluate(() => getHuddleStats())) });
}
console.table(rows);
const dead = rows.filter((r) => r.framesDecoded < SECONDS * 10);   // expect roughly 30 fps; accept 10
console.log(dead.length === 0 ? 'all clients decoding' : `${dead.length} client(s) below threshold`);

await browser.close();
server.close();
process.exit(dead.length === 0 ? 0 : 1);

npm install playwright-core@1.64.1
CLIENTS=4 SECONDS=8 node webrtc-load.mjs