Visual Regression Testing

Error: A snapshot doesn't exist at /work/t/default-names.spec.ts-snapshots/default-names-1-linux.webp, writing actual.
Error: A snapshot doesn't exist at /work/t/default-names.spec.ts-snapshots/landing-linux.png, writing actual.

import { defineConfig, devices } from '@playwright/test';

export default defineConfig({
  testDir: './tests',
  // One name per browser project, no platform suffix: every baseline comes from the same Docker image.
  snapshotPathTemplate: '{testDir}/__screenshots__/{testFilePath}/{projectName}/{arg}{ext}',
  expect: {
    toHaveScreenshot: {
      maxDiffPixelRatio: 0.01,
      animations: 'disabled',
      caret: 'hide',
      scale: 'css',
    },
    toMatchSnapshot: { maxDiffPixelRatio: 0.01 },
  },
  use: {
    baseURL: process.env.BASE_URL ?? 'http://localhost:3963',
    testIdAttribute: 'data-testid',
    viewport: { width: 1280, height: 720 },
    locale: 'en-US',
    timezoneId: 'UTC',
  },
  projects: [
    { name: 'chromium', use: { ...devices['Desktop Chrome'], viewport: { width: 1280, height: 720 } } },
    { name: 'firefox', use: { ...devices['Desktop Firefox'], viewport: { width: 1280, height: 720 } } },
    { name: 'webkit', use: { ...devices['Desktop Safari'], viewport: { width: 1280, height: 720 } } },
  ],
});

import { test, expect } from './fixtures';

test('every region and format option', async ({ page }) => {
  await page.goto('/books');
  await expect(page.getByTestId('book-list')).toBeVisible();

  // Only a rectangle of the page, in CSS pixels.
  await expect(page).toHaveScreenshot('clip-header.png', { clip: { x: 0, y: 0, width: 1280, height: 64 } });

  // Whole page, with a per-test tolerance and a longer stabilisation timeout.
  await expect(page).toHaveScreenshot('full.png', { fullPage: true, maxDiffPixels: 100, timeout: 10_000 });

  // Device pixels instead of CSS pixels: the file is larger on high-density screens.
  await expect(page.getByTestId('site-header')).toHaveScreenshot('header-device-scale.png', { scale: 'device' });

  // Name given as path segments.
  await expect(page.getByTestId('pagination')).toHaveScreenshot(['pagination', 'bar.png']);
});

test('transparent background', async ({ page, browserName }) => {
  // Firefox 157 answers "Not implemented" to omitBackground.
  test.skip(browserName === 'firefox', 'omitBackground is not implemented in Firefox');
  await page.goto('/books');
  // The page background is left out of the image (alpha channel) instead of being painted white.
  await expect(page.getByTestId('book-card').first()).toHaveScreenshot('card-transparent.png', { omitBackground: true });
});

import { test, expect } from './fixtures';

test.describe('visual', () => {
  test('home page', async ({ page }) => {
    await page.goto('/');
    await expect(page).toHaveScreenshot('home.png', { fullPage: true });
  });

  test('catalogue', async ({ page }) => {
    await page.goto('/books');
    await expect(page.getByTestId('book-list')).toBeVisible();
    await expect(page).toHaveScreenshot('catalogue.png');
  });

  test('book card element', async ({ page }) => {
    await page.goto('/books');
    await expect(page.getByTestId('book-card').first()).toHaveScreenshot('first-card.png');
  });

  test('dark theme', async ({ page }) => {
    await page.emulateMedia({ colorScheme: 'dark' });
    await page.goto('/books/1');
    await expect(page.getByTestId('book-title')).toBeVisible();
    await expect(page).toHaveScreenshot('book-detail-dark.png');
  });

  test('phone layout', async ({ page }) => {
    await page.setViewportSize({ width: 390, height: 844 });
    await page.goto('/books');
    await expect(page).toHaveScreenshot('catalogue-phone.png', { fullPage: true });
  });
});

import { test, expect } from '@playwright/test';

test('strict compare of the catalogue', async ({ page }) => {
  await page.goto('/books?page=2');
  await expect(page).toHaveScreenshot('catalogue-page-2.png', { threshold: 0, maxDiffPixelRatio: 0 });
});

158645 pixels (ratio 0.18 of all image pixels) are different.

New baselines created for:
  t/default-names.spec.ts
  git apply ../test-results/rebaselines.patch

import { test, expect } from './fixtures';

test.beforeEach(async ({ page }) => {
  // A page that shows the current time, as many dashboards do.
  await page.addInitScript(() => {
    window.addEventListener('DOMContentLoaded', () => {
      const stamp = document.createElement('p');
      stamp.dataset.testid = 'build-stamp';
      stamp.textContent = `Rendered at ${new Date().toISOString()}`;
      document.querySelector('footer')?.append(stamp);
    });
  });
});

test('masked timestamp', async ({ page }) => {
  await page.goto('/books');
  await expect(page).toHaveScreenshot('masked-stamp.png', {
    fullPage: true,
    mask: [page.getByTestId('build-stamp')],
    maskColor: '#ff00ff',
  });
});

test('frozen clock', async ({ page }) => {
  await page.clock.setFixedTime(new Date('2026-10-25T10:00:00Z'));
  await page.goto('/books');
  await expect(page.getByTestId('build-stamp')).toHaveText('Rendered at 2026-10-25T10:00:00.000Z');
  await expect(page).toHaveScreenshot('frozen-clock.png', { fullPage: true });
});

test('style path hides the stamp', async ({ page }) => {
  await page.goto('/books');
  await expect(page).toHaveScreenshot('hidden-stamp.png', { fullPage: true, stylePath: './styles/hide-stamp.css' });
});

test('webp baseline and generic snapshot', async ({ page }) => {
  await page.goto('/books');
  await expect(page).toHaveScreenshot('catalogue.webp');
  const buffer = await page.getByTestId('book-list').screenshot({ animations: 'disabled' });
  expect(buffer).toMatchSnapshot('book-list-buffer.png');
});

docker run --rm --init --ipc=host \
  -e BASE_URL=http://host.docker.internal:3963 \
  -v "$PWD":/work -w /work \
  mcr.microsoft.com/playwright:v1.64.1-noble \
  npx playwright test --update-snapshots

import { test as base, expect } from '@playwright/test';

// Every visual test starts from the same eight books, so catalogue screenshots never depend on what other
// tests (or earlier runs) left in the application.
export const test = base.extend<{ fixedCatalogue: void }>({
  fixedCatalogue: [
    async ({ request }, use) => {
      await request.post('/api/test/seed', {
        data: {
          books: Array.from({ length: 8 }, (_, i) => ({
            title: `Seeded Book ${i + 1}`,
            author: 'Test Author',
            genre: 'Fiction',
            price: 10 + i,
            color: ['#2563eb', '#059669', '#d97706', '#7c3aed'][i % 4],
          })),
          reviews: [],
        },
      });
      await use();
    },
    { auto: true },
  ],
});

export { expect };

/* Applied only while the screenshot is taken. */
[data-testid='build-stamp'] { visibility: hidden !important; }

import functools
import io
import os
from pathlib import Path

from PIL import Image, ImageChops

BASELINES = Path(__file__).parent.parent / "baselines"
ACTUALS = Path(__file__).parent.parent / "actual"


def assert_matches_baseline(png: bytes, name: str, max_diff_ratio: float = 0.01, channel_tolerance: int = 24) -> None:
    """Compare a PNG screenshot with baselines/<name>.png.

    UPDATE_BASELINES=1 rewrites the baseline. A missing baseline is written and the test fails, so it is reviewed.
    """
    baseline = BASELINES / f"{name}.png"
    actual = Image.open(io.BytesIO(png)).convert("RGB")
    if os.environ.get("UPDATE_BASELINES") == "1":
        BASELINES.mkdir(exist_ok=True)
        actual.save(baseline)
        return
    if not baseline.exists():
        BASELINES.mkdir(exist_ok=True)
        actual.save(baseline)
        raise AssertionError(f"{baseline} did not exist and was written; review it and run again")
    expected = Image.open(baseline).convert("RGB")
    assert actual.size == expected.size, f"size differs: expected {expected.size}, got {actual.size}"
    # A pixel differs when any channel moves by more than channel_tolerance (0-255).
    diff = ImageChops.difference(expected, actual)
    changed_mask = functools.reduce(
        ImageChops.lighter, [band.point(lambda v: 255 if v > channel_tolerance else 0) for band in diff.split()])
    ratio = changed_mask.histogram()[255] / (actual.width * actual.height)
    if ratio > max_diff_ratio:
        ACTUALS.mkdir(exist_ok=True)
        actual.save(ACTUALS / f"{name}-actual.png")
        changed_mask.save(ACTUALS / f"{name}-diff.png")
        raise AssertionError(f"{name}: {ratio:.2%} of the pixels differ (limit {max_diff_ratio:.2%})")

import json
import os
import urllib.request

import pytest
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait

from visual import assert_matches_baseline

BASE_URL = os.environ.get("BASE_URL", "http://localhost:3963")
SEED_URL = os.environ.get("SEED_URL", BASE_URL)
GRID = os.environ.get("GRID_URL")  # for example http://localhost:43963


@pytest.fixture(autouse=True)
def fixed_catalogue():
    books = [
        {"title": f"Seeded Book {i + 1}", "author": "Test Author", "genre": "Fiction", "price": 10 + i,
         "color": ["#2563eb", "#059669", "#d97706", "#7c3aed"][i % 4]}
        for i in range(8)
    ]
    request = urllib.request.Request(
        f"{SEED_URL}/api/test/seed", data=json.dumps({"books": books, "reviews": []}).encode(),
        headers={"content-type": "application/json"}, method="POST")
    urllib.request.urlopen(request)


@pytest.fixture
def driver():
    options = webdriver.ChromeOptions()
    options.add_argument("--headless=new")
    options.add_argument("--hide-scrollbars")
    options.add_argument("--force-device-scale-factor=1")
    d = webdriver.Remote(command_executor=GRID, options=options) if GRID else webdriver.Chrome(options=options)
    d.set_window_size(1280, 800)
    # Make the viewport exactly 1280 x 720 whatever the browser chrome takes.
    inner = d.execute_script("return [window.innerWidth, window.innerHeight]")
    d.set_window_size(1280 + 1280 - inner[0], 720 + 800 - inner[1])
    yield d
    d.quit()


def covers_loaded(driver):
    return driver.execute_script("return [...document.images].every(i => i.complete && i.naturalWidth > 0)")


def test_catalogue_page(driver):
    driver.get(f"{BASE_URL}/books?page=2")
    driver.execute_script("document.documentElement.dataset.theme = 'light'")
    WebDriverWait(driver, 10).until(covers_loaded)
    assert_matches_baseline(driver.get_screenshot_as_png(), "catalogue-page-2")


def test_first_book_card(driver):
    driver.get(f"{BASE_URL}/books")
    driver.execute_script("document.documentElement.dataset.theme = 'light'")
    WebDriverWait(driver, 10).until(covers_loaded)
    card = driver.find_element(By.CSS_SELECTOR, "[data-testid=book-card]")
    assert_matches_baseline(card.screenshot_as_png, "first-book-card")

package demo;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;

import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import javax.imageio.ImageIO;

final class ImageAssert {
    private static final Path BASELINES = Path.of("baselines");
    private static final Path ACTUALS = Path.of("actual");

    private ImageAssert() {}

    /** Compares a PNG screenshot with baselines/name.png; a missing baseline is written and the test fails. */
    static void matchesBaseline(byte[] png, String name, double maxDiffRatio, int channelTolerance) throws IOException {
        BufferedImage actual = ImageIO.read(new ByteArrayInputStream(png));
        Path baseline = BASELINES.resolve(name + ".png");
        if ("1".equals(System.getenv("UPDATE_BASELINES"))) {
            Files.createDirectories(BASELINES);
            ImageIO.write(actual, "png", baseline.toFile());
            return;
        }
        if (!Files.exists(baseline)) {
            Files.createDirectories(BASELINES);
            ImageIO.write(actual, "png", baseline.toFile());
            fail(baseline + " did not exist and was written; review it and run again");
        }
        BufferedImage expected = ImageIO.read(baseline.toFile());
        assertEquals(expected.getWidth(), actual.getWidth(), "width");
        assertEquals(expected.getHeight(), actual.getHeight(), "height");
        BufferedImage diff = new BufferedImage(actual.getWidth(), actual.getHeight(), BufferedImage.TYPE_INT_RGB);
        long changed = 0;
        for (int y = 0; y < actual.getHeight(); y++) {
            for (int x = 0; x < actual.getWidth(); x++) {
                int a = expected.getRGB(x, y);
                int b = actual.getRGB(x, y);
                boolean different = Math.abs(((a >> 16) & 255) - ((b >> 16) & 255)) > channelTolerance
                        || Math.abs(((a >> 8) & 255) - ((b >> 8) & 255)) > channelTolerance
                        || Math.abs((a & 255) - (b & 255)) > channelTolerance;
                if (different) {
                    changed++;
                }
                diff.setRGB(x, y, different ? 0xFF0000 : 0xFFFFFF);
            }
        }
        double ratio = (double) changed / ((double) actual.getWidth() * actual.getHeight());
        if (ratio > maxDiffRatio) {
            Files.createDirectories(ACTUALS);
            ImageIO.write(actual, "png", ACTUALS.resolve(name + "-actual.png").toFile());
            ImageIO.write(diff, "png", ACTUALS.resolve(name + "-diff.png").toFile());
            fail(String.format("%s: %.2f%% of the pixels differ (limit %.2f%%)", name, ratio * 100, maxDiffRatio * 100));
        }
    }
}

package demo;

import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.openqa.selenium.By;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.OutputType;
import org.openqa.selenium.TakesScreenshot;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.remote.RemoteWebDriver;
import org.openqa.selenium.support.ui.WebDriverWait;

class VisualTest {
    private static final String BASE_URL = System.getenv().getOrDefault("BASE_URL", "http://localhost:3963");
    private static final String SEED_URL = System.getenv().getOrDefault("SEED_URL", BASE_URL);
    private static final String GRID_URL = System.getenv().getOrDefault("GRID_URL", "http://localhost:4444");

    private WebDriver driver;

    @BeforeEach
    void setUp() throws Exception {
        StringBuilder books = new StringBuilder("[");
        String[] colors = {"#2563eb", "#059669", "#d97706", "#7c3aed"};
        for (int i = 0; i < 8; i++) {
            books.append(i > 0 ? "," : "").append(String.format(
                    "{\"title\":\"Seeded Book %d\",\"author\":\"Test Author\",\"genre\":\"Fiction\",\"price\":%d,\"color\":\"%s\"}",
                    i + 1, 10 + i, colors[i % 4]));
        }
        books.append("]");
        HttpRequest seed = HttpRequest.newBuilder(URI.create(SEED_URL + "/api/test/seed"))
                .header("content-type", "application/json")
                .POST(HttpRequest.BodyPublishers.ofString("{\"books\":" + books + ",\"reviews\":[]}")).build();
        HttpClient.newHttpClient().send(seed, HttpResponse.BodyHandlers.discarding());

        ChromeOptions options = new ChromeOptions();
        options.addArguments("--headless=new", "--hide-scrollbars", "--force-device-scale-factor=1");
        driver = new RemoteWebDriver(URI.create(GRID_URL).toURL(), options);
        driver.manage().window().setSize(new org.openqa.selenium.Dimension(1280, 800));
        @SuppressWarnings("unchecked")
        java.util.List<Long> inner = (java.util.List<Long>) ((JavascriptExecutor) driver)
                .executeScript("return [window.innerWidth, window.innerHeight]");
        driver.manage().window().setSize(new org.openqa.selenium.Dimension(
                (int) (1280 + 1280 - inner.get(0)), (int) (720 + 800 - inner.get(1))));
    }

    @AfterEach
    void tearDown() {
        driver.quit();
    }

    private void openAndWaitForCovers(String path) {
        driver.get(BASE_URL + path);
        JavascriptExecutor js = (JavascriptExecutor) driver;
        js.executeScript("document.documentElement.dataset.theme = 'light'");
        new WebDriverWait(driver, Duration.ofSeconds(10)).until(d -> (Boolean) js.executeScript(
                "return [...document.images].every(i => i.complete && i.naturalWidth > 0)"));
    }

    @Test
    void cataloguePageMatchesBaseline() throws Exception {
        openAndWaitForCovers("/books?page=2");
        byte[] png = ((TakesScreenshot) driver).getScreenshotAs(OutputType.BYTES);
        ImageAssert.matchesBaseline(png, "catalogue-page-2", 0.01, 24);
    }

    @Test
    void firstBookCardMatchesBaseline() throws Exception {
        openAndWaitForCovers("/books");
        WebElement card = driver.findElement(By.cssSelector("[data-testid=book-card]"));
        ImageAssert.matchesBaseline(card.getScreenshotAs(OutputType.BYTES), "first-book-card", 0.01, 24);
    }
}

import { join } from 'node:path';
import type { VisualServiceOptions } from '@wdio/visual-service';

// REMOTE=grid sends the commands to a Selenium Grid (WebDriver), otherwise a local chromedriver is started.
const grid = process.env.REMOTE === 'grid';

export const config: WebdriverIO.Config = {
  runner: 'local',
  logLevel: 'warn',
  specs: ['./test/**/*.ts'],
  maxInstances: 1,
  baseUrl: process.env.BASE_URL ?? 'http://localhost:3963',
  ...(grid ? { hostname: 'localhost', port: Number(process.env.GRID_PORT ?? 4444), path: '/' } : {}),
  capabilities: [
    {
      browserName: 'chrome',
      'goog:chromeOptions': { args: ['--headless=new', '--window-size=1280,720', '--hide-scrollbars'] },
      'wdio-ics:options': { logName: grid ? 'chrome-grid' : 'chrome-local' },
    },
  ],
  services: [
    [
      'visual',
      {
        baselineFolder: join(process.cwd(), 'visual', 'baseline'),
        screenshotPath: join(process.cwd(), 'visual', 'tmp'),
        formatImageName: '{tag}-{logName}-{width}x{height}',
        autoSaveBaseline: true,
        savePerInstance: true,
        disableBlinkingCursor: true,
        disableCSSAnimation: true,
        hideScrollBars: true,
      } satisfies VisualServiceOptions,
    ],
  ],
  framework: 'mocha',
  reporters: ['spec'],
  mochaOpts: { ui: 'bdd', timeout: 60000 },
};

describe('Bookshelf catalogue (WebDriver)', () => {
  before(async () => {
    await browser.call(() =>
      fetch(`${process.env.SEED_URL ?? browser.options.baseUrl}/api/test/seed`, {
        method: 'POST',
        headers: { 'content-type': 'application/json' },
        body: JSON.stringify({
          books: Array.from({ length: 8 }, (_, i) => ({
            title: `Seeded Book ${i + 1}`,
            author: 'Test Author',
            genre: 'Fiction',
            price: 10 + i,
            color: ['#2563eb', '#059669', '#d97706', '#7c3aed'][i % 4],
          })),
          reviews: [],
        }),
      }),
    );
  });

  beforeEach(async () => {
    await browser.url('/books?page=2');
    // The page follows the operating system's colour scheme; pin it so the baseline does not depend on the machine.
    await browser.execute(() => {
      document.documentElement.dataset.theme = 'light';
    });
    await $('[data-testid=book-list]').waitForDisplayed();
    await browser.waitUntil(async () =>
      browser.execute(() => [...document.images].every((image) => image.complete && image.naturalWidth > 0)),
    );
  });

  it('matches the screen baseline', async () => {
    await expect(browser).toMatchScreenSnapshot('catalogue-page-2', 0.5);
  });

  it('matches one book card', async () => {
    const [firstCard] = await $$('[data-testid=book-card]');
    await expect(firstCard).toMatchElementSnapshot('first-book-card', 0.5);
  });

  it('returns a mismatch percentage with checkScreen', async () => {
    const mismatch = await browser.checkScreen('catalogue-page-2');
    expect(mismatch).toBeLessThan(0.5);
  });
});

npm install --save-dev @percy/cli @percy/playwright @percy/cypress
export PERCY_TOKEN="<your project token>"
npx percy exec -- npx playwright test
npx percy exec -- npx cypress run

// Playwright: tests/percy.spec.ts
import { test } from '@playwright/test';
import percySnapshot from '@percy/playwright';

test('catalogue on Percy', async ({ page }) => {
  await page.goto('/books');
  await percySnapshot(page, 'Catalogue');
});

// Cypress: cypress/support/e2e.ts has: import '@percy/cypress';
describe('catalogue on Percy', () => {
  it('takes a snapshot', () => {
    cy.visit('/books');
    cy.percySnapshot('Catalogue', { widths: [768, 1280] });
  });
});

# Selenium Python: pip install percy-selenium
from percy import percy_snapshot

driver.get("http://localhost:3963/books")
percy_snapshot(driver, "Catalogue")

// Selenium Java: io.percy:percy-java-selenium
import io.percy.selenium.Percy;

Percy percy = new Percy(driver);
driver.get("http://localhost:3963/books");
percy.snapshot("Catalogue");

npm install -D @applitools/eyes-playwright
npx eyes-setup
export APPLITOOLS_API_KEY="<your API key>"
npx playwright test

// Playwright: import the Eyes fixture instead of the Playwright test object
import { test } from '@applitools/eyes-playwright/fixture';

test('catalogue on Applitools', async ({ page, eyes }) => {
  await page.goto('/books');
  await eyes.check('Catalogue', { fully: true, matchLevel: 'Dynamic' });
});

// Cypress: cypress/support/e2e.ts has: import '@applitools/eyes-cypress/commands';
describe('catalogue on Applitools', () => {
  it('checks the window', () => {
    cy.visit('/books');
    cy.eyesOpen({ appName: 'Bookshelf', testName: 'Catalogue', browser: { width: 1280, height: 720 } });
    cy.eyesCheckWindow('Catalogue');
    cy.eyesClose();
  });
});

npm install --save-dev chromatic @chromatic-com/playwright
npx playwright test
npx chromatic --playwright -t=<project token>

// Playwright: swap the import, the rest of the test stays the same
import { test, expect } from '@chromatic-com/playwright';

test('catalogue on Chromatic', async ({ page }) => {
  await page.goto('/books');
  await expect(page.getByTestId('book-list')).toBeVisible();
});

npm install --save-dev chromatic @chromatic-com/cypress
# cypress/support/e2e.js: import '@chromatic-com/cypress/support';
# cypress.config.js: installPlugin(on, config) from '@chromatic-com/cypress' in setupNodeEvents
ELECTRON_EXTRA_LAUNCH_ARGS=--remote-debugging-port=9222 npx cypress run
npx chromatic --cypress -t=<project token>