Compression with zlib

import { gzipSync, gunzipSync, deflateSync, deflateRawSync, unzipSync,
  brotliCompressSync, brotliDecompressSync, gzip, gunzip, constants } from 'node:zlib';
import { promisify } from 'node:util';

const input = Buffer.from(JSON.stringify(Array.from({ length: 2000 }, (_, i) => ({ isbn: `978-${i}`, title: `Book ${i}`, rating: i % 5 + 1 }))));
console.log({
  raw: input.length,
  deflate: deflateSync(input).length,
  deflateRaw: deflateRawSync(input).length,
  gzip: gzipSync(input).length,
  brotli: brotliCompressSync(input).length,
  brotliText: brotliCompressSync(input, { params: {
    [constants.BROTLI_PARAM_MODE]: constants.BROTLI_MODE_TEXT,
    [constants.BROTLI_PARAM_QUALITY]: 4,
    [constants.BROTLI_PARAM_SIZE_HINT]: input.length } }).length,
});
console.log(gunzipSync(gzipSync(input)).equals(input), brotliDecompressSync(brotliCompressSync(input)).equals(input));
console.log(unzipSync(gzipSync('hi')).toString(), unzipSync(deflateSync('hi')).toString());

const gzipAsync = promisify(gzip);
const gunzipAsync = promisify(gunzip);
console.log((await gunzipAsync(await gzipAsync('async round trip'))).toString());

{
  raw: 99781,
  deflate: 10439,
  deflateRaw: 10433,
  gzip: 10451,
  brotli: 4349,
  brotliText: 3388
}
true true
hi hi
async round trip

import { createReadStream, createWriteStream } from 'node:fs';
import { stat, writeFile, mkdtemp } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { pipeline } from 'node:stream/promises';
import { createGzip, createGunzip } from 'node:zlib';

const dir = await mkdtemp(join(tmpdir(), 'bookshelf-'));
const csv = join(dir, 'books.csv');
const rows = ['isbn,title,rating'];
for (let i = 0; i < 50_000; i++) rows.push(`978-${i},Book number ${i},${i % 5 + 1}`);
await writeFile(csv, rows.join('\n'));

// compress
await pipeline(createReadStream(csv), createGzip({ level: 9 }), createWriteStream(`${csv}.gz`));
// decompress and count bytes without keeping them
let bytes = 0;
await pipeline(
  createReadStream(`${csv}.gz`),
  createGunzip(),
  async function* (source) { for await (const chunk of source) bytes += chunk.length; },
);
console.log({ original: (await stat(csv)).size, gzipped: (await stat(`${csv}.gz`)).size, inflated: bytes });

{ original: 1477797, gzipped: 253458, inflated: 1477797 }

import { createReadStream, createWriteStream } from 'node:fs';
import { mkdtemp } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { createInterface } from 'node:readline';
import { pipeline } from 'node:stream/promises';
import { Readable } from 'node:stream';
import { createGzip, createGunzip } from 'node:zlib';

const dir = await mkdtemp(join(tmpdir(), 'import-'));
const file = join(dir, 'reviews.csv.gz');
await pipeline(Readable.from(['isbn,rating\n', '978-1,5\n', '978-2,4\n']), createGzip(), createWriteStream(file));

const lines = createInterface({ input: createReadStream(file).pipe(createGunzip()), crlfDelay: Infinity });
for await (const line of lines) console.log(line);

// a corrupt upload rejects the pipeline instead of crashing
try {
  await pipeline(Readable.from([Buffer.from('not gzip at all')]), createGunzip(), async function* (s) { for await (const _ of s); });
} catch (e) { console.log('rejected with', e.code); }

isbn,rating
978-1,5
978-2,4
rejected with Z_DATA_ERROR

import { gzipSync, gunzipSync } from 'node:zlib';

const bomb = gzipSync(Buffer.alloc(50 * 1024 * 1024));
console.log('bomb compressed to', bomb.length, 'bytes');
try { gunzipSync(bomb, { maxOutputLength: 1024 * 1024 }); } catch (e) { console.log('bomb:', e.code); }

bomb compressed to 50988 bytes
bomb: ERR_BUFFER_TOO_LARGE

import { gzipSync, gunzipSync, constants } from 'node:zlib';

const good = gzipSync('The quick brown fox jumps over the lazy dog. '.repeat(50));
const truncated = good.subarray(0, good.length - 12);
try { gunzipSync(truncated); } catch (e) { console.log('truncated:', e.code, e.message); }
// inspect the beginning of a file you know is incomplete
const partial = gunzipSync(truncated, { finishFlush: constants.Z_SYNC_FLUSH });
console.log('partial output bytes:', partial.length);

try { gunzipSync(Buffer.from('definitely not gzip')); } catch (e) { console.log('garbage:', e.code, e.errno); }

// concatenated members are decoded as one stream
const two = Buffer.concat([gzipSync('one,'), gzipSync('two')]);
console.log(gunzipSync(two).toString());
// ...unless you refuse anything after the first member
try { gunzipSync(two, { rejectGarbageAfterEnd: true }); } catch (e) { console.log('second member:', e.code); }

truncated: Z_BUF_ERROR unexpected end of file
partial output bytes: 1594
garbage: Z_DATA_ERROR -3
one,two
second member: ERR_TRAILING_JUNK_AFTER_STREAM_END

import zlib from 'node:zlib';

const input = Buffer.from('zstd '.repeat(10_000));
const c = zlib.zstdCompressSync(input, { params: { [zlib.constants.ZSTD_c_compressionLevel]: 10, [zlib.constants.ZSTD_c_checksumFlag]: 1 } });
console.log(c.length, zlib.zstdDecompressSync(c).equals(input));

27 true

import { crc32 } from 'node:zlib';

let crc = crc32('hello');
console.log(crc, crc32('world', crc), crc32('helloworld'));
console.log(crc32(Buffer.from('hello', 'utf16le')));

907060870 4192936109 4192936109
1427272415

import { ZipEntry, ZipFile, ZipBuffer, createZipArchive, zipFiles, getMaxZipContentSize } from 'node:zlib';
import { createWriteStream } from 'node:fs';
import { mkdtemp, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { pipeline } from 'node:stream/promises';

const dir = await mkdtemp(join(tmpdir(), 'zip-'));
await writeFile(join(dir, 'review.txt'), 'Great book. '.repeat(100));
const archive = join(dir, 'export.zip');
await pipeline(
  zipFiles([[join(dir, 'review.txt'), 'reviews/review.txt']]),
  createWriteStream(archive),
);
const zip = await ZipFile.open(archive);
try {
  for (const [name, pending] of zip.entries()) {
    const e = await pending;
    console.log(name, e.size, e.compressedSize, e.method, e.crc32);
  }
  const entry = await zip.get('reviews/review.txt');
  console.log((await entry.content({ maxSize: 1024 * 1024 })).toString().slice(0, 22));
} finally { await zip.close(); }
console.log(getMaxZipContentSize());

const entries = [await ZipEntry.create('hello.txt', Buffer.from('Hello, world!'))];
const hello = await new ZipBuffer(Buffer.concat(await Array.fromAsync(createZipArchive(entries)))).get('hello.txt');
console.log((await hello.content()).toString());

reviews/review.txt 1200 25 8 3854544580
Great book. Great book
268435456
Hello, world!

import { createServer, get } from 'node:http';
import { gzipSync, brotliCompressSync, constants } from 'node:zlib';
import { once } from 'node:events';

const catalogue = Buffer.from(JSON.stringify(Array.from({ length: 5000 }, (_, i) => ({ isbn: `978-${i}`, title: `Book ${i}` }))));
// compress once, serve many times
const variants = new Map([
  ['br', brotliCompressSync(catalogue, { params: { [constants.BROTLI_PARAM_QUALITY]: 5 } })],
  ['gzip', gzipSync(catalogue)],
]);

function pickEncoding(header = '') {
  const accepted = new Map();
  for (const part of header.split(',')) {
    const [name, ...params] = part.trim().split(';');
    const q = params.map((p) => p.trim()).find((p) => p.startsWith('q='));
    accepted.set(name.trim().toLowerCase(), q ? Number(q.slice(2)) : 1);
  }
  for (const name of ['br', 'gzip']) if ((accepted.get(name) ?? accepted.get('*') ?? 0) > 0) return name;
  return 'identity';
}

const server = createServer((req, res) => {
  const encoding = pickEncoding(req.headers['accept-encoding']);
  res.setHeader('Content-Type', 'application/json');
  res.setHeader('Vary', 'Accept-Encoding');
  const body = encoding === 'identity' ? catalogue : variants.get(encoding);
  if (encoding !== 'identity') res.setHeader('Content-Encoding', encoding);
  res.setHeader('Content-Length', body.length);
  res.end(body);
});
server.listen(0);
await once(server, 'listening');
const url = `http://127.0.0.1:${server.address().port}/books`;

for (const ae of ['br, gzip', 'gzip', 'gzip;q=0, identity']) {
  const r = await fetch(url, { headers: { 'Accept-Encoding': ae } });
  const text = await r.text();
  console.log(ae, '->', r.headers.get('content-encoding'), r.headers.get('content-length'), text.length);
}
// http.get does not decompress: you see the bytes on the wire
get(url, { headers: { 'Accept-Encoding': 'gzip' } }, (res) => {
  let n = 0; res.on('data', (c) => { n += c.length; });
  res.on('end', () => { console.log('raw http.get bytes', n, res.headers['content-encoding']); server.close(); });
});

br, gzip -> br 8764 197781
gzip -> gzip 23768 197781
gzip;q=0, identity -> null 197781 197781
raw http.get bytes 23768 gzip

import { createGzip, createGunzip } from 'node:zlib';

const gz = createGzip();
const gun = createGunzip();
gz.pipe(gun);
gun.on('data', (c) => console.log('received', JSON.stringify(c.toString())));
gz.write('tick 1\n', () => {
  console.log('written (probably still buffered inside zlib)');
  gz.flush(() => { gz.write('tick 2\n', () => gz.flush(() => gz.end())); });
});

written (probably still buffered inside zlib)
received "tick 1\n"
received "tick 2\n"

import { gzip, gzipSync } from 'node:zlib';
import { randomBytes } from 'node:crypto';

const data = randomBytes(16 * 1024 * 1024);
let maxGap = 0, last = performance.now();
const timer = setInterval(() => { const now = performance.now(); maxGap = Math.max(maxGap, now - last - 10); last = now; }, 10);
await new Promise((r) => setTimeout(r, 50));
maxGap = 0;
gzipSync(data, { level: 9 });
await new Promise((r) => setTimeout(r, 30));
console.log('sync: worst timer delay', Math.round(maxGap), 'ms');
maxGap = 0; last = performance.now();
await new Promise((r) => gzip(data, { level: 9 }, r));
console.log('async: worst timer delay', Math.round(maxGap), 'ms');
clearInterval(timer);

sync: worst timer delay 184 ms
async: worst timer delay 0 ms

import { gzip } from 'node:zlib';
import { promisify } from 'node:util';
import { randomBytes } from 'node:crypto';

const gz = promisify(gzip);
const data = randomBytes(8 * 1024 * 1024);
const t = performance.now();
await Promise.all(Array.from({ length: 8 }, () => gz(data, { level: 9 })));
console.log(`UV_THREADPOOL_SIZE=${process.env.UV_THREADPOOL_SIZE ?? 'default'}: ${Math.round(performance.now() - t)} ms for 8 parallel gzip calls`);

UV_THREADPOOL_SIZE=1 node pool.js
UV_THREADPOOL_SIZE=4 node pool.js
UV_THREADPOOL_SIZE=8 node pool.js

const compressed = new Blob(['web streams compress too. '.repeat(100)])
  .stream()
  .pipeThrough(new CompressionStream('gzip'));
const bytes = new Uint8Array(await new Response(compressed).arrayBuffer());
console.log('gzip bytes:', bytes.length);
const text = await new Response(new Blob([bytes]).stream().pipeThrough(new DecompressionStream('gzip'))).text();
console.log(text.length);

gzip bytes: 66
2600

import { from, pull, bytes, text } from 'node:stream/iter';
import { compressGzip, decompressGzip } from 'node:zlib/iter';

const compressed = await bytes(pull(from('hello'), compressGzip()));
console.log(await text(pull(from(compressed), decompressGzip())));

node --experimental-stream-iter iter.mjs
# (node:...) ExperimentalWarning: stream/iter is an experimental feature and might change at any time
# (node:...) ExperimentalWarning: zlib/iter is an experimental feature and might change at any time
# hello