examples/file-pipeline.mjs
GitHub// @node-skills {"added":["v25.9.0","v24.20.0"],"flags":["--experimental-stream-iter"]}
// file -> transforms -> file with filehandle.pull()/writer() and pipeTo(); stateless and stateful transforms.
import { open, mkdtemp, rm, writeFile, readFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { pipeTo, pull, from, text, tap } from 'node:stream/iter';
import assert from 'node:assert/strict';
const dir = await mkdtemp(join(tmpdir(), 'iter-file-'));
try {
await writeFile(join(dir, 'in.txt'), 'alpha\nbeta\ngamma\n');
// Stateless transform: called once per batch (Uint8Array[]), then once with null (flush).
const upper = (chunks) => chunks === null ? null
: chunks.map((c) => new TextEncoder().encode(new TextDecoder().decode(c).toUpperCase()));
// Stateful transform: a generator over the whole upstream; it sees null after the source ends.
const numberLines = {
async *transform(source) {
let partial = '';
let n = 0;
for await (const chunks of source) {
if (chunks === null) { if (partial) yield [new TextEncoder().encode(`${++n} ${partial}`)]; continue; }
for (const c of chunks) {
const lines = (partial + new TextDecoder().decode(c)).split('\n');
partial = lines.pop();
yield lines.map((l) => new TextEncoder().encode(`${++n} ${l}\n`));
}
}
},
};
let seen = 0;
const count = tap((chunks) => { if (chunks) for (const c of chunks) seen += c.byteLength; });
const src = await open(join(dir, 'in.txt'));
const dst = await open(join(dir, 'out.txt'), 'w');
const written = await pipeTo(
src.pull({ autoClose: true }), // AsyncIterable<Uint8Array[]>, 128 KiB reads by default
upper, numberLines, count,
dst.writer({ autoClose: true }), // end() closes the handle
);
assert.equal(await readFile(join(dir, 'out.txt'), 'utf8'), '1 ALPHA\n2 BETA\n3 GAMMA\n');
assert.equal(written, 23);
assert.equal(seen, 23);
await assert.rejects(src.stat(), { code: 'EBADF' }); // autoClose closed the source too
// Ranged read: start/limit use positioned reads.
const fh = await open(join(dir, 'in.txt'));
assert.equal(await text(fh.pull({ start: 6, limit: 4, autoClose: true })), 'beta');
// Transforms can also be passed straight to filehandle.pull().
const fh2 = await open(join(dir, 'in.txt'));
assert.equal(await text(fh2.pull(upper, { autoClose: true })), 'ALPHA\nBETA\nGAMMA\n');
// Writer without autoClose: end() resolves to the byte count; close the handle yourself.
const out = await open(join(dir, 'w.txt'), 'w');
const w = out.writer();
await w.write('a');
await w.writev(['b', new Uint8Array([0x63])]);
assert.equal(await w.end(), 3);
await out.close();
assert.equal(await readFile(join(dir, 'w.txt'), 'utf8'), 'abc');
// pull() is lazy: nothing is read until the result is iterated.
const lazy = pull(from('xyz'), upper);
assert.equal(await text(lazy), 'XYZ');
} finally {
await rm(dir, { recursive: true, force: true });
}