scripts/stitch_full_page_capture.mjs
GitHub#!/usr/bin/env node
import fs from 'node:fs/promises';
import path from 'node:path';
import { spawn } from 'node:child_process';
import { createRequire } from 'node:module';
function usage() {
console.log(`Usage:
node stitch_full_page_capture.mjs --manifest <manifest.json> [options]
Options:
--item <n> Repair one 1-based item index instead of all items.
--viewport <WxH> Viewport size. Default: 1440x1100.
--step <px> Scroll/crop step. Default: min(height - 150, height).
--wait <ms> Wait after each scroll before screenshot. Default: 2000.
--quality <n> ffmpeg JPEG quality. Default: 3.
`);
}
function parseArgs(argv) {
const args = {
manifest: '',
item: 0,
viewport: '1440x1100',
step: 0,
wait: 2000,
quality: 3
};
for (let i = 0; i < argv.length; i++) {
const arg = argv[i];
if (arg === '--help' || arg === '-h') args.help = true;
else if (arg === '--manifest') args.manifest = argv[++i] || '';
else if (arg === '--item') args.item = Number(argv[++i] || 0);
else if (arg === '--viewport') args.viewport = argv[++i] || args.viewport;
else if (arg === '--step') args.step = Number(argv[++i] || 0);
else if (arg === '--wait') args.wait = Number(argv[++i] || args.wait);
else if (arg === '--quality') args.quality = Number(argv[++i] || args.quality);
else throw new Error(`Unknown argument: ${arg}`);
}
const match = args.viewport.match(/^(\d+)x(\d+)$/);
if (!match) throw new Error(`Invalid --viewport: ${args.viewport}`);
args.width = Number(match[1]);
args.height = Number(match[2]);
if (!args.step) args.step = Math.max(1, args.height - 150);
return args;
}
function run(command, args) {
return new Promise((resolve, reject) => {
const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] });
let stderr = '';
child.stderr.on('data', d => stderr += d);
child.on('close', code => {
if (code === 0) resolve();
else reject(new Error(`${command} ${args.join(' ')}\n${stderr}`));
});
});
}
function rel(fromDir, file) {
return path.relative(fromDir, file).replaceAll(path.sep, '/');
}
async function imageHeight(file) {
const { execFile } = await import('node:child_process');
return new Promise((resolve, reject) => {
execFile('sips', ['-g', 'pixelHeight', file], (error, stdout, stderr) => {
if (error) reject(new Error(stderr || error.message));
else resolve(Number(stdout.match(/pixelHeight:\s*(\d+)/)?.[1] || 0));
});
});
}
async function main() {
const args = parseArgs(process.argv.slice(2));
if (args.help || !args.manifest) {
usage();
process.exit(args.help ? 0 : 1);
}
const manifestPath = path.resolve(args.manifest);
const articleDir = path.dirname(manifestPath);
const manifest = JSON.parse(await fs.readFile(manifestPath, 'utf8'));
const tempRoot = path.join(articleDir, '.stitched-full-page-temp');
await fs.rm(tempRoot, { recursive: true, force: true });
await fs.mkdir(tempRoot, { recursive: true });
const requireFromWorkspace = createRequire(path.join(process.cwd(), 'package.json'));
const { chromium } = requireFromWorkspace('playwright');
const browser = await chromium.launch({ headless: true });
const targets = manifest.items
.map((item, index) => ({ item, index }))
.filter(({ index }) => !args.item || index === args.item - 1);
if (!targets.length) throw new Error(`No manifest items matched --item ${args.item}`);
try {
for (const { item, index } of targets) {
if (!item.pageUrl) throw new Error(`Item ${index + 1} is missing pageUrl`);
if (!item.fullPageImage) throw new Error(`Item ${index + 1} is missing fullPageImage`);
if (!Array.isArray(item.sectionImages) || item.sectionImages.length === 0) {
throw new Error(`Item ${index + 1} is missing sectionImages`);
}
const name = `${String(index + 1).padStart(2, '0')}-${(item.title || 'item').toLowerCase().replace(/[^a-z0-9]+/g, '-')}`;
const workDir = path.join(tempRoot, name);
const rawDir = path.join(workDir, 'raw');
const segmentDir = path.join(workDir, 'segments');
await fs.mkdir(rawDir, { recursive: true });
await fs.mkdir(segmentDir, { recursive: true });
const page = await browser.newPage({
viewport: { width: args.width, height: args.height },
deviceScaleFactor: 1
});
await page.goto(item.pageUrl, { waitUntil: 'domcontentloaded', timeout: 60000 });
await page.waitForTimeout(5000);
const warmHeight = await page.evaluate(() => Math.max(document.documentElement.scrollHeight, document.body?.scrollHeight || 0));
for (let y = 0; y <= Math.max(0, warmHeight - args.height); y += args.height) {
await page.evaluate(scrollY => window.scrollTo(0, scrollY), y);
await page.waitForTimeout(350);
}
await page.evaluate(() => window.scrollTo(0, 0));
await page.waitForTimeout(args.wait);
const scrollHeight = await page.evaluate(() => Math.max(document.documentElement.scrollHeight, document.body?.scrollHeight || 0));
const positions = [];
for (let y = 0; y < scrollHeight; y += args.step) positions.push(y);
const segmentFiles = [];
for (let s = 0; s < positions.length; s++) {
const y = positions[s];
await page.evaluate(scrollY => window.scrollTo(0, scrollY), y);
await page.waitForTimeout(args.wait);
const rawFile = path.join(rawDir, `${String(s + 1).padStart(2, '0')}.jpg`);
await page.screenshot({ path: rawFile, type: 'jpeg', quality: 86, fullPage: false });
const segmentHeight = Math.min(args.step, scrollHeight - y);
const segmentFile = path.join(segmentDir, `${String(s + 1).padStart(2, '0')}.jpg`);
await run('ffmpeg', ['-y', '-i', rawFile, '-vf', `crop=${args.width}:${segmentHeight}:0:0`, '-q:v', String(args.quality), segmentFile]);
segmentFiles.push(segmentFile);
}
await page.close();
const fullFile = path.join(articleDir, item.fullPageImage);
const stackInputs = segmentFiles.flatMap(file => ['-i', file]);
const stackFilter = segmentFiles.map((_, i) => `[${i}:v]`).join('') + `vstack=inputs=${segmentFiles.length}`;
await run('ffmpeg', ['-y', ...stackInputs, '-filter_complex', stackFilter, '-q:v', String(args.quality), fullFile]);
const stitchedHeight = await imageHeight(fullFile);
item.sectionImages = item.sectionImages.map((section, sectionIndex) => {
const yStart = Math.round((stitchedHeight * sectionIndex) / item.sectionImages.length);
const yEnd = sectionIndex === item.sectionImages.length - 1
? stitchedHeight
: Math.round((stitchedHeight * (sectionIndex + 1)) / item.sectionImages.length);
return {
...section,
source: rel(articleDir, fullFile),
yStart,
yEnd,
height: Math.max(1, yEnd - yStart)
};
});
for (const section of item.sectionImages) {
const cropFile = path.join(articleDir, section.file);
await run('ffmpeg', ['-y', '-i', fullFile, '-vf', `crop=${args.width}:${section.height}:0:${section.yStart}`, '-q:v', String(args.quality), cropFile]);
}
item.sourceInspection = {
...(item.sourceInspection || {}),
scrollHeight: stitchedHeight,
fullPageCaptureMethod: 'stitchedViewportScreenshots',
fullPageCaptureStep: args.step,
fullPageCaptureViewport: { width: args.width, height: args.height },
fullPageCaptureSegmentCount: segmentFiles.length
};
}
} finally {
await browser.close();
await fs.rm(tempRoot, { recursive: true, force: true });
}
await fs.writeFile(manifestPath, JSON.stringify(manifest, null, 2) + '\n');
}
main().catch(error => {
console.error(error.stack || error.message);
process.exit(1);
});