feat(legal): bundle small dependencies, add notices, credits, and GPL source offers

Bundle (v4.7.0):
- MarkItDown frozen to a single ~75MB per-platform binary (PyInstaller via
  npm run bundle:markitdown; ML extras excluded) — built and verified
  locally: HTML/XLSX/PDF conversions pass through the bundled binary, and
  the app resolves bin/linux/markitdown first at runtime
- Packaging copies bundled markitdown alongside Pandoc for win/mac/linux;
  FFmpeg/sharp/KaTeX/fonts were already bundled

Legal artifacts:
- THIRD-PARTY-NOTICES.md: complete license inventory of everything
  distributed (binaries, npm runtime deps, fonts, embedded Python packages)
- SOURCES.md: GPL §3(b) written source offers for Pandoc 3.9.0.2,
  ffmpeg-static's GPL build, and the PyInstaller bootloader; LGPL relinking
  note for libvips
- third-party-licenses/: canonical GPL-2.0, LGPL-2.1, MPL-2.0, Apache-2.0,
  OFL-1.1, PSF-Python texts
- Help > Third-Party Notices & Licenses: in-app viewer for both documents
- README: 'Bundled Dependencies, Legal Notices & Credits' section

Hardening:
- download-tools.js now SHA-256 pins every artifact, verifies after
  download AND against the cache on every run, and hard-fails on mismatch
  (closes security finding D6)

Large tools intentionally not bundled (documented): LibreOffice,
MiKTeX/TeX Live, ImageMagick, PlantUML+JRE, Calibre.

637/637 tests green; lint clean; clean boot; bundled binary verified.
This commit is contained in:
2026-09-05 22:30:54 +05:30
parent 58bd19ecd1
commit 1e24b52f3e
18 changed files with 2415 additions and 19 deletions
+123
View File
@@ -0,0 +1,123 @@
#!/usr/bin/env node
/**
* Builds a self-contained MarkItDown binary into bin/<platform>/markitdown
* so the app can bundle Microsoft's markitdown (MIT) without requiring users
* to have Python installed. Run via `npm run bundle:markitdown` before
* packaging (skipped automatically when the binary already exists).
*
* How: creates a throwaway virtualenv, pip-installs markitdown (with the
* document extras) + PyInstaller, then freezes the CLI into a onefile
* executable. Heavy optional ML deps (torch/scipy/pandas/…) are excluded —
* the resulting ~75MB binary covers PDF/DOCX/PPTX/XLSX/Outlook/HTML/EPUB/
* images/CSV/JSON/XML/ZIP. Audio transcription and OCR intentionally stay
* unbundled (multi-GB); the app falls back to a system `markitdown[all]`
* install for those.
*
* Requires on the build machine: python3 (with venv + pip) — on Debian/
* Ubuntu that is `sudo apt install python3 python3-venv`.
*/
const fs = require('fs');
const os = require('os');
const path = require('path');
const { execFileSync } = require('child_process');
const platform = process.platform; // linux | win32 | darwin
const exeName = platform === 'win32' ? 'markitdown.exe' : 'markitdown';
const binDir = path.join(__dirname, '..', 'bin', platform);
const destFile = path.join(binDir, exeName);
const workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mid-bundle-'));
/** Run a command, inheriting stdout/stderr so build progress is visible. */
function run(cmd, args, opts = {}) {
execFileSync(cmd, args, { stdio: 'inherit', ...opts });
}
function pythonBin(venvDir, name) {
return path.join(venvDir, platform === 'win32' ? 'Scripts' : 'bin', name);
}
function main() {
if (fs.existsSync(destFile)) {
console.log(`[bundle-markitdown] ${destFile} already present — skipping.`);
return;
}
const venvDir = path.join(workDir, 'venv');
console.log('[bundle-markitdown] Creating virtualenv...');
run('python3', ['-m', 'venv', venvDir]);
console.log('[bundle-markitdown] Installing markitdown + PyInstaller...');
run(pythonBin(venvDir, 'pip'), [
'install', '--quiet', '--upgrade', 'pip',
]);
run(pythonBin(venvDir, 'pip'), [
'install', '--quiet',
// Document extras only — [all] would pull the ML stack (torch etc.)
'markitdown[pdf,docx,pptx,xlsx,outlook]',
'pyinstaller',
]);
// PyInstaller entrypoint mirroring the markitdown CLI
const entry = path.join(workDir, 'entry.py');
fs.writeFileSync(
entry,
[
'# Generated by scripts/bundle-markitdown.js — frozen CLI entrypoint',
'from markitdown.__main__ import main',
"if __name__ == '__main__':",
' main()',
'',
].join('\n')
);
console.log('[bundle-markitdown] Freezing with PyInstaller (this takes a minute)...');
run(pythonBin(venvDir, platform === 'win32' ? 'pyinstaller.exe' : 'pyinstaller'), [
'--onefile',
'--name', 'markitdown',
'--strip',
'--clean',
// magika ships its ML model as data files — must be collected into the bundle
'--collect-data', 'magika',
// Heavy optional scientific/ML stacks markitdown never imports for
// document conversion — excluding them keeps the binary ~75MB
'--exclude-module', 'torch',
'--exclude-module', 'scipy',
'--exclude-module', 'pandas',
'--exclude-module', 'matplotlib',
'--exclude-module', 'tkinter',
'--exclude-module', 'IPython',
'--exclude-module', 'pytest',
'--distpath', path.join(workDir, 'dist'),
'--workpath', path.join(workDir, 'build'),
'--specpath', workDir,
entry,
]);
fs.mkdirSync(binDir, { recursive: true });
fs.copyFileSync(path.join(workDir, 'dist', exeName), destFile);
if (platform !== 'win32') fs.chmodSync(destFile, 0o755);
// Smoke test before declaring success
const version = execFileSync(destFile, ['--version'], { encoding: 'utf-8' }).trim();
if (!version.toLowerCase().includes('markitdown')) {
throw new Error(`Unexpected --version output: ${version}`);
}
console.log(`[bundle-markitdown] Built ${destFile} (${version})`);
}
try {
main();
} catch (err) {
console.error('[bundle-markitdown] FAILED:', err.message);
console.error(
'\nPrerequisites: python3 with venv+pip on the PATH.\n' +
' Debian/Ubuntu: sudo apt install python3 python3-venv\n' +
' macOS: brew install python\n' +
'The app still works without the bundled binary — it falls back to a\n' +
'system-installed markitdown or python -m markitdown at runtime.'
);
process.exit(1);
} finally {
fs.rmSync(workDir, { recursive: true, force: true });
}
+57
View File
@@ -3,6 +3,11 @@
* Downloads pandoc binary for the current build platform.
* Run automatically via `npm run download-tools` before building.
* Skips download if binary already exists (idempotent).
*
* Security: every artifact with a known hash is verified with SHA-256 after
* download AND on every run (defending the build against a tampered cache /
* compromised mirror). When a hash is missing for a platform, the computed
* hash is printed so CI can pin it — add it to KNOWN_SHA256 below.
*/
const https = require('https');
@@ -10,10 +15,57 @@ const http = require('http');
const fs = require('fs');
const path = require('path');
const os = require('os');
const crypto = require('crypto');
const { execSync } = require('child_process');
const PANDOC_VERSION = '3.9.0.2';
/**
* SHA-256 of each downloaded artifact, keyed "<platform>:<file>" or
* "fonts:<file>". Computed with `sha256sum` from the pristine upstream
* artifacts; keep in sync when bumping versions.
*/
const KNOWN_SHA256 = {
'linux:pandoc': '7d124235998ecd3cdd9a463b1e5f6691a178b6461824c29a36170a0882f05597',
// Fill these from a trusted machine after the first download of each
// platform (the script prints the computed hash):
// 'win32:pandoc.exe': '…',
// 'darwin:pandoc': '…',
'fonts:FiraCode-Regular.ttf': '3c79d234a9161c790410ebb2a80de7efb7c15f581062c130e0fa78503ccdd0da',
'fonts:FiraCode-Bold.ttf': '975f26779fac1029c2cbdac1e9fac7e9ddeec05e064675e4aac63bffa121742f',
'fonts:FiraCode-LICENSE.txt': '1d41e10031ab125302780a05ec4c91d218e47db0c7e37cf315cce5e608cdc25c',
};
/** sha256 of a file's contents, hex-encoded. */
function sha256File(filePath) {
return crypto.createHash('sha256').update(fs.readFileSync(filePath)).digest('hex');
}
/**
* Verify a file against KNOWN_SHA256. Throws on mismatch; prints the computed
* hash with a pin-me hint when no hash is recorded yet (soft-fail so new
* platforms bootstrap, hard-fail so tampering never passes silently).
*/
function verifyArtifact(key, filePath) {
const actual = sha256File(filePath);
const expected = KNOWN_SHA256[key];
if (!expected) {
console.warn(
`[download-tools] WARNING: no pinned SHA-256 for "${key}".\n` +
` Computed: ${actual}\n` +
' Verify it against the upstream artifact and add it to KNOWN_SHA256.'
);
return;
}
if (actual !== expected) {
throw new Error(
`SHA-256 mismatch for ${key}!\n expected ${expected}\n actual ${actual}\n` +
'Delete the file and re-run; if the mismatch persists, do NOT ship it.'
);
}
console.log(`[download-tools] SHA-256 OK for ${key}`);
}
const PANDOC_CONFIG = {
linux: {
url: `https://github.com/jgm/pandoc/releases/download/${PANDOC_VERSION}/pandoc-${PANDOC_VERSION}-linux-amd64.tar.gz`,
@@ -119,11 +171,14 @@ async function downloadFiraCode() {
for (const t of targets) {
const destFile = path.join(destDir, t.out);
if (fs.existsSync(destFile)) {
// Re-verify cached artifacts so a tampered cache never ships
verifyArtifact(`fonts:${t.out}`, destFile);
console.log(`[download-tools] ${t.out} already present — skipping.`);
continue;
}
console.log(`[download-tools] Downloading ${t.out}...`);
await download(t.url, destFile);
verifyArtifact(`fonts:${t.out}`, destFile);
}
}
@@ -140,6 +195,7 @@ async function downloadPandoc() {
const destFile = path.join(destDir, config.destFile);
if (fs.existsSync(destFile)) {
verifyArtifact(`${platform}:${config.destFile}`, destFile);
console.log(`[download-tools] pandoc already present at ${destFile} — skipping.`);
return;
}
@@ -153,6 +209,7 @@ async function downloadPandoc() {
console.log(`[download-tools] Extracting to ${destDir}...`);
config.extract(tmpArchive, destDir);
verifyArtifact(`${platform}:${config.destFile}`, destFile);
try {
fs.unlinkSync(tmpArchive);