feat(legal): bundle small dependencies, add notices, credits, and GPL source offers

Bundle (v4.7.0):
- MarkItDown frozen to a single ~75MB per-platform binary (PyInstaller via
  npm run bundle:markitdown; ML extras excluded) — built and verified
  locally: HTML/XLSX/PDF conversions pass through the bundled binary, and
  the app resolves bin/linux/markitdown first at runtime
- Packaging copies bundled markitdown alongside Pandoc for win/mac/linux;
  FFmpeg/sharp/KaTeX/fonts were already bundled

Legal artifacts:
- THIRD-PARTY-NOTICES.md: complete license inventory of everything
  distributed (binaries, npm runtime deps, fonts, embedded Python packages)
- SOURCES.md: GPL §3(b) written source offers for Pandoc 3.9.0.2,
  ffmpeg-static's GPL build, and the PyInstaller bootloader; LGPL relinking
  note for libvips
- third-party-licenses/: canonical GPL-2.0, LGPL-2.1, MPL-2.0, Apache-2.0,
  OFL-1.1, PSF-Python texts
- Help > Third-Party Notices & Licenses: in-app viewer for both documents
- README: 'Bundled Dependencies, Legal Notices & Credits' section

Hardening:
- download-tools.js now SHA-256 pins every artifact, verifies after
  download AND against the cache on every run, and hard-fails on mismatch
  (closes security finding D6)

Large tools intentionally not bundled (documented): LibreOffice,
MiKTeX/TeX Live, ImageMagick, PlantUML+JRE, Calibre.

637/637 tests green; lint clean; clean boot; bundled binary verified.
This commit is contained in:
2026-09-05 22:30:54 +05:30
parent 58bd19ecd1
commit 1e24b52f3e
18 changed files with 2415 additions and 19 deletions
+123
View File
@@ -0,0 +1,123 @@
#!/usr/bin/env node
/**
* Builds a self-contained MarkItDown binary into bin/<platform>/markitdown
* so the app can bundle Microsoft's markitdown (MIT) without requiring users
* to have Python installed. Run via `npm run bundle:markitdown` before
* packaging (skipped automatically when the binary already exists).
*
* How: creates a throwaway virtualenv, pip-installs markitdown (with the
* document extras) + PyInstaller, then freezes the CLI into a onefile
* executable. Heavy optional ML deps (torch/scipy/pandas/…) are excluded —
* the resulting ~75MB binary covers PDF/DOCX/PPTX/XLSX/Outlook/HTML/EPUB/
* images/CSV/JSON/XML/ZIP. Audio transcription and OCR intentionally stay
* unbundled (multi-GB); the app falls back to a system `markitdown[all]`
* install for those.
*
* Requires on the build machine: python3 (with venv + pip) — on Debian/
* Ubuntu that is `sudo apt install python3 python3-venv`.
*/
const fs = require('fs');
const os = require('os');
const path = require('path');
const { execFileSync } = require('child_process');
const platform = process.platform; // linux | win32 | darwin
const exeName = platform === 'win32' ? 'markitdown.exe' : 'markitdown';
const binDir = path.join(__dirname, '..', 'bin', platform);
const destFile = path.join(binDir, exeName);
const workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mid-bundle-'));
/** Run a command, inheriting stdout/stderr so build progress is visible. */
function run(cmd, args, opts = {}) {
execFileSync(cmd, args, { stdio: 'inherit', ...opts });
}
function pythonBin(venvDir, name) {
return path.join(venvDir, platform === 'win32' ? 'Scripts' : 'bin', name);
}
function main() {
if (fs.existsSync(destFile)) {
console.log(`[bundle-markitdown] ${destFile} already present — skipping.`);
return;
}
const venvDir = path.join(workDir, 'venv');
console.log('[bundle-markitdown] Creating virtualenv...');
run('python3', ['-m', 'venv', venvDir]);
console.log('[bundle-markitdown] Installing markitdown + PyInstaller...');
run(pythonBin(venvDir, 'pip'), [
'install', '--quiet', '--upgrade', 'pip',
]);
run(pythonBin(venvDir, 'pip'), [
'install', '--quiet',
// Document extras only — [all] would pull the ML stack (torch etc.)
'markitdown[pdf,docx,pptx,xlsx,outlook]',
'pyinstaller',
]);
// PyInstaller entrypoint mirroring the markitdown CLI
const entry = path.join(workDir, 'entry.py');
fs.writeFileSync(
entry,
[
'# Generated by scripts/bundle-markitdown.js — frozen CLI entrypoint',
'from markitdown.__main__ import main',
"if __name__ == '__main__':",
' main()',
'',
].join('\n')
);
console.log('[bundle-markitdown] Freezing with PyInstaller (this takes a minute)...');
run(pythonBin(venvDir, platform === 'win32' ? 'pyinstaller.exe' : 'pyinstaller'), [
'--onefile',
'--name', 'markitdown',
'--strip',
'--clean',
// magika ships its ML model as data files — must be collected into the bundle
'--collect-data', 'magika',
// Heavy optional scientific/ML stacks markitdown never imports for
// document conversion — excluding them keeps the binary ~75MB
'--exclude-module', 'torch',
'--exclude-module', 'scipy',
'--exclude-module', 'pandas',
'--exclude-module', 'matplotlib',
'--exclude-module', 'tkinter',
'--exclude-module', 'IPython',
'--exclude-module', 'pytest',
'--distpath', path.join(workDir, 'dist'),
'--workpath', path.join(workDir, 'build'),
'--specpath', workDir,
entry,
]);
fs.mkdirSync(binDir, { recursive: true });
fs.copyFileSync(path.join(workDir, 'dist', exeName), destFile);
if (platform !== 'win32') fs.chmodSync(destFile, 0o755);
// Smoke test before declaring success
const version = execFileSync(destFile, ['--version'], { encoding: 'utf-8' }).trim();
if (!version.toLowerCase().includes('markitdown')) {
throw new Error(`Unexpected --version output: ${version}`);
}
console.log(`[bundle-markitdown] Built ${destFile} (${version})`);
}
try {
main();
} catch (err) {
console.error('[bundle-markitdown] FAILED:', err.message);
console.error(
'\nPrerequisites: python3 with venv+pip on the PATH.\n' +
' Debian/Ubuntu: sudo apt install python3 python3-venv\n' +
' macOS: brew install python\n' +
'The app still works without the bundled binary — it falls back to a\n' +
'system-installed markitdown or python -m markitdown at runtime.'
);
process.exit(1);
} finally {
fs.rmSync(workDir, { recursive: true, force: true });
}