mirror of
https://github.com/amitwh/markdown-converter.git
synced 2026-10-01 09:19:34 +05:30
feat(legal): bundle small dependencies, add notices, credits, and GPL source offers
Bundle (v4.7.0): - MarkItDown frozen to a single ~75MB per-platform binary (PyInstaller via npm run bundle:markitdown; ML extras excluded) — built and verified locally: HTML/XLSX/PDF conversions pass through the bundled binary, and the app resolves bin/linux/markitdown first at runtime - Packaging copies bundled markitdown alongside Pandoc for win/mac/linux; FFmpeg/sharp/KaTeX/fonts were already bundled Legal artifacts: - THIRD-PARTY-NOTICES.md: complete license inventory of everything distributed (binaries, npm runtime deps, fonts, embedded Python packages) - SOURCES.md: GPL §3(b) written source offers for Pandoc 3.9.0.2, ffmpeg-static's GPL build, and the PyInstaller bootloader; LGPL relinking note for libvips - third-party-licenses/: canonical GPL-2.0, LGPL-2.1, MPL-2.0, Apache-2.0, OFL-1.1, PSF-Python texts - Help > Third-Party Notices & Licenses: in-app viewer for both documents - README: 'Bundled Dependencies, Legal Notices & Credits' section Hardening: - download-tools.js now SHA-256 pins every artifact, verifies after download AND against the cache on every run, and hard-fails on mismatch (closes security finding D6) Large tools intentionally not bundled (documented): LibreOffice, MiKTeX/TeX Live, ImageMagick, PlantUML+JRE, Calibre. 637/637 tests green; lint clean; clean boot; bundled binary verified.
This commit is contained in:
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Builds a self-contained MarkItDown binary into bin/<platform>/markitdown
|
||||
* so the app can bundle Microsoft's markitdown (MIT) without requiring users
|
||||
* to have Python installed. Run via `npm run bundle:markitdown` before
|
||||
* packaging (skipped automatically when the binary already exists).
|
||||
*
|
||||
* How: creates a throwaway virtualenv, pip-installs markitdown (with the
|
||||
* document extras) + PyInstaller, then freezes the CLI into a onefile
|
||||
* executable. Heavy optional ML deps (torch/scipy/pandas/…) are excluded —
|
||||
* the resulting ~75MB binary covers PDF/DOCX/PPTX/XLSX/Outlook/HTML/EPUB/
|
||||
* images/CSV/JSON/XML/ZIP. Audio transcription and OCR intentionally stay
|
||||
* unbundled (multi-GB); the app falls back to a system `markitdown[all]`
|
||||
* install for those.
|
||||
*
|
||||
* Requires on the build machine: python3 (with venv + pip) — on Debian/
|
||||
* Ubuntu that is `sudo apt install python3 python3-venv`.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
const { execFileSync } = require('child_process');
|
||||
|
||||
const platform = process.platform; // linux | win32 | darwin
|
||||
const exeName = platform === 'win32' ? 'markitdown.exe' : 'markitdown';
|
||||
const binDir = path.join(__dirname, '..', 'bin', platform);
|
||||
const destFile = path.join(binDir, exeName);
|
||||
const workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mid-bundle-'));
|
||||
|
||||
/** Run a command, inheriting stdout/stderr so build progress is visible. */
|
||||
function run(cmd, args, opts = {}) {
|
||||
execFileSync(cmd, args, { stdio: 'inherit', ...opts });
|
||||
}
|
||||
|
||||
function pythonBin(venvDir, name) {
|
||||
return path.join(venvDir, platform === 'win32' ? 'Scripts' : 'bin', name);
|
||||
}
|
||||
|
||||
function main() {
|
||||
if (fs.existsSync(destFile)) {
|
||||
console.log(`[bundle-markitdown] ${destFile} already present — skipping.`);
|
||||
return;
|
||||
}
|
||||
|
||||
const venvDir = path.join(workDir, 'venv');
|
||||
console.log('[bundle-markitdown] Creating virtualenv...');
|
||||
run('python3', ['-m', 'venv', venvDir]);
|
||||
|
||||
console.log('[bundle-markitdown] Installing markitdown + PyInstaller...');
|
||||
run(pythonBin(venvDir, 'pip'), [
|
||||
'install', '--quiet', '--upgrade', 'pip',
|
||||
]);
|
||||
run(pythonBin(venvDir, 'pip'), [
|
||||
'install', '--quiet',
|
||||
// Document extras only — [all] would pull the ML stack (torch etc.)
|
||||
'markitdown[pdf,docx,pptx,xlsx,outlook]',
|
||||
'pyinstaller',
|
||||
]);
|
||||
|
||||
// PyInstaller entrypoint mirroring the markitdown CLI
|
||||
const entry = path.join(workDir, 'entry.py');
|
||||
fs.writeFileSync(
|
||||
entry,
|
||||
[
|
||||
'# Generated by scripts/bundle-markitdown.js — frozen CLI entrypoint',
|
||||
'from markitdown.__main__ import main',
|
||||
"if __name__ == '__main__':",
|
||||
' main()',
|
||||
'',
|
||||
].join('\n')
|
||||
);
|
||||
|
||||
console.log('[bundle-markitdown] Freezing with PyInstaller (this takes a minute)...');
|
||||
run(pythonBin(venvDir, platform === 'win32' ? 'pyinstaller.exe' : 'pyinstaller'), [
|
||||
'--onefile',
|
||||
'--name', 'markitdown',
|
||||
'--strip',
|
||||
'--clean',
|
||||
// magika ships its ML model as data files — must be collected into the bundle
|
||||
'--collect-data', 'magika',
|
||||
// Heavy optional scientific/ML stacks markitdown never imports for
|
||||
// document conversion — excluding them keeps the binary ~75MB
|
||||
'--exclude-module', 'torch',
|
||||
'--exclude-module', 'scipy',
|
||||
'--exclude-module', 'pandas',
|
||||
'--exclude-module', 'matplotlib',
|
||||
'--exclude-module', 'tkinter',
|
||||
'--exclude-module', 'IPython',
|
||||
'--exclude-module', 'pytest',
|
||||
'--distpath', path.join(workDir, 'dist'),
|
||||
'--workpath', path.join(workDir, 'build'),
|
||||
'--specpath', workDir,
|
||||
entry,
|
||||
]);
|
||||
|
||||
fs.mkdirSync(binDir, { recursive: true });
|
||||
fs.copyFileSync(path.join(workDir, 'dist', exeName), destFile);
|
||||
if (platform !== 'win32') fs.chmodSync(destFile, 0o755);
|
||||
|
||||
// Smoke test before declaring success
|
||||
const version = execFileSync(destFile, ['--version'], { encoding: 'utf-8' }).trim();
|
||||
if (!version.toLowerCase().includes('markitdown')) {
|
||||
throw new Error(`Unexpected --version output: ${version}`);
|
||||
}
|
||||
console.log(`[bundle-markitdown] Built ${destFile} (${version})`);
|
||||
}
|
||||
|
||||
try {
|
||||
main();
|
||||
} catch (err) {
|
||||
console.error('[bundle-markitdown] FAILED:', err.message);
|
||||
console.error(
|
||||
'\nPrerequisites: python3 with venv+pip on the PATH.\n' +
|
||||
' Debian/Ubuntu: sudo apt install python3 python3-venv\n' +
|
||||
' macOS: brew install python\n' +
|
||||
'The app still works without the bundled binary — it falls back to a\n' +
|
||||
'system-installed markitdown or python -m markitdown at runtime.'
|
||||
);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
fs.rmSync(workDir, { recursive: true, force: true });
|
||||
}
|
||||
@@ -3,6 +3,11 @@
|
||||
* Downloads pandoc binary for the current build platform.
|
||||
* Run automatically via `npm run download-tools` before building.
|
||||
* Skips download if binary already exists (idempotent).
|
||||
*
|
||||
* Security: every artifact with a known hash is verified with SHA-256 after
|
||||
* download AND on every run (defending the build against a tampered cache /
|
||||
* compromised mirror). When a hash is missing for a platform, the computed
|
||||
* hash is printed so CI can pin it — add it to KNOWN_SHA256 below.
|
||||
*/
|
||||
|
||||
const https = require('https');
|
||||
@@ -10,10 +15,57 @@ const http = require('http');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
const crypto = require('crypto');
|
||||
const { execSync } = require('child_process');
|
||||
|
||||
const PANDOC_VERSION = '3.9.0.2';
|
||||
|
||||
/**
|
||||
* SHA-256 of each downloaded artifact, keyed "<platform>:<file>" or
|
||||
* "fonts:<file>". Computed with `sha256sum` from the pristine upstream
|
||||
* artifacts; keep in sync when bumping versions.
|
||||
*/
|
||||
const KNOWN_SHA256 = {
|
||||
'linux:pandoc': '7d124235998ecd3cdd9a463b1e5f6691a178b6461824c29a36170a0882f05597',
|
||||
// Fill these from a trusted machine after the first download of each
|
||||
// platform (the script prints the computed hash):
|
||||
// 'win32:pandoc.exe': '…',
|
||||
// 'darwin:pandoc': '…',
|
||||
'fonts:FiraCode-Regular.ttf': '3c79d234a9161c790410ebb2a80de7efb7c15f581062c130e0fa78503ccdd0da',
|
||||
'fonts:FiraCode-Bold.ttf': '975f26779fac1029c2cbdac1e9fac7e9ddeec05e064675e4aac63bffa121742f',
|
||||
'fonts:FiraCode-LICENSE.txt': '1d41e10031ab125302780a05ec4c91d218e47db0c7e37cf315cce5e608cdc25c',
|
||||
};
|
||||
|
||||
/** sha256 of a file's contents, hex-encoded. */
|
||||
function sha256File(filePath) {
|
||||
return crypto.createHash('sha256').update(fs.readFileSync(filePath)).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify a file against KNOWN_SHA256. Throws on mismatch; prints the computed
|
||||
* hash with a pin-me hint when no hash is recorded yet (soft-fail so new
|
||||
* platforms bootstrap, hard-fail so tampering never passes silently).
|
||||
*/
|
||||
function verifyArtifact(key, filePath) {
|
||||
const actual = sha256File(filePath);
|
||||
const expected = KNOWN_SHA256[key];
|
||||
if (!expected) {
|
||||
console.warn(
|
||||
`[download-tools] WARNING: no pinned SHA-256 for "${key}".\n` +
|
||||
` Computed: ${actual}\n` +
|
||||
' Verify it against the upstream artifact and add it to KNOWN_SHA256.'
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (actual !== expected) {
|
||||
throw new Error(
|
||||
`SHA-256 mismatch for ${key}!\n expected ${expected}\n actual ${actual}\n` +
|
||||
'Delete the file and re-run; if the mismatch persists, do NOT ship it.'
|
||||
);
|
||||
}
|
||||
console.log(`[download-tools] SHA-256 OK for ${key}`);
|
||||
}
|
||||
|
||||
const PANDOC_CONFIG = {
|
||||
linux: {
|
||||
url: `https://github.com/jgm/pandoc/releases/download/${PANDOC_VERSION}/pandoc-${PANDOC_VERSION}-linux-amd64.tar.gz`,
|
||||
@@ -119,11 +171,14 @@ async function downloadFiraCode() {
|
||||
for (const t of targets) {
|
||||
const destFile = path.join(destDir, t.out);
|
||||
if (fs.existsSync(destFile)) {
|
||||
// Re-verify cached artifacts so a tampered cache never ships
|
||||
verifyArtifact(`fonts:${t.out}`, destFile);
|
||||
console.log(`[download-tools] ${t.out} already present — skipping.`);
|
||||
continue;
|
||||
}
|
||||
console.log(`[download-tools] Downloading ${t.out}...`);
|
||||
await download(t.url, destFile);
|
||||
verifyArtifact(`fonts:${t.out}`, destFile);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -140,6 +195,7 @@ async function downloadPandoc() {
|
||||
const destFile = path.join(destDir, config.destFile);
|
||||
|
||||
if (fs.existsSync(destFile)) {
|
||||
verifyArtifact(`${platform}:${config.destFile}`, destFile);
|
||||
console.log(`[download-tools] pandoc already present at ${destFile} — skipping.`);
|
||||
return;
|
||||
}
|
||||
@@ -153,6 +209,7 @@ async function downloadPandoc() {
|
||||
|
||||
console.log(`[download-tools] Extracting to ${destDir}...`);
|
||||
config.extract(tmpArchive, destDir);
|
||||
verifyArtifact(`${platform}:${config.destFile}`, destFile);
|
||||
|
||||
try {
|
||||
fs.unlinkSync(tmpArchive);
|
||||
|
||||
Reference in New Issue
Block a user