markdown-converter/scripts/bundle-markitdown.js

#!/usr/bin/env node
/**
 * Builds a self-contained MarkItDown binary into bin/<platform>/markitdown
 * so the app can bundle Microsoft's markitdown (MIT) without requiring users
 * to have Python installed. Run via `npm run bundle:markitdown` before
 * packaging (skipped automatically when the binary already exists).
 *
 * How: creates a throwaway virtualenv, pip-installs markitdown (with the
 * document extras) + PyInstaller, then freezes the CLI into a onefile
 * executable. Heavy optional ML deps (torch/scipy/pandas/…) are excluded —
 * the resulting ~75MB binary covers PDF/DOCX/PPTX/XLSX/Outlook/HTML/EPUB/
 * images/CSV/JSON/XML/ZIP. Audio transcription and OCR intentionally stay
 * unbundled (multi-GB); the app falls back to a system `markitdown[all]`
 * install for those.
 *
 * Requires on the build machine: python3 (with venv + pip) — on Debian/
 * Ubuntu that is `sudo apt install python3 python3-venv`.
 */

const fs = require('fs');
const os = require('os');
const path = require('path');
const { execFileSync } = require('child_process');

const platform = process.platform; // linux | win32 | darwin
const exeName = platform === 'win32' ? 'markitdown.exe' : 'markitdown';
const binDir = path.join(__dirname, '..', 'bin', platform);
const destFile = path.join(binDir, exeName);
const workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mid-bundle-'));

/** Run a command, inheriting stdout/stderr so build progress is visible. */
function run(cmd, args, opts = {}) {
  execFileSync(cmd, args, { stdio: 'inherit', ...opts });
}

function pythonBin(venvDir, name) {
  return path.join(venvDir, platform === 'win32' ? 'Scripts' : 'bin', name);
}

function main() {
  if (fs.existsSync(destFile)) {
    console.log(`[bundle-markitdown] ${destFile} already present — skipping.`);
    return;
  }

  const venvDir = path.join(workDir, 'venv');
  console.log('[bundle-markitdown] Creating virtualenv...');
  run('python3', ['-m', 'venv', venvDir]);

  console.log('[bundle-markitdown] Installing markitdown + PyInstaller...');
  run(pythonBin(venvDir, 'pip'), [
    'install', '--quiet', '--upgrade', 'pip',
  ]);
  run(pythonBin(venvDir, 'pip'), [
    'install', '--quiet',
    // Document extras only — [all] would pull the ML stack (torch etc.)
    'markitdown[pdf,docx,pptx,xlsx,outlook]',
    'pyinstaller',
  ]);

  // PyInstaller entrypoint mirroring the markitdown CLI
  const entry = path.join(workDir, 'entry.py');
  fs.writeFileSync(
    entry,
    [
      '# Generated by scripts/bundle-markitdown.js — frozen CLI entrypoint',
      'from markitdown.__main__ import main',
      "if __name__ == '__main__':",
      '    main()',
      '',
    ].join('\n')
  );

  console.log('[bundle-markitdown] Freezing with PyInstaller (this takes a minute)...');
  run(pythonBin(venvDir, platform === 'win32' ? 'pyinstaller.exe' : 'pyinstaller'), [
    '--onefile',
    '--name', 'markitdown',
    '--strip',
    '--clean',
    // magika ships its ML model as data files — must be collected into the bundle
    '--collect-data', 'magika',
    // Heavy optional scientific/ML stacks markitdown never imports for
    // document conversion — excluding them keeps the binary ~75MB
    '--exclude-module', 'torch',
    '--exclude-module', 'scipy',
    '--exclude-module', 'pandas',
    '--exclude-module', 'matplotlib',
    '--exclude-module', 'tkinter',
    '--exclude-module', 'IPython',
    '--exclude-module', 'pytest',
    '--distpath', path.join(workDir, 'dist'),
    '--workpath', path.join(workDir, 'build'),
    '--specpath', workDir,
    entry,
  ]);

  fs.mkdirSync(binDir, { recursive: true });
  fs.copyFileSync(path.join(workDir, 'dist', exeName), destFile);
  if (platform !== 'win32') fs.chmodSync(destFile, 0o755);

  // Smoke test before declaring success
  const version = execFileSync(destFile, ['--version'], { encoding: 'utf-8' }).trim();
  if (!version.toLowerCase().includes('markitdown')) {
    throw new Error(`Unexpected --version output: ${version}`);
  }
  console.log(`[bundle-markitdown] Built ${destFile} (${version})`);
}

try {
  main();
} catch (err) {
  console.error('[bundle-markitdown] FAILED:', err.message);
  console.error(
    '\nPrerequisites: python3 with venv+pip on the PATH.\n' +
      '  Debian/Ubuntu: sudo apt install python3 python3-venv\n' +
      '  macOS: brew install python\n' +
      'The app still works without the bundled binary — it falls back to a\n' +
      'system-installed markitdown or python -m markitdown at runtime.'
  );
  process.exit(1);
} finally {
  fs.rmSync(workDir, { recursive: true, force: true });
}