From 58bd19ecd13b75af1db25a1670f983a90211d839 Mon Sep 17 00:00:00 2001 From: Amit Haridas Date: Sat, 5 Sep 2026 22:10:35 +0530 Subject: [PATCH] =?UTF-8?q?feat(import):=20embed=20Microsoft=20MarkItDown?= =?UTF-8?q?=20for=20any-file=20=E2=86=92=20Markdown=20import?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - File → Import with MarkItDown (Any Format)…: PDF, DOCX, PPTX, XLSX, Outlook .msg/.eml, EPUB, images, CSV/JSON/XML, ZIP (audio/OCR via the [all] extras) — verified live against HTML, XLSX (our own exporter's output), and PDF fixtures - Command auto-resolution with caching: markitdown binary → python -m markitdown → python3 -m markitdown - SEC-1 argv discipline (execFile only, user paths never through a shell), 50MB cap, 120s timeout, sanitized errors that surface markitdown's own "pip install 'markitdown[pdf]'" hints for missing format extras - Output lands next to the source as .md (numeric suffix, never overwrites) and opens in a new tab; markitdown:available/convert IPC allowlisted for renderer flows - Help → Dependencies lists MarkItDown; README/UPDATES updated (v4.6.1) 12 new tests (629 green); lint clean; clean app boot --- README.md | 4 +- UPDATES.md | 31 ++++++ package-lock.json | 4 +- package.json | 2 +- src/main.js | 103 +++++++++++++++++++ src/main/MarkItDown.js | 157 +++++++++++++++++++++++++++++ src/preload.js | 4 + tests/main/MarkItDown.test.js | 179 ++++++++++++++++++++++++++++++++++ 8 files changed, 480 insertions(+), 4 deletions(-) create mode 100644 src/main/MarkItDown.js create mode 100644 tests/main/MarkItDown.test.js diff --git a/README.md b/README.md index ec3e02f..3a91031 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ A powerful cross-platform Markdown editor and document converter powered by Pand - **ASCII Art Generator** - Create text banners and diagrams - **Word templates** - Use custom Word templates for enhanced exports - **Import documents** - Import from 30+ formats (DOCX, PDF, HTML, etc.) +- **MarkItDown import** - Any file → Markdown via [Microsoft MarkItDown](https://github.com/microsoft/markitdown): PDF, DOCX, PPTX, XLSX, Outlook .msg, EPUB, images, ZIP (audio/OCR with the `[all]` extras) - **Excel export** - Markdown tables to native .xlsx workbooks (one sheet per table) - **AI Assistant** - Multi-provider AI help (OpenAI/Anthropic/Ollama/LM Studio): chat panel, summarize/improve/translate commands, grammar proofreading - **Inline comments** - Anchor-based document comments in `.comments/` sidecars with F8 navigation @@ -67,6 +68,7 @@ A powerful cross-platform Markdown editor and document converter powered by Pand ### Prerequisites - [Node.js](https://nodejs.org/) (v16 or later) - [Pandoc](https://pandoc.org/installing.html) (required for export functionality) +- Optional: [MarkItDown](https://github.com/microsoft/markitdown) (`pip install "markitdown[all]"`) for any-file → Markdown import ### Install Dependencies ```bash @@ -181,4 +183,4 @@ Amit Haridas (amit.wh@gmail.com) ## Version -v4.6.0 +v4.6.1 diff --git a/UPDATES.md b/UPDATES.md index d96e396..25e2e4a 100644 --- a/UPDATES.md +++ b/UPDATES.md @@ -1,5 +1,36 @@ # PanConverter - Updates & Changelog +## Version 4.6.1 (2026-09-05) + +### New Features +- **MarkItDown import** — "File → Import with MarkItDown (Any Format)…" embeds + Microsoft's [markitdown](https://github.com/microsoft/markitdown) (MIT) as an + any-file → Markdown path: PDF, DOCX, PPTX, XLSX, Outlook .msg/.eml, EPUB, + images, CSV/JSON/XML, ZIP; audio transcription and OCR with the `[all]` extras + - Command auto-resolution: `markitdown` binary, then `python -m markitdown` / + `python3 -m markitdown` (probed once, cached) + - Same SEC-1 argv discipline as Pandoc (execFile only, paths never through a shell), + 50MB input cap, 120s timeout, path-sanitized errors that surface markitdown's + actionable `pip install 'markitdown[...]'` hints + - Output written next to the source as `.md` (numeric suffix instead of + overwriting) and opened in a new tab; `markitdown:available` / `markitdown:convert` + IPC for future renderer flows +- **AI Assistant: Anthropic-compatible provider** — any base URL speaking the + Anthropic messages schema (LiteLLM proxies, Bedrock gateways, local servers); + x-api-key + Bearer auth, keyless proxies supported, tolerates bases with or + without a trailing `/v1` + +### Bug Fixes +- File → Open PDF crashed the PDF editor (null operation matched no section; now + defaults to Merge) +- Backlinks panel required the wrong module path (failed at registration) +- writing-studio engines/panels now await their IPC-backed settings/file backends + (eliminates `JSON.parse("[object Promise]")` crashes) +- Manuscript panel's window.prompt (unsupported in Electron) replaced with an + inline dialog; collaboration comment store made async to match its IO + +--- + ## Version 4.6.0 (2026-09-05) ### New Features diff --git a/package-lock.json b/package-lock.json index 42843b2..98ba00f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "markdown-converter", - "version": "4.6.0", + "version": "4.6.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "markdown-converter", - "version": "4.6.0", + "version": "4.6.1", "license": "MIT", "dependencies": { "@cantoo/pdf-lib": "^2.9.1", diff --git a/package.json b/package.json index bbe4ed9..c9e02d7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "markdown-converter", - "version": "4.6.0", + "version": "4.6.1", "description": "Professional Markdown editor and universal file converter with PDF editing, batch processing, and syntax highlighting", "main": "src/main.js", "scripts": { diff --git a/src/main.js b/src/main.js index 994cb95..df45e15 100644 --- a/src/main.js +++ b/src/main.js @@ -840,6 +840,12 @@ function createMenu() { accelerator: 'CmdOrCtrl+I', click: importDocument, }, + { + // Microsoft MarkItDown: any file → Markdown (PDF/DOCX/PPTX/XLSX/ + // MSG/EPUB/images/ZIP/…, audio/OCR with the [all] extras) + label: 'Import with MarkItDown (Any Format)...', + click: importWithMarkItDown, + }, { label: 'Export', submenu: [ @@ -1763,6 +1769,12 @@ function showDependenciesDialog() { https://miktex.org/download +
+
MarkItDown Optional
+
Microsoft's any-file-to-Markdown importer (PDF, DOCX, PPTX, XLSX, Outlook .msg, EPUB, images, ZIP). Install with: pip install "markitdown[all]"
+ https://github.com/microsoft/markitdown +
+

Bundled Libraries

@@ -3717,6 +3729,97 @@ function importDocument() { }); } } + +// ============================================ +// MarkItDown import (Microsoft markitdown, any file → Markdown) +// ============================================ +const MarkItDown = require('./main/MarkItDown'); + +// Probed once on first use; {command, argsPrefix, version} or null +let markItDownResolved = undefined; // undefined = not probed yet + +/** Resolve (and cache) the markitdown command via the bridge module. */ +async function getMarkItDown() { + if (markItDownResolved === undefined) { + markItDownResolved = await MarkItDown.resolveMarkItDown(require('child_process').execFile); + } + return markItDownResolved; +} + +/** IPC: availability probe for renderer hints (no version spam, cached). */ +ipcMain.handle('markitdown:available', async () => { + const resolved = await getMarkItDown(); + return { + available: Boolean(resolved), + version: resolved?.version || null, + // How the tool was found, e.g. "markitdown" or "python3 -m markitdown" + via: resolved ? [resolved.command, ...resolved.argsPrefix].join(' ') : null, + }; +}); + +/** IPC: convert one file to markdown (renderer-driven flows). */ +ipcMain.handle('markitdown:convert', async (_event, { path: inputPath } = {}) => { + const validation = validatePath(inputPath); + if (!validation.valid) throw new Error('Invalid file path'); + const stats = fs.statSync(validation.resolved); + if (stats.size > MAX_FILE_SIZE) { + throw new Error(`File exceeds the ${MAX_FILE_SIZE_MB}MB size limit.`); + } + const resolved = await getMarkItDown(); + return MarkItDown.convertToMarkdown(validation.resolved, { resolved }); +}); + +/** + * Menu flow: pick any file, convert with markitdown, write .md next to + * the source (mirroring importDocument's UX), and open it in a new tab. + * Existing outputs are never overwritten — a numeric suffix is appended. + */ +async function importWithMarkItDown() { + const files = dialog.showOpenDialogSync(mainWindow, { + properties: ['openFile'], + title: 'Import with MarkItDown (any format)', + }); + if (!files || !files[0]) return; + const inputFile = files[0]; + + try { + const stats = fs.statSync(inputFile); + if (stats.size > MAX_FILE_SIZE) { + dialog.showErrorBox('File Too Large', `File exceeds the ${MAX_FILE_SIZE_MB}MB size limit.`); + return; + } + + const resolved = await getMarkItDown(); + const { content } = await MarkItDown.convertToMarkdown(inputFile, { resolved }); + + // .md, then -1.md, -2.md, … when it already exists + const base = inputFile.replace(/\.[^/.]+$/, ''); + let outputFile = `${base}.md`; + for (let i = 1; fs.existsSync(outputFile); i++) outputFile = `${base}-${i}.md`; + fs.writeFileSync(outputFile, content, 'utf-8'); + + currentFile = outputFile; + mainWindow.webContents.send('file-opened', { path: outputFile, content }); + dialog.showMessageBox(mainWindow, { + type: 'info', + title: 'Import Complete', + message: `Imported as ${path.basename(outputFile)} via MarkItDown\n\nOriginal: ${path.basename(inputFile)}`, + buttons: ['OK'], + }); + } catch (error) { + dialog.showErrorBox( + 'MarkItDown Import', + sanitizeErrorMessage( + (error.code === 'not_installed' + ? error.message + : `Import failed: ${error.message}`) + + '\n\nMarkItDown is an optional Python tool from Microsoft (MIT):\n' + + ' pip install "markitdown[all]"' + ) + ); + } +} + function setTheme(theme) { store.set('theme', theme); mainWindow.webContents.send('theme-changed', theme); diff --git a/src/main/MarkItDown.js b/src/main/MarkItDown.js new file mode 100644 index 0000000..2b93adc --- /dev/null +++ b/src/main/MarkItDown.js @@ -0,0 +1,157 @@ +/** + * MarkItDown bridge — "any file → Markdown" import via Microsoft's + * markitdown Python tool (https://github.com/microsoft/markitdown, MIT). + * + * markitdown is a Python CLI, so like Pandoc/LibreOffice/FFmpeg it is used + * when installed rather than bundled: the module probes for it once + * (`markitdown` binary, then `python -m markitdown` / `python3 -m markitdown`) + * and caches the resolved command. Every invocation goes through an argv + * array via execFile — user-controlled paths are never passed through a + * shell (same SEC-1 discipline as the Pandoc path). + * + * Supported by markitdown (core install): PDF, DOCX, PPTX, XLSX, Outlook + * .msg/.eml, HTML, EPUB, images (EXIF), CSV/JSON/XML, ZIP archives, YouTube + * URLs. Audio transcription and image OCR need the `[all]` extras: + * pip install 'markitdown[all]' + * + * The execFile runner is injectable so tests can stub process spawning. + * + * @module MarkItDown + */ + +const CONVERT_TIMEOUT_MS = 120000; +const MAX_OUTPUT_BUFFER = 20 * 1024 * 1024; + +/** + * Candidate command templates probed in order. `argsPrefix` is prepended to + * the user path when invoking (e.g. ['-m', 'markitdown'] for module-style + * invocation through a python launcher). + */ +const COMMAND_CANDIDATES = [ + { command: 'markitdown', argsPrefix: [] }, + { command: process.platform === 'win32' ? 'python' : 'python3', argsPrefix: ['-m', 'markitdown'] }, + { command: 'python3', argsPrefix: ['-m', 'markitdown'] }, +]; + +/** Run one probe: `--version` exits 0 when the tool is importable. */ +function probeCandidate(runner, candidate) { + return new Promise((resolve) => { + runner( + candidate.command, + [...candidate.argsPrefix, '--version'], + { timeout: 15000 }, + (error, stdout) => { + if (error) return resolve(null); + const match = /(\d+\.\d+(?:\.\d+)?)/.exec(String(stdout || '')); + resolve({ + command: candidate.command, + argsPrefix: candidate.argsPrefix, + version: match ? match[1] : null, + }); + } + ); + }); +} + +/** + * Resolve the markitdown invocation. Probes candidates in order and returns + * the first that answers `--version`, or null when none is installed. + * + * @param {Function} runner execFile-style (cmd, args, opts, cb) + * @returns {Promise<{command: string, argsPrefix: string[], version: string|null}|null>} + */ +async function resolveMarkItDown(runner) { + for (const candidate of COMMAND_CANDIDATES) { + const resolved = await probeCandidate(runner, candidate); + if (resolved) return resolved; + } + return null; +} + +/** + * Convert any supported file to Markdown. + * + * @param {string} inputPath Absolute path to the source file + * @param {object} [options] + * @param {Function} [options.runner] injectable execFile (defaults to child_process) + * @param {Function} [options.pathUtil] injected path module + * @param {{command: string, argsPrefix: string[]}} [options.resolved] skip + * probing when the caller already knows the command (main caches it) + * @returns {Promise<{content: string}>} Markdown printed by markitdown on stdout + * @throws {Error} with a user-safe message when the tool is missing or fails + */ +async function convertToMarkdown(inputPath, options = {}) { + const runner = options.runner || require('child_process').execFile; + const pathUtil = options.pathUtil || require('path'); + + // Validate the path BEFORE probing for the tool: a bad path is the user's + // most actionable error and shouldn't be masked by an install hint + if (typeof inputPath !== 'string' || !pathUtil.isAbsolute(inputPath)) { + const err = new Error('MarkItDown import needs an absolute file path.'); + err.code = 'bad_path'; + throw err; + } + + const resolved = options.resolved || (await resolveMarkItDown(runner)); + if (!resolved) { + const err = new Error( + 'MarkItDown is not installed. Install it with:\n\npip install "markitdown[all]"\n\n' + + '(or the lighter core: pip install markitdown)' + ); + err.code = 'not_installed'; + throw err; + } + + return new Promise((resolve, reject) => { + // markitdown prints the converted Markdown to stdout; keep everything in + // argv so the path is never re-interpreted by a shell. + runner( + resolved.command, + [...resolved.argsPrefix, inputPath], + { timeout: CONVERT_TIMEOUT_MS, maxBuffer: MAX_OUTPUT_BUFFER }, + (error, stdout, stderr) => { + if (error) { + // Strip absolute paths, then keep the first informative line from + // either the error or stderr — markitdown writes tracebacks there + // (e.g. a missing optional dependency like pdfminer.six for PDFs). + const safe = (text) => + String(text || '') + .replace(/[A-Z]:\\[^\s"']+/gi, '(path)') + .replace(/\/(?:home|Users|tmp|mnt)\/[^\s"']+/g, '(path)'); + // Prefer the LAST informative stderr line: markitdown ends its + // output with the actionable install hint ("pip install + // 'markitdown[pdf]'"), while the first lines are a traceback. + const stderrLines = safe(stderr).split('\n').filter((l) => l.trim().length > 0); + const detail = + stderrLines[stderrLines.length - 1] || + safe(error.message).split('\n')[0] || + 'unknown error'; + const err2 = new Error(`MarkItDown conversion failed: ${detail}`); + err2.code = String(error.code || '').startsWith('ETIMEDOUT') ? 'timeout' : 'failed'; + err2.stderr = safe(stderr).slice(0, 2000); + reject(err2); + return; + } + const content = String(stdout || ''); + if (!content.trim()) { + const err3 = new Error( + 'MarkItDown produced no output — the file type may be unsupported ' + + '(core install covers PDF/DOCX/PPTX/XLSX/MSG/HTML/EPUB/images/CSV/JSON/XML/ZIP; ' + + "audio/OCR need pip install 'markitdown[all]')." + ); + err3.code = 'empty_output'; + reject(err3); + return; + } + // Strip a UTF-8 BOM if present so downstream markdown tooling is happy + resolve({ content: content.replace(/^\uFEFF/, '') }); + } + ); + }); +} + +module.exports = { + resolveMarkItDown, + convertToMarkdown, + COMMAND_CANDIDATES, +}; diff --git a/src/preload.js b/src/preload.js index 3bcd53f..d3afbd2 100644 --- a/src/preload.js +++ b/src/preload.js @@ -168,6 +168,10 @@ const ALLOWED_SEND_CHANNELS = [ 'plantuml:available', 'plantuml:render', + // MarkItDown import (optional Python CLI) + 'markitdown:available', + 'markitdown:convert', + // Quick Note scratchpad 'quick-note:save', diff --git a/tests/main/MarkItDown.test.js b/tests/main/MarkItDown.test.js new file mode 100644 index 0000000..af04101 --- /dev/null +++ b/tests/main/MarkItDown.test.js @@ -0,0 +1,179 @@ +/** + * @jest-environment node + * + * MarkItDown bridge tests with a stubbed execFile — no Python needed. + * Covers command resolution order, argv-array invocation (SEC-1), stdout + * capture, error surfacing, and path validation. + */ +const os = require('os'); +const path = require('path'); +const { resolveMarkItDown, convertToMarkdown, COMMAND_CANDIDATES } = require('../../src/main/MarkItDown'); +const { setImmediate } = require('timers'); + +/** + * Build a runner stub. `script` maps "cmd argline" -> {error?, stdout?, stderr?}. + * Unmatched invocations fail with ENOENT (an uninstalled command), which is + * exactly what the real execFile does. Records every call for argv assertions. + */ +function makeRunner(script = {}) { + const calls = []; + const runner = (cmd, args, opts, cb) => { + calls.push({ cmd, args, opts }); + const isVersionProbe = args.length > 0 && args[args.length - 1] === '--version'; + const key = isVersionProbe + ? [cmd, ...args.slice(0, -1), '--version'].join(' ') + : [cmd, ...args].join(' '); + const result = script[key]; + setImmediate(() => + cb( + result ? result.error || null : Object.assign(new Error('spawn ENOENT'), { code: 'ENOENT' }), + result?.stdout ?? '', + result?.stderr ?? '' + ) + ); + }; + runner.calls = calls; + return runner; +} + +describe('MarkItDown', () => { + describe('resolveMarkItDown', () => { + it('prefers a direct markitdown binary when present', async () => { + const runner = makeRunner({ + 'markitdown --version': { stdout: 'markitdown 0.1.7\n' }, + }); + const resolved = await resolveMarkItDown(runner); + expect(resolved).toEqual({ command: 'markitdown', argsPrefix: [], version: '0.1.7' }); + }); + + it('falls back to python -m markitdown', async () => { + const py = process.platform === 'win32' ? 'python' : 'python3'; + const runner = makeRunner({ + [`${py} -m markitdown --version`]: { stdout: 'markitdown 0.1.6' }, + }); + const resolved = await resolveMarkItDown(runner); + expect(resolved).toMatchObject({ command: py, argsPrefix: ['-m', 'markitdown'] }); + }); + + it('probes every candidate before giving up (null)', async () => { + const runner = makeRunner({}); + expect(await resolveMarkItDown(runner)).toBeNull(); + // One probe per candidate + expect(runner.calls).toHaveLength(COMMAND_CANDIDATES.length); + }); + + it('treats a non-zero probe exit as unavailable', async () => { + const runner = makeRunner({ + 'markitdown --version': { error: Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) }, + }); + const resolved = await resolveMarkItDown(runner); + // Falls through to python candidates which also fail here -> null + expect(resolved).toBeNull(); + }); + }); + + describe('convertToMarkdown', () => { + const inputPath = path.join(os.tmpdir(), 'any-file.docx'); + + it('passes the user path as a literal argv element (no shell)', async () => { + const runner = makeRunner({ + [`markitdown ${inputPath}`]: { stdout: '# Converted\n' }, + }); + const { content } = await convertToMarkdown(inputPath, { + runner, + resolved: { command: 'markitdown', argsPrefix: [], version: null }, + }); + expect(content).toBe('# Converted\n'); + expect(runner.calls[0].cmd).toBe('markitdown'); + expect(runner.calls[0].args).toEqual([inputPath]); + }); + + it('prepends the module prefix for python-style invocation', async () => { + const calls = []; + const fake = (cmd, args, opts, cb) => { + calls.push({ cmd, args, opts }); + setImmediate(() => cb(null, 'ok', '')); + }; + await convertToMarkdown(inputPath, { + runner: fake, + resolved: { command: 'python3', argsPrefix: ['-m', 'markitdown'], version: null }, + }); + expect(calls[0].cmd).toBe('python3'); + expect(calls[0].args).toEqual(['-m', 'markitdown', inputPath]); + }); + + it('strips a UTF-8 BOM from the output', async () => { + const runner = makeRunner({ + [`markitdown ${inputPath}`]: { stdout: '\uFEFF# Title\n' }, + }); + const { content } = await convertToMarkdown(inputPath, { + runner, + resolved: { command: 'markitdown', argsPrefix: [], version: null }, + }); + expect(content.startsWith('\uFEFF')).toBe(false); + }); + + it('surfaces the actionable last stderr line and sanitizes paths', async () => { + const runner = makeRunner({ + [`markitdown ${inputPath}`]: { + error: Object.assign(new Error('Command failed'), { code: 1 }), + stderr: + 'Traceback (most recent call last):\n File "/home/user/x.py", line 1\n' + + "* pip install 'markitdown[pdf]'", + }, + }); + await expect( + convertToMarkdown(inputPath, { + runner, + resolved: { command: 'markitdown', argsPrefix: [], version: null }, + }) + ).rejects.toThrow(/pip install 'markitdown\[pdf\]'/); + }); + + it('classifies timeouts', async () => { + const runner = makeRunner({ + [`markitdown ${inputPath}`]: { + error: Object.assign(new Error(' timeout'), { code: 'ETIMEDOUT' }), + }, + }); + await expect( + convertToMarkdown(inputPath, { + runner, + resolved: { command: 'markitdown', argsPrefix: [], version: null }, + }) + ).rejects.toMatchObject({ code: 'timeout' }); + }); + + it('rejects empty output with the supported-formats hint', async () => { + const runner = makeRunner({ + [`markitdown ${inputPath}`]: { stdout: ' ' }, + }); + await expect( + convertToMarkdown(inputPath, { + runner, + resolved: { command: 'markitdown', argsPrefix: [], version: null }, + }) + ).rejects.toMatchObject({ code: 'empty_output' }); + }); + + it('rejects non-absolute and non-string paths', async () => { + const runner = makeRunner({}); + await expect( + convertToMarkdown('relative.docx', { + runner, + resolved: { command: 'markitdown', argsPrefix: [], version: null }, + }) + ).rejects.toMatchObject({ code: 'bad_path' }); + await expect(convertToMarkdown(null, { runner })).rejects.toMatchObject({ + code: 'bad_path', + }); + }); + + it('fails with install instructions when not installed', async () => { + const runner = makeRunner({}); + await expect(convertToMarkdown(inputPath, { runner })).rejects.toMatchObject({ + code: 'not_installed', + }); + }); + }); +});