mirror of
https://github.com/amitwh/markdown-converter.git
synced 2026-10-01 09:19:34 +05:30
feat(import): embed Microsoft MarkItDown for any-file → Markdown import
- File → Import with MarkItDown (Any Format)…: PDF, DOCX, PPTX, XLSX, Outlook .msg/.eml, EPUB, images, CSV/JSON/XML, ZIP (audio/OCR via the [all] extras) — verified live against HTML, XLSX (our own exporter's output), and PDF fixtures - Command auto-resolution with caching: markitdown binary → python -m markitdown → python3 -m markitdown - SEC-1 argv discipline (execFile only, user paths never through a shell), 50MB cap, 120s timeout, sanitized errors that surface markitdown's own "pip install 'markitdown[pdf]'" hints for missing format extras - Output lands next to the source as <name>.md (numeric suffix, never overwrites) and opens in a new tab; markitdown:available/convert IPC allowlisted for renderer flows - Help → Dependencies lists MarkItDown; README/UPDATES updated (v4.6.1) 12 new tests (629 green); lint clean; clean app boot
This commit is contained in:
+103
@@ -840,6 +840,12 @@ function createMenu() {
|
||||
accelerator: 'CmdOrCtrl+I',
|
||||
click: importDocument,
|
||||
},
|
||||
{
|
||||
// Microsoft MarkItDown: any file → Markdown (PDF/DOCX/PPTX/XLSX/
|
||||
// MSG/EPUB/images/ZIP/…, audio/OCR with the [all] extras)
|
||||
label: 'Import with MarkItDown (Any Format)...',
|
||||
click: importWithMarkItDown,
|
||||
},
|
||||
{
|
||||
label: 'Export',
|
||||
submenu: [
|
||||
@@ -1763,6 +1769,12 @@ function showDependenciesDialog() {
|
||||
<a class="dep-link" href="https://miktex.org/download" target="_blank">https://miktex.org/download</a>
|
||||
</div>
|
||||
|
||||
<div class="dep-card optional">
|
||||
<div class="dep-name">MarkItDown <span class="tag tag-optional">Optional</span></div>
|
||||
<div class="dep-desc">Microsoft's any-file-to-Markdown importer (PDF, DOCX, PPTX, XLSX, Outlook .msg, EPUB, images, ZIP). Install with: pip install "markitdown[all]"</div>
|
||||
<a class="dep-link" href="https://github.com/microsoft/markitdown" target="_blank">https://github.com/microsoft/markitdown</a>
|
||||
</div>
|
||||
|
||||
<h2>Bundled Libraries</h2>
|
||||
|
||||
<div class="dep-card">
|
||||
@@ -3717,6 +3729,97 @@ function importDocument() {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// MarkItDown import (Microsoft markitdown, any file → Markdown)
|
||||
// ============================================
|
||||
const MarkItDown = require('./main/MarkItDown');
|
||||
|
||||
// Probed once on first use; {command, argsPrefix, version} or null
|
||||
let markItDownResolved = undefined; // undefined = not probed yet
|
||||
|
||||
/** Resolve (and cache) the markitdown command via the bridge module. */
|
||||
async function getMarkItDown() {
|
||||
if (markItDownResolved === undefined) {
|
||||
markItDownResolved = await MarkItDown.resolveMarkItDown(require('child_process').execFile);
|
||||
}
|
||||
return markItDownResolved;
|
||||
}
|
||||
|
||||
/** IPC: availability probe for renderer hints (no version spam, cached). */
|
||||
ipcMain.handle('markitdown:available', async () => {
|
||||
const resolved = await getMarkItDown();
|
||||
return {
|
||||
available: Boolean(resolved),
|
||||
version: resolved?.version || null,
|
||||
// How the tool was found, e.g. "markitdown" or "python3 -m markitdown"
|
||||
via: resolved ? [resolved.command, ...resolved.argsPrefix].join(' ') : null,
|
||||
};
|
||||
});
|
||||
|
||||
/** IPC: convert one file to markdown (renderer-driven flows). */
|
||||
ipcMain.handle('markitdown:convert', async (_event, { path: inputPath } = {}) => {
|
||||
const validation = validatePath(inputPath);
|
||||
if (!validation.valid) throw new Error('Invalid file path');
|
||||
const stats = fs.statSync(validation.resolved);
|
||||
if (stats.size > MAX_FILE_SIZE) {
|
||||
throw new Error(`File exceeds the ${MAX_FILE_SIZE_MB}MB size limit.`);
|
||||
}
|
||||
const resolved = await getMarkItDown();
|
||||
return MarkItDown.convertToMarkdown(validation.resolved, { resolved });
|
||||
});
|
||||
|
||||
/**
|
||||
* Menu flow: pick any file, convert with markitdown, write <name>.md next to
|
||||
* the source (mirroring importDocument's UX), and open it in a new tab.
|
||||
* Existing outputs are never overwritten — a numeric suffix is appended.
|
||||
*/
|
||||
async function importWithMarkItDown() {
|
||||
const files = dialog.showOpenDialogSync(mainWindow, {
|
||||
properties: ['openFile'],
|
||||
title: 'Import with MarkItDown (any format)',
|
||||
});
|
||||
if (!files || !files[0]) return;
|
||||
const inputFile = files[0];
|
||||
|
||||
try {
|
||||
const stats = fs.statSync(inputFile);
|
||||
if (stats.size > MAX_FILE_SIZE) {
|
||||
dialog.showErrorBox('File Too Large', `File exceeds the ${MAX_FILE_SIZE_MB}MB size limit.`);
|
||||
return;
|
||||
}
|
||||
|
||||
const resolved = await getMarkItDown();
|
||||
const { content } = await MarkItDown.convertToMarkdown(inputFile, { resolved });
|
||||
|
||||
// <name>.md, then <name>-1.md, <name>-2.md, … when it already exists
|
||||
const base = inputFile.replace(/\.[^/.]+$/, '');
|
||||
let outputFile = `${base}.md`;
|
||||
for (let i = 1; fs.existsSync(outputFile); i++) outputFile = `${base}-${i}.md`;
|
||||
fs.writeFileSync(outputFile, content, 'utf-8');
|
||||
|
||||
currentFile = outputFile;
|
||||
mainWindow.webContents.send('file-opened', { path: outputFile, content });
|
||||
dialog.showMessageBox(mainWindow, {
|
||||
type: 'info',
|
||||
title: 'Import Complete',
|
||||
message: `Imported as ${path.basename(outputFile)} via MarkItDown\n\nOriginal: ${path.basename(inputFile)}`,
|
||||
buttons: ['OK'],
|
||||
});
|
||||
} catch (error) {
|
||||
dialog.showErrorBox(
|
||||
'MarkItDown Import',
|
||||
sanitizeErrorMessage(
|
||||
(error.code === 'not_installed'
|
||||
? error.message
|
||||
: `Import failed: ${error.message}`) +
|
||||
'\n\nMarkItDown is an optional Python tool from Microsoft (MIT):\n' +
|
||||
' pip install "markitdown[all]"'
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function setTheme(theme) {
|
||||
store.set('theme', theme);
|
||||
mainWindow.webContents.send('theme-changed', theme);
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
/**
|
||||
* MarkItDown bridge — "any file → Markdown" import via Microsoft's
|
||||
* markitdown Python tool (https://github.com/microsoft/markitdown, MIT).
|
||||
*
|
||||
* markitdown is a Python CLI, so like Pandoc/LibreOffice/FFmpeg it is used
|
||||
* when installed rather than bundled: the module probes for it once
|
||||
* (`markitdown` binary, then `python -m markitdown` / `python3 -m markitdown`)
|
||||
* and caches the resolved command. Every invocation goes through an argv
|
||||
* array via execFile — user-controlled paths are never passed through a
|
||||
* shell (same SEC-1 discipline as the Pandoc path).
|
||||
*
|
||||
* Supported by markitdown (core install): PDF, DOCX, PPTX, XLSX, Outlook
|
||||
* .msg/.eml, HTML, EPUB, images (EXIF), CSV/JSON/XML, ZIP archives, YouTube
|
||||
* URLs. Audio transcription and image OCR need the `[all]` extras:
|
||||
* pip install 'markitdown[all]'
|
||||
*
|
||||
* The execFile runner is injectable so tests can stub process spawning.
|
||||
*
|
||||
* @module MarkItDown
|
||||
*/
|
||||
|
||||
const CONVERT_TIMEOUT_MS = 120000;
|
||||
const MAX_OUTPUT_BUFFER = 20 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Candidate command templates probed in order. `argsPrefix` is prepended to
|
||||
* the user path when invoking (e.g. ['-m', 'markitdown'] for module-style
|
||||
* invocation through a python launcher).
|
||||
*/
|
||||
const COMMAND_CANDIDATES = [
|
||||
{ command: 'markitdown', argsPrefix: [] },
|
||||
{ command: process.platform === 'win32' ? 'python' : 'python3', argsPrefix: ['-m', 'markitdown'] },
|
||||
{ command: 'python3', argsPrefix: ['-m', 'markitdown'] },
|
||||
];
|
||||
|
||||
/** Run one probe: `--version` exits 0 when the tool is importable. */
|
||||
function probeCandidate(runner, candidate) {
|
||||
return new Promise((resolve) => {
|
||||
runner(
|
||||
candidate.command,
|
||||
[...candidate.argsPrefix, '--version'],
|
||||
{ timeout: 15000 },
|
||||
(error, stdout) => {
|
||||
if (error) return resolve(null);
|
||||
const match = /(\d+\.\d+(?:\.\d+)?)/.exec(String(stdout || ''));
|
||||
resolve({
|
||||
command: candidate.command,
|
||||
argsPrefix: candidate.argsPrefix,
|
||||
version: match ? match[1] : null,
|
||||
});
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the markitdown invocation. Probes candidates in order and returns
|
||||
* the first that answers `--version`, or null when none is installed.
|
||||
*
|
||||
* @param {Function} runner execFile-style (cmd, args, opts, cb)
|
||||
* @returns {Promise<{command: string, argsPrefix: string[], version: string|null}|null>}
|
||||
*/
|
||||
async function resolveMarkItDown(runner) {
|
||||
for (const candidate of COMMAND_CANDIDATES) {
|
||||
const resolved = await probeCandidate(runner, candidate);
|
||||
if (resolved) return resolved;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert any supported file to Markdown.
|
||||
*
|
||||
* @param {string} inputPath Absolute path to the source file
|
||||
* @param {object} [options]
|
||||
* @param {Function} [options.runner] injectable execFile (defaults to child_process)
|
||||
* @param {Function} [options.pathUtil] injected path module
|
||||
* @param {{command: string, argsPrefix: string[]}} [options.resolved] skip
|
||||
* probing when the caller already knows the command (main caches it)
|
||||
* @returns {Promise<{content: string}>} Markdown printed by markitdown on stdout
|
||||
* @throws {Error} with a user-safe message when the tool is missing or fails
|
||||
*/
|
||||
async function convertToMarkdown(inputPath, options = {}) {
|
||||
const runner = options.runner || require('child_process').execFile;
|
||||
const pathUtil = options.pathUtil || require('path');
|
||||
|
||||
// Validate the path BEFORE probing for the tool: a bad path is the user's
|
||||
// most actionable error and shouldn't be masked by an install hint
|
||||
if (typeof inputPath !== 'string' || !pathUtil.isAbsolute(inputPath)) {
|
||||
const err = new Error('MarkItDown import needs an absolute file path.');
|
||||
err.code = 'bad_path';
|
||||
throw err;
|
||||
}
|
||||
|
||||
const resolved = options.resolved || (await resolveMarkItDown(runner));
|
||||
if (!resolved) {
|
||||
const err = new Error(
|
||||
'MarkItDown is not installed. Install it with:\n\npip install "markitdown[all]"\n\n' +
|
||||
'(or the lighter core: pip install markitdown)'
|
||||
);
|
||||
err.code = 'not_installed';
|
||||
throw err;
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
// markitdown prints the converted Markdown to stdout; keep everything in
|
||||
// argv so the path is never re-interpreted by a shell.
|
||||
runner(
|
||||
resolved.command,
|
||||
[...resolved.argsPrefix, inputPath],
|
||||
{ timeout: CONVERT_TIMEOUT_MS, maxBuffer: MAX_OUTPUT_BUFFER },
|
||||
(error, stdout, stderr) => {
|
||||
if (error) {
|
||||
// Strip absolute paths, then keep the first informative line from
|
||||
// either the error or stderr — markitdown writes tracebacks there
|
||||
// (e.g. a missing optional dependency like pdfminer.six for PDFs).
|
||||
const safe = (text) =>
|
||||
String(text || '')
|
||||
.replace(/[A-Z]:\\[^\s"']+/gi, '(path)')
|
||||
.replace(/\/(?:home|Users|tmp|mnt)\/[^\s"']+/g, '(path)');
|
||||
// Prefer the LAST informative stderr line: markitdown ends its
|
||||
// output with the actionable install hint ("pip install
|
||||
// 'markitdown[pdf]'"), while the first lines are a traceback.
|
||||
const stderrLines = safe(stderr).split('\n').filter((l) => l.trim().length > 0);
|
||||
const detail =
|
||||
stderrLines[stderrLines.length - 1] ||
|
||||
safe(error.message).split('\n')[0] ||
|
||||
'unknown error';
|
||||
const err2 = new Error(`MarkItDown conversion failed: ${detail}`);
|
||||
err2.code = String(error.code || '').startsWith('ETIMEDOUT') ? 'timeout' : 'failed';
|
||||
err2.stderr = safe(stderr).slice(0, 2000);
|
||||
reject(err2);
|
||||
return;
|
||||
}
|
||||
const content = String(stdout || '');
|
||||
if (!content.trim()) {
|
||||
const err3 = new Error(
|
||||
'MarkItDown produced no output — the file type may be unsupported ' +
|
||||
'(core install covers PDF/DOCX/PPTX/XLSX/MSG/HTML/EPUB/images/CSV/JSON/XML/ZIP; ' +
|
||||
"audio/OCR need pip install 'markitdown[all]')."
|
||||
);
|
||||
err3.code = 'empty_output';
|
||||
reject(err3);
|
||||
return;
|
||||
}
|
||||
// Strip a UTF-8 BOM if present so downstream markdown tooling is happy
|
||||
resolve({ content: content.replace(/^\uFEFF/, '') });
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
resolveMarkItDown,
|
||||
convertToMarkdown,
|
||||
COMMAND_CANDIDATES,
|
||||
};
|
||||
@@ -168,6 +168,10 @@ const ALLOWED_SEND_CHANNELS = [
|
||||
'plantuml:available',
|
||||
'plantuml:render',
|
||||
|
||||
// MarkItDown import (optional Python CLI)
|
||||
'markitdown:available',
|
||||
'markitdown:convert',
|
||||
|
||||
// Quick Note scratchpad
|
||||
'quick-note:save',
|
||||
|
||||
|
||||
Reference in New Issue
Block a user