mirror of
https://github.com/amitwh/markdown-converter.git
synced 2026-10-01 17:29:29 +05:30
feat(templates,qa): template gallery + pluggable DocQA engine
Two more features from the deferred menu:
Daily-note template gallery:
- src/main/DailyNotesTemplates.js — pure module: listTemplates() /
saveTemplate() / deleteTemplate() / labelFor() with injectable IO.
- src/main/DailyNotes.js — openOrCreate() now accepts seedContent so a
non-default template can seed a NEW note (existing notes never get
clobbered).
- src/main.js — IPC channels daily-templates:list / save / delete /
apply. apply renders the chosen template (with {date}/{weekday}
substitution) and pipes through DailyNotes.openOrCreate.
- src/sidebar/daily-templates-panel.js — gallery UI: list, +New
(prompt for name + content), Use (applies to today's note),
delete (refuses to remove the last template so the default survives).
- src/renderer.js — registers the panel.
- src/index.html — icon (already added).
Pluggable DocQA engine (semantic search hook):
- src/main/SemanticEngine.js — engine interface with defaultEngine() (TF-idF,
always available) and neuralEngine() (lazy @xenova/transformers,
falls back gracefully when the dep is missing). getEngine(name)
resolves either.
- src/main/DocQA.js — ask() is now async and accepts an engine arg.
TF-idF path unchanged; neural path calls engine.rank(question, chunks)
directly. The chunk corpus is built up front regardless of engine so
ranking is consistent.
- src/main.js — doc-qa:ask IPC resolves the engine via SemanticEngine.getEngine(name)
before calling DocQA.ask. The renderer can pass {engine: 'transformers'}
to opt in once @xenova/transformers is installed.
Tests (51 new across this batch):
- tests/main/DailyNotesTemplates.test.js (18): labelFor separators /
edge cases / non-string safety, listTemplates empty / present / sort,
saveTemplate nested dir + .md extension + validation + null content,
deleteTemplate success / missing / validation.
- tests/daily-templates-panel.test.js (12): mount + empty state + list +
XSS safety, Use button (apply + error path), Delete button (success +
last-template guard), New template (save + cancel), refresh.
- tests/main/SemanticEngine.test.js (8): default engine shape + rank
matches WorkspaceSearch, getEngine for tf-idf / unknown / transformers
(graceful fallback when @xenova/transformers missing), parity check.
- DocQA: 5 new tests for engine arg (custom engine.rank called, default
fallback, neural hit shape translation); existing tests updated to
await the now-async ask().
Full suite: 76 suites, 914 tests, lint+format clean.
Activation for the neural engine:
npm install @xenova/transformers
(heavy; ~50 MiB with deps) — then 'transformers' is selectable in
doc-qa:ask. Until then, all calls use TF-idF transparently.
Amit Haridas
This commit is contained in:
+46
-20
@@ -21,6 +21,7 @@
|
||||
*/
|
||||
|
||||
const WorkspaceSearch = require('./WorkspaceSearch');
|
||||
const SemanticEngine = require('./SemanticEngine');
|
||||
|
||||
const QUESTION_WORDS = new Set([
|
||||
'what',
|
||||
@@ -172,51 +173,76 @@ function chunkDocument(content, maxChunkChars = 800) {
|
||||
* @param {Array<{path:string, content:string, mtimeMs?:number}>} args.files
|
||||
* @param {number} [args.topK=5] number of chunks to return
|
||||
* @param {number} [args.nowMs=Date.now()]
|
||||
* @param {object} [args.engine] Optional SemanticEngine instance. When
|
||||
* omitted, the default TF-idF engine is used. Pass a neural engine to
|
||||
* swap in semantic embeddings.
|
||||
* @returns {{question:string, chunks:Array<{filePath:string, snippet:string, score:number, mtimeMs:number}>}}
|
||||
*/
|
||||
function ask({ question, files, topK = 5, nowMs = Date.now() }) {
|
||||
async function ask({
|
||||
question,
|
||||
files,
|
||||
topK = 5,
|
||||
nowMs = Date.now(),
|
||||
engine = null,
|
||||
}) {
|
||||
const cleaned = cleanQuestion(question);
|
||||
if (!cleaned || !Array.isArray(files) || files.length === 0) {
|
||||
return { question: String(question || ''), chunks: [] };
|
||||
}
|
||||
|
||||
// First, find the docs that match at all (cheap, broad pass).
|
||||
const docHits = WorkspaceSearch.search({ query: cleaned, files, limit: 20, nowMs });
|
||||
|
||||
// Then re-rank at the chunk level within those docs.
|
||||
// Chunk the corpus up front — both default and neural engines rank at
|
||||
// chunk granularity.
|
||||
const chunkCorpus = [];
|
||||
for (const hit of docHits) {
|
||||
const file = files.find((f) => f.path === hit.filePath);
|
||||
for (const file of files) {
|
||||
if (!file || typeof file.content !== 'string') continue;
|
||||
const chunks = chunkDocument(file.content);
|
||||
for (const chunk of chunks) {
|
||||
chunkCorpus.push({
|
||||
path: `${hit.filePath}#${chunk.start}`,
|
||||
path: `${file.path}#${chunk.start}`,
|
||||
content: chunk.text,
|
||||
offset: chunk.start,
|
||||
mtimeMs: file.mtimeMs,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (chunkCorpus.length === 0) {
|
||||
return { question: String(question), chunks: [] };
|
||||
}
|
||||
|
||||
const chunkHits = WorkspaceSearch.search({
|
||||
query: cleaned,
|
||||
files: chunkCorpus,
|
||||
limit: topK,
|
||||
nowMs,
|
||||
});
|
||||
// Resolve engine (default = tf-idf)
|
||||
const eng = engine || SemanticEngine.defaultEngine();
|
||||
|
||||
// Translate the per-chunk hits back into the public shape. The fake path
|
||||
// "<file>#<offset>" carries the chunk start; the renderer's existing
|
||||
// file-opened handler can split it on '#' if it wants to deep-link.
|
||||
const chunks = chunkHits.map((h) => {
|
||||
let chunkHits;
|
||||
if (eng.isNeural) {
|
||||
// Neural: rank directly on the question against the chunk corpus.
|
||||
chunkHits = await eng.rank(cleaned, chunkCorpus);
|
||||
} else {
|
||||
// TF-idF: broad doc pass first (caps the chunk corpus), then chunk rank.
|
||||
const docHits = WorkspaceSearch.search({ query: cleaned, files, limit: 20, nowMs });
|
||||
const docPaths = new Set(docHits.map((h) => h.filePath));
|
||||
const filtered = chunkCorpus.filter((c) => {
|
||||
const filePath = c.path.replace(/#\d+$/, '');
|
||||
return docPaths.has(filePath);
|
||||
});
|
||||
chunkHits = WorkspaceSearch.search({
|
||||
query: cleaned,
|
||||
files: filtered.length > 0 ? filtered : chunkCorpus,
|
||||
limit: topK,
|
||||
nowMs,
|
||||
});
|
||||
}
|
||||
|
||||
// Translate the per-chunk hits back into the public shape.
|
||||
const chunks = chunkHits.slice(0, topK).map((h) => {
|
||||
const offsetMatch = /#(\d+)$/.exec(h.filePath);
|
||||
const offset = offsetMatch ? Number(offsetMatch[1]) : 0;
|
||||
const offset = offsetMatch ? Number(offsetMatch[1]) : h.offset || 0;
|
||||
return {
|
||||
filePath: h.filePath.replace(/#\d+$/, ''),
|
||||
offset,
|
||||
snippet: h.snippet,
|
||||
score: h.score,
|
||||
mtimeMs: chunkCorpus.find((c) => c.path === h.filePath)?.mtimeMs || 0,
|
||||
mtimeMs:
|
||||
chunkCorpus.find((c) => c.path === h.filePath)?.mtimeMs || h.mtimeMs || 0,
|
||||
};
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user