From a22b40e7caee3e00aaf8fcb93eba3b0daf46080c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 14 Nov 2025 05:23:45 +0000 Subject: [PATCH 1/2] perf(obsidian): add caching to vault file scanning The recursive directory scan was running every time the user opened the note browser, rescanning 13k+ files repeatedly. This was slow and wasteful. Changes: - Add vaultFilesCache object with files, timestamp, and TTL (5 min) - Check cache before scanning in getVaultFiles() - Cache results after successful scan - Add clearVaultCache() function for manual refresh - Add forceRefresh parameter to bypass cache - Log cache hits vs misses for debugging Performance: - First load: Scans vault (slow, but only once) - Subsequent loads: Instant (uses cache) - Cache expires after 5 minutes (auto-refresh) - User can manually clear cache if needed This makes opening the note browser instant on subsequent uses. --- modules/io/obsidian-bridge.js | 37 ++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/modules/io/obsidian-bridge.js b/modules/io/obsidian-bridge.js index 2a7278fe..0d2cbb7e 100644 --- a/modules/io/obsidian-bridge.js +++ b/modules/io/obsidian-bridge.js @@ -13,6 +13,13 @@ const ObsidianBridge = (() => { vaultName: "" }; + // Cache for vault file list + let vaultFilesCache = { + files: null, + timestamp: null, + ttl: 5 * 60 * 1000 // 5 minutes cache + }; + // Initialize from localStorage function init() { const stored = localStorage.getItem("obsidianConfig"); @@ -94,19 +101,42 @@ const ObsidianBridge = (() => { return mdFiles; } - // Get all markdown files from vault (recursively) - async function getVaultFiles() { + // Clear the vault files cache + function clearVaultCache() { + vaultFilesCache.files = null; + vaultFilesCache.timestamp = null; + INFO && console.log("Vault file cache cleared"); + } + + // Get all markdown files from vault (recursively, with caching) + async function getVaultFiles(forceRefresh = false) { if (!config.enabled) { throw new Error("Obsidian not connected"); } + // Check cache + const now = Date.now(); + const cacheValid = vaultFilesCache.files !== null && + vaultFilesCache.timestamp !== null && + (now - vaultFilesCache.timestamp) < vaultFilesCache.ttl; + + if (cacheValid && !forceRefresh) { + INFO && console.log(`getVaultFiles: Using cached list (${vaultFilesCache.files.length} files)`); + return vaultFilesCache.files; + } + try { TIME && console.time("getVaultFiles"); + INFO && console.log("getVaultFiles: Scanning vault (cache miss or expired)..."); // Recursively scan all directories const mdFiles = await scanDirectory(""); - INFO && console.log(`getVaultFiles: Found ${mdFiles.length} markdown files (recursive scan)`); + // Update cache + vaultFilesCache.files = mdFiles; + vaultFilesCache.timestamp = now; + + INFO && console.log(`getVaultFiles: Found ${mdFiles.length} markdown files (recursive scan, cached)`); DEBUG && console.log("Sample files:", mdFiles.slice(0, 10)); TIME && console.timeEnd("getVaultFiles"); @@ -520,6 +550,7 @@ Add your lore here... saveConfig, testConnection, getVaultFiles, + clearVaultCache, getNote, updateNote, createNote, From f575631e30572bd360205108d91550da1018d56e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 14 Nov 2025 05:29:36 +0000 Subject: [PATCH 2/2] feat(obsidian): add FMG ID index for instant note lookups Problem: Every time you click on a burg/marker, it had to scan through all 13,496 vault files looking for a matching fmg-id. This was extremely slow and made the feature unusable for large vaults. Solution: Implement a smart index that maps fmg-id to file paths. How it works: - Index stored in memory (fmgIdIndex object) - Persisted to localStorage (survives page reloads) - Loaded on init, saved on changes - When looking up by fmg-id: 1. Check index first (instant!) 2. If found, verify file still exists and ID matches 3. If not in index, search vault and add to index - Automatically updates when notes are created/saved through FMG - Handles stale entries (file deleted/modified) Performance improvement: - Before: O(n) - scan all 13k files (very slow) - After: O(1) - instant lookup from index - First click on burg: May need to search (builds index) - Second click on same burg: Instant! Opens note directly This makes the Obsidian integration actually usable. Create a note once for a burg, and every time you click that burg again, it opens instantly. --- modules/io/obsidian-bridge.js | 82 ++++++++++++++++++++++++++++- modules/ui/obsidian-notes-editor.js | 18 +++++++ 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/modules/io/obsidian-bridge.js b/modules/io/obsidian-bridge.js index 0d2cbb7e..86e32f15 100644 --- a/modules/io/obsidian-bridge.js +++ b/modules/io/obsidian-bridge.js @@ -20,6 +20,9 @@ const ObsidianBridge = (() => { ttl: 5 * 60 * 1000 // 5 minutes cache }; + // Index: fmg-id → file path for fast lookups + let fmgIdIndex = {}; + // Initialize from localStorage function init() { const stored = localStorage.getItem("obsidianConfig"); @@ -31,6 +34,18 @@ const ObsidianBridge = (() => { ERROR && console.error("Failed to load Obsidian config:", error); } } + + // Load FMG ID index from localStorage + const storedIndex = localStorage.getItem("obsidianFmgIdIndex"); + if (storedIndex) { + try { + fmgIdIndex = JSON.parse(storedIndex); + INFO && console.log(`Loaded FMG ID index with ${Object.keys(fmgIdIndex).length} entries`); + } catch (error) { + ERROR && console.error("Failed to load FMG ID index:", error); + fmgIdIndex = {}; + } + } } // Save configuration @@ -108,6 +123,29 @@ const ObsidianBridge = (() => { INFO && console.log("Vault file cache cleared"); } + // Save FMG ID index to localStorage + function saveFmgIdIndex() { + try { + localStorage.setItem("obsidianFmgIdIndex", JSON.stringify(fmgIdIndex)); + DEBUG && console.log(`Saved FMG ID index with ${Object.keys(fmgIdIndex).length} entries`); + } catch (error) { + ERROR && console.error("Failed to save FMG ID index:", error); + } + } + + // Add entry to FMG ID index + function addToFmgIdIndex(fmgId, filePath) { + if (!fmgId) return; + fmgIdIndex[fmgId] = filePath; + saveFmgIdIndex(); + DEBUG && console.log(`Added to index: ${fmgId} → ${filePath}`); + } + + // Get file path from FMG ID index + function getFromFmgIdIndex(fmgId) { + return fmgIdIndex[fmgId] || null; + } + // Get all markdown files from vault (recursively, with caching) async function getVaultFiles(forceRefresh = false) { if (!config.enabled) { @@ -374,9 +412,43 @@ const ObsidianBridge = (() => { } } - // Find note by FMG ID in frontmatter + // Find note by FMG ID in frontmatter (with index for fast lookup) async function findNoteByFmgId(fmgId) { + if (!fmgId) return null; + try { + // First, check the index for instant lookup + const indexedPath = getFromFmgIdIndex(fmgId); + if (indexedPath) { + INFO && console.log(`Found note in index: ${fmgId} → ${indexedPath}`); + try { + const content = await getNote(indexedPath); + const {frontmatter} = parseFrontmatter(content); + + // Verify the fmg-id still matches (file might have been modified) + if (frontmatter["fmg-id"] === fmgId || frontmatter.fmgId === fmgId) { + return { + path: indexedPath, + name: indexedPath.replace(/\.md$/, "").split("/").pop(), + content, + frontmatter + }; + } else { + // Index is stale, remove the entry + WARN && console.warn(`Index entry stale for ${fmgId}, removing`); + delete fmgIdIndex[fmgId]; + saveFmgIdIndex(); + } + } catch (error) { + // File no longer exists, remove from index + WARN && console.warn(`Indexed file not found: ${indexedPath}, removing from index`); + delete fmgIdIndex[fmgId]; + saveFmgIdIndex(); + } + } + + // Not in index or index was stale, search all files + INFO && console.log(`Searching vault for fmg-id: ${fmgId}`); const files = await getVaultFiles(); const mdFiles = files.filter(f => f.endsWith(".md")); @@ -386,6 +458,10 @@ const ObsidianBridge = (() => { const {frontmatter} = parseFrontmatter(content); if (frontmatter["fmg-id"] === fmgId || frontmatter.fmgId === fmgId) { + // Found it! Add to index for next time + addToFmgIdIndex(fmgId, filePath); + INFO && console.log(`Found note and added to index: ${fmgId} → ${filePath}`); + return { path: filePath, name: filePath.replace(/\.md$/, "").split("/").pop(), @@ -560,7 +636,9 @@ Add your lore here... generateNoteTemplate, searchNotes, listAllNotes, - listAllNotePaths + listAllNotePaths, + addToFmgIdIndex, + getFromFmgIdIndex }; })(); diff --git a/modules/ui/obsidian-notes-editor.js b/modules/ui/obsidian-notes-editor.js index 0c52e96b..86a28946 100644 --- a/modules/ui/obsidian-notes-editor.js +++ b/modules/ui/obsidian-notes-editor.js @@ -216,6 +216,13 @@ async function promptCreateNewNote(elementId, elementType, coordinates) { const {frontmatter} = ObsidianBridge.parseFrontmatter(template); + // Add to FMG ID index for instant future lookups + const fmgId = frontmatter["fmg-id"] || frontmatter.fmgId; + if (fmgId) { + ObsidianBridge.addToFmgIdIndex(fmgId, notePath); + INFO && console.log(`New note added to index: ${fmgId} → ${notePath}`); + } + resolve({ path: notePath, name, @@ -590,6 +597,17 @@ async function saveObsidianNote() { try { await ObsidianBridge.updateNote(path, content); + + // Update the FMG ID index if this note has an fmg-id + if (elementId) { + const {frontmatter} = ObsidianBridge.parseFrontmatter(content); + const fmgId = frontmatter["fmg-id"] || frontmatter.fmgId; + if (fmgId) { + // Add to index using internal method + ObsidianBridge.addToFmgIdIndex(fmgId, path); + } + } + showMarkdownEditor.originalContent = content; // Update the editor to show the new frontmatter byId("obsidianMarkdownEditor").value = content;