mirror of
https://github.com/docsifyjs/docsify.git
synced 2026-10-16 20:21:38 +08:00
feat: improve indexing and embedded search (#2734)
This commit is contained in:
Luffy
+154
-18
@@ -4,6 +4,11 @@ import {
|
||||
removeAtag,
|
||||
escapeHtml,
|
||||
} from '../../core/render/utils.js';
|
||||
import {
|
||||
getPath,
|
||||
getParentPath,
|
||||
isAbsolutePath,
|
||||
} from '../../core/router/util.js';
|
||||
import { markdownToTxt } from './markdown-to-txt.js';
|
||||
import Dexie from 'dexie';
|
||||
|
||||
@@ -16,10 +21,29 @@ db.version(1).stores({
|
||||
});
|
||||
|
||||
async function saveData(maxAge, expireKey) {
|
||||
INDEXES = Object.values(INDEXES).flatMap(innerData =>
|
||||
Object.values(innerData),
|
||||
);
|
||||
await /** @type {any} */ (db).search.bulkPut(INDEXES);
|
||||
const records = [];
|
||||
|
||||
Object.values(INDEXES).forEach(entry => {
|
||||
if (!entry || typeof entry !== 'object') {
|
||||
return;
|
||||
}
|
||||
|
||||
// Entry may already be a flat record read from IndexedDB.
|
||||
if ('slug' in entry) {
|
||||
records.push(entry);
|
||||
return;
|
||||
}
|
||||
|
||||
// Entry may be a per-path map of slug -> record produced by genIndex().
|
||||
Object.values(entry).forEach(item => {
|
||||
if (item && typeof item === 'object' && 'slug' in item) {
|
||||
records.push(item);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
INDEXES = records;
|
||||
await /** @type {any} */ (db).search.bulkPut(records);
|
||||
await /** @type {any} */ (db).expires.put({
|
||||
key: expireKey,
|
||||
value: Date.now() + maxAge,
|
||||
@@ -96,6 +120,111 @@ function getListData(token) {
|
||||
return token.text;
|
||||
}
|
||||
|
||||
function extractFragmentContent(text, fragment, fullLine) {
|
||||
if (!fragment) {
|
||||
return text;
|
||||
}
|
||||
|
||||
let fragmentRegex = `(?:###|\\/\\/\\/)\\s*\\[${fragment}\\]`;
|
||||
if (fullLine) {
|
||||
fragmentRegex = `.*${fragmentRegex}.*\n`;
|
||||
}
|
||||
|
||||
const pattern = new RegExp(
|
||||
`(?:${fragmentRegex})([\\s\\S]*?)(?:${fragmentRegex})`,
|
||||
);
|
||||
const match = text.match(pattern);
|
||||
return ((match || [])[1] || '').trim();
|
||||
}
|
||||
|
||||
function collectEmbedRequests(raw = '', path, vm) {
|
||||
const tokens = window.marked.lexer(raw);
|
||||
const requests = [];
|
||||
|
||||
const maybePushEmbed = inlineToken => {
|
||||
if (
|
||||
!inlineToken ||
|
||||
(inlineToken.type !== 'link' && inlineToken.type !== 'image')
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { config } = getAndRemoveConfig(inlineToken.title || '');
|
||||
if (!config.include || !inlineToken.href) {
|
||||
return;
|
||||
}
|
||||
|
||||
const href = isAbsolutePath(inlineToken.href)
|
||||
? inlineToken.href
|
||||
: getPath(vm.router.getBasePath(), getParentPath(path), inlineToken.href);
|
||||
|
||||
let type = 'code';
|
||||
if (/\.(md|markdown)/.test(href)) {
|
||||
type = 'markdown';
|
||||
} else if (/\.mmd/.test(href)) {
|
||||
type = 'mermaid';
|
||||
}
|
||||
|
||||
requests.push({
|
||||
url: href,
|
||||
type,
|
||||
fragment: config.fragment,
|
||||
omitFragmentLine: config.omitFragmentLine,
|
||||
});
|
||||
};
|
||||
|
||||
tokens.forEach(token => {
|
||||
if (token.type === 'paragraph') {
|
||||
(token.tokens || []).forEach(maybePushEmbed);
|
||||
} else if (token.type === 'table') {
|
||||
(token.header || []).forEach(cell => {
|
||||
(cell.tokens || []).forEach(maybePushEmbed);
|
||||
});
|
||||
(token.rows || []).forEach(row => {
|
||||
row.forEach(cell => {
|
||||
(cell.tokens || []).forEach(maybePushEmbed);
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return requests;
|
||||
}
|
||||
|
||||
async function getEmbeddedContent(raw = '', path, vm) {
|
||||
const requests = collectEmbedRequests(raw, path, vm);
|
||||
if (!requests.length) {
|
||||
return '';
|
||||
}
|
||||
|
||||
const results = await Promise.all(
|
||||
requests.map(
|
||||
request =>
|
||||
new Promise(resolve => {
|
||||
Docsify.get(request.url, false, vm.config.requestHeaders).then(
|
||||
text => {
|
||||
let content = text || '';
|
||||
if (request.fragment) {
|
||||
content = extractFragmentContent(
|
||||
content,
|
||||
request.fragment,
|
||||
request.omitFragmentLine,
|
||||
);
|
||||
}
|
||||
|
||||
resolve(
|
||||
request.type === 'markdown' ? content : markdownToTxt(content),
|
||||
);
|
||||
},
|
||||
() => resolve(''),
|
||||
);
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
return results.filter(Boolean).join('\n');
|
||||
}
|
||||
|
||||
export function genIndex(path, content = '', router, depth, indexKey) {
|
||||
const tokens = window.marked.lexer(content);
|
||||
const slugify = window.Docsify.slugify;
|
||||
@@ -205,8 +334,6 @@ export function search(query) {
|
||||
),
|
||||
'gi',
|
||||
);
|
||||
let indexTitle = -1;
|
||||
let indexContent = -1;
|
||||
handlePostTitle = postTitle
|
||||
? escapeHtml(ignoreDiacriticalMarks(postTitle))
|
||||
: postTitle;
|
||||
@@ -214,8 +341,8 @@ export function search(query) {
|
||||
? escapeHtml(ignoreDiacriticalMarks(postContent))
|
||||
: postContent;
|
||||
|
||||
indexTitle = postTitle ? handlePostTitle.search(regEx) : -1;
|
||||
indexContent = postContent ? handlePostContent.search(regEx) : -1;
|
||||
const indexTitle = postTitle ? handlePostTitle.search(regEx) : -1;
|
||||
let indexContent = postContent ? handlePostContent.search(regEx) : -1;
|
||||
|
||||
if (indexTitle >= 0 || indexContent >= 0) {
|
||||
matchesScore += indexTitle >= 0 ? 3 : indexContent >= 0 ? 2 : 0;
|
||||
@@ -223,11 +350,8 @@ export function search(query) {
|
||||
indexContent = 0;
|
||||
}
|
||||
|
||||
let start = 0;
|
||||
let end = 0;
|
||||
|
||||
start = indexContent < 11 ? 0 : indexContent - 10;
|
||||
end = start === 0 ? 100 : indexContent + keyword.length + 90;
|
||||
const start = indexContent < 11 ? 0 : indexContent - 10;
|
||||
let end = start === 0 ? 100 : indexContent + keyword.length + 90;
|
||||
|
||||
if (handlePostContent && end > handlePostContent.length) {
|
||||
end = handlePostContent.length;
|
||||
@@ -306,26 +430,38 @@ export async function init(config, vm) {
|
||||
const len = paths.length;
|
||||
let count = 0;
|
||||
|
||||
const markComplete = async () => {
|
||||
if (len === ++count) {
|
||||
await saveData(config.maxAge, expireKey);
|
||||
}
|
||||
};
|
||||
|
||||
paths.forEach(path => {
|
||||
const pathExists = Array.isArray(INDEXES)
|
||||
? INDEXES.some(obj => obj.path === path)
|
||||
: false;
|
||||
if (pathExists) {
|
||||
return count++;
|
||||
void markComplete();
|
||||
return;
|
||||
}
|
||||
|
||||
Docsify.get(vm.router.getFile(path), false, vm.config.requestHeaders).then(
|
||||
async result => {
|
||||
const embeddedContent = await getEmbeddedContent(result, path, vm);
|
||||
const contentToIndex = embeddedContent
|
||||
? `${result}\n${embeddedContent}`
|
||||
: result;
|
||||
INDEXES[path] = genIndex(
|
||||
path,
|
||||
result,
|
||||
contentToIndex,
|
||||
vm.router,
|
||||
config.depth,
|
||||
indexKey,
|
||||
);
|
||||
if (len === ++count) {
|
||||
await saveData(config.maxAge, expireKey);
|
||||
}
|
||||
return markComplete();
|
||||
},
|
||||
() => {
|
||||
return markComplete();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
+179
-1
@@ -198,6 +198,130 @@ test.describe('Search Plugin Tests', () => {
|
||||
await expect(resultsHeadingElm).toHaveText('EmptyContent');
|
||||
});
|
||||
|
||||
test('keeps saving index when one auto path request fails with cached records', async ({
|
||||
page,
|
||||
}) => {
|
||||
const indexKey = 'docsify.search.index';
|
||||
const expireKey = 'docsify.search.expires';
|
||||
|
||||
const pageErrors = [];
|
||||
page.on('pageerror', error => pageErrors.push(error.message));
|
||||
|
||||
await page.evaluate(
|
||||
({ indexKey, expireKey }) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = indexedDB.open('docsify', 1);
|
||||
|
||||
request.onupgradeneeded = () => {
|
||||
const db = request.result;
|
||||
|
||||
if (!db.objectStoreNames.contains('search')) {
|
||||
db.createObjectStore('search', { keyPath: 'slug' });
|
||||
}
|
||||
|
||||
if (!db.objectStoreNames.contains('expires')) {
|
||||
db.createObjectStore('expires', { keyPath: 'key' });
|
||||
}
|
||||
};
|
||||
|
||||
request.onerror = () => reject(request.error);
|
||||
request.onsuccess = () => {
|
||||
const db = request.result;
|
||||
const tx = db.transaction(['search', 'expires'], 'readwrite');
|
||||
|
||||
tx.objectStore('search').put({
|
||||
slug: '/cached',
|
||||
title: 'Cached Page',
|
||||
body: 'cached record',
|
||||
path: '/cached',
|
||||
indexKey,
|
||||
});
|
||||
tx.objectStore('expires').put({
|
||||
key: expireKey,
|
||||
value: Date.now() + 60 * 1000,
|
||||
});
|
||||
|
||||
tx.oncomplete = () => {
|
||||
db.close();
|
||||
resolve();
|
||||
};
|
||||
tx.onerror = () => reject(tx.error);
|
||||
};
|
||||
});
|
||||
},
|
||||
{ indexKey, expireKey },
|
||||
);
|
||||
|
||||
await docsifyInit({
|
||||
markdown: {
|
||||
homepage: '# Home',
|
||||
sidebar: `
|
||||
- [Cached](cached)
|
||||
- [Success](success)
|
||||
- [Fail](fail)
|
||||
`,
|
||||
},
|
||||
routes: {
|
||||
'/success.md': '# Success\n\nregressionKeyword',
|
||||
'/fail.md': {
|
||||
status: 404,
|
||||
body: 'Not Found',
|
||||
contentType: 'text/markdown',
|
||||
},
|
||||
},
|
||||
scriptURLs: ['/dist/plugins/search.js'],
|
||||
});
|
||||
|
||||
await expect
|
||||
.poll(async () => {
|
||||
return await page.evaluate(indexKey => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = indexedDB.open('docsify');
|
||||
|
||||
request.onerror = () => reject(request.error);
|
||||
request.onsuccess = () => {
|
||||
const db = request.result;
|
||||
const tx = db.transaction(['search', 'expires'], 'readonly');
|
||||
const searchStore = tx.objectStore('search');
|
||||
const expiresStore = tx.objectStore('expires');
|
||||
const searchReq = searchStore.getAll();
|
||||
const expiresReq = expiresStore.get('docsify.search.expires');
|
||||
|
||||
tx.onerror = () => reject(tx.error);
|
||||
tx.oncomplete = () => {
|
||||
const records = Array.isArray(searchReq.result)
|
||||
? searchReq.result
|
||||
: [];
|
||||
const hasSuccessRecord = records.some(
|
||||
record =>
|
||||
record &&
|
||||
record.indexKey === indexKey &&
|
||||
record.path === '/success',
|
||||
);
|
||||
const hasInvalidRecord = records.some(
|
||||
record => !record || typeof record.slug !== 'string',
|
||||
);
|
||||
const hasExpireRecord = Boolean(expiresReq.result?.value);
|
||||
|
||||
db.close();
|
||||
resolve(
|
||||
hasSuccessRecord && hasExpireRecord && !hasInvalidRecord,
|
||||
);
|
||||
};
|
||||
};
|
||||
});
|
||||
}, indexKey);
|
||||
})
|
||||
.toBe(true);
|
||||
|
||||
const searchFieldElm = page.locator('input[type=search]');
|
||||
const resultsHeadingElm = page.locator('.results-panel .title');
|
||||
|
||||
await searchFieldElm.fill('regressionKeyword');
|
||||
await expect(resultsHeadingElm).toHaveText('Success');
|
||||
expect(pageErrors).toEqual([]);
|
||||
});
|
||||
|
||||
test('handles default focusSearch binding', async ({ page }) => {
|
||||
const docsifyInitConfig = {
|
||||
scriptURLs: ['/dist/plugins/search.js'],
|
||||
@@ -277,10 +401,64 @@ console.log('Hello World');
|
||||
await docsifyInit(docsifyInitConfig);
|
||||
await searchFieldElm.fill('filename');
|
||||
expect(await resultsHeadingElm.textContent()).toContain(
|
||||
'...filename _media/example.js :include :type=code :fragment=demo...',
|
||||
'filename _media/example.js :include :type=code :fragment=demo',
|
||||
);
|
||||
});
|
||||
|
||||
test('search should index embedded include content', async ({ page }) => {
|
||||
const docsifyInitConfig = {
|
||||
markdown: {
|
||||
homepage: `
|
||||
# Include Search
|
||||
|
||||
|
||||
`,
|
||||
},
|
||||
routes: {
|
||||
'/snippet.js': `
|
||||
const embeddedSearchKeyword = 'ok';
|
||||
`,
|
||||
},
|
||||
scriptURLs: ['/dist/plugins/search.js'],
|
||||
};
|
||||
|
||||
const searchFieldElm = page.locator('input[type=search]');
|
||||
const resultsHeadingElm = page.locator('.results-panel .title');
|
||||
|
||||
await docsifyInit(docsifyInitConfig);
|
||||
await searchFieldElm.fill('embeddedSearchKeyword');
|
||||
await expect(resultsHeadingElm).toHaveText('Include Search');
|
||||
});
|
||||
|
||||
test('search should index embedded include content from relative path', async ({
|
||||
page,
|
||||
}) => {
|
||||
const docsifyInitConfig = {
|
||||
markdown: {
|
||||
homepage: '# Home',
|
||||
sidebar: '- [Guide Intro](guide/intro)',
|
||||
},
|
||||
routes: {
|
||||
'/guide/intro.md': `
|
||||
# Relative Include Search
|
||||
|
||||
|
||||
`,
|
||||
'/guide/snippets/demo.js': `
|
||||
const embeddedRelativeKeyword = 'ok';
|
||||
`,
|
||||
},
|
||||
scriptURLs: ['/dist/plugins/search.js'],
|
||||
};
|
||||
|
||||
const searchFieldElm = page.locator('input[type=search]');
|
||||
const resultsHeadingElm = page.locator('.results-panel .title');
|
||||
|
||||
await docsifyInit(docsifyInitConfig);
|
||||
await searchFieldElm.fill('embeddedRelativeKeyword');
|
||||
await expect(resultsHeadingElm).toHaveText('Relative Include Search');
|
||||
});
|
||||
|
||||
test('search result should remove checkbox markdown and keep related values', async ({
|
||||
page,
|
||||
}) => {
|
||||
|
||||
Reference in New Issue
Block a user