mirror of
https://github.com/DIYgod/RSSHub.git
synced 2026-10-17 05:11:57 +08:00
DIYgod
b4ae78140d
Merge current master while retaining the completed route review and retired service removals. Keep upstream source migrations and new feed functionality, including the ACEA API, Xiaomi data helpers, Yahoo Finance topic helper, Bilibili list URLs, Douban IP locations, and Douyin live-stream handling. Resolve overlapping changes without losing existing limit validation, bounded detail requests, stable item metadata, or route configuration corrections. Keep RSS image MIME inference and explicit media precedence alongside upstream attachment extraction, emitting a single enclosure per item.
471 lines
21 KiB
TypeScript
471 lines
21 KiB
TypeScript
import type { CheerioAPI } from 'cheerio';
|
|
import { load } from 'cheerio';
|
|
import type { Element } from 'domhandler';
|
|
import { decodeHTMLStrict } from 'entities';
|
|
import type { MiddlewareHandler } from 'hono';
|
|
import { convert } from 'html-to-text';
|
|
import markdownit from 'markdown-it';
|
|
import { RE2JS } from 're2js';
|
|
import sanitizeHtml from 'sanitize-html';
|
|
|
|
import { config } from '@/config';
|
|
import type { Data, DataItem } from '@/types';
|
|
import { extractAttachments } from '@/utils/attachments';
|
|
import cache from '@/utils/cache';
|
|
import { isWorker } from '@/utils/is-worker';
|
|
import ofetch from '@/utils/ofetch';
|
|
|
|
const md = markdownit({
|
|
html: true,
|
|
});
|
|
|
|
const resolveRelativeLink = ($: CheerioAPI, elem: Element, attr: string, baseUrl?: string) => {
|
|
const $elem = $(elem);
|
|
|
|
if (baseUrl) {
|
|
try {
|
|
const oldAttr = $elem.attr(attr);
|
|
if (oldAttr) {
|
|
// e.g. <video><source src="https://example.com"></video> should leave <video> unchanged
|
|
$elem.attr(attr, new URL(oldAttr, baseUrl).href);
|
|
}
|
|
} catch {
|
|
// no-empty
|
|
}
|
|
}
|
|
};
|
|
|
|
const getAiCompletion = async (prompt: string, text: string) => {
|
|
const apiUrl = `${config.openai.endpoint}/chat/completions`;
|
|
const response = await ofetch(apiUrl, {
|
|
method: 'POST',
|
|
body: {
|
|
model: config.openai.model,
|
|
max_tokens: config.openai.maxTokens,
|
|
messages: [
|
|
{ role: 'system', content: prompt },
|
|
{ role: 'user', content: text },
|
|
],
|
|
temperature: config.openai.temperature,
|
|
},
|
|
headers: {
|
|
Authorization: `Bearer ${config.openai.apiKey}`,
|
|
},
|
|
});
|
|
|
|
return response.choices[0].message.content;
|
|
};
|
|
|
|
const getAuthorString = (item: DataItem) => {
|
|
let author = '';
|
|
if (item.author) {
|
|
author = Array.isArray(item.author) ? item.author.map((i) => i.name).join(' ') : item.author;
|
|
}
|
|
return author;
|
|
};
|
|
|
|
const middleware: MiddlewareHandler = async (ctx, next) => {
|
|
await next();
|
|
|
|
const data: Data = ctx.get('data');
|
|
if (data) {
|
|
if ((!data.item || data.item.length === 0) && !data.allowEmpty) {
|
|
throw new Error('this route is empty, please check the original site or <a href="https://github.com/DIYgod/RSSHub/issues/new/choose">create an issue</a>');
|
|
}
|
|
|
|
// fix allowEmpty
|
|
data.item ||= [];
|
|
|
|
// decode HTML entities
|
|
// oxlint-disable-next-line @typescript-eslint/no-unnecessary-type-conversion -- routes may return non-string values at runtime
|
|
data.title &&= decodeHTMLStrict(data.title + '');
|
|
// oxlint-disable-next-line @typescript-eslint/no-unnecessary-type-conversion -- routes may return non-string values at runtime
|
|
data.description &&= decodeHTMLStrict(data.description + '');
|
|
|
|
// sort items
|
|
const sorted = ctx.req.query('sorted');
|
|
if (sorted === undefined ? data.sortItems !== false : sorted !== 'false') {
|
|
data.item = data.item.toSorted((a: DataItem, b: DataItem) => +new Date(b.pubDate || 0) - +new Date(a.pubDate || 0));
|
|
}
|
|
|
|
const hideImages = ctx.req.query('show_image') === 'false';
|
|
const extractEnclosures = ctx.req.query('enclosure') === 'true';
|
|
if (hideImages) {
|
|
delete data.image;
|
|
}
|
|
|
|
const handleItem = (item: DataItem) => {
|
|
if (hideImages) {
|
|
delete item.image;
|
|
delete item.banner;
|
|
delete item.itunes_item_image;
|
|
if (item.enclosure_type?.startsWith('image/')) {
|
|
delete item.enclosure_url;
|
|
delete item.enclosure_type;
|
|
delete item.enclosure_length;
|
|
}
|
|
if (item.media) {
|
|
delete item.media.thumbnail;
|
|
if (item.media.content?.type?.startsWith('image/')) {
|
|
delete item.media.content;
|
|
}
|
|
}
|
|
item.attachments = item.attachments?.filter((attachment) => !attachment.mime_type?.startsWith('image/'));
|
|
}
|
|
// oxlint-disable-next-line @typescript-eslint/no-unnecessary-type-conversion -- routes may return non-string values at runtime
|
|
item.title &&= decodeHTMLStrict(item.title + '');
|
|
item.description ||= item.content?.html;
|
|
|
|
// handle pubDate
|
|
if (item.pubDate) {
|
|
item.pubDate = new Date(item.pubDate).toUTCString();
|
|
}
|
|
|
|
// handle link
|
|
if (item.link) {
|
|
let baseUrl = data.link;
|
|
if (baseUrl && !/^https?:\/\//.test(baseUrl)) {
|
|
baseUrl = baseUrl.startsWith('//') ? 'http:' + baseUrl : 'http://' + baseUrl;
|
|
}
|
|
|
|
item.link = new URL(item.link, baseUrl).href;
|
|
}
|
|
|
|
// handle description
|
|
if (item.description) {
|
|
const $ = load(item.description);
|
|
let baseUrl = item.link || data.link;
|
|
|
|
if (baseUrl && !/^https?:\/\//.test(baseUrl)) {
|
|
baseUrl = baseUrl.startsWith('//') ? 'http:' + baseUrl : 'http://' + baseUrl;
|
|
}
|
|
|
|
$('script').remove();
|
|
if (hideImages) {
|
|
$('img, picture').remove();
|
|
$('video[poster]').removeAttr('poster');
|
|
}
|
|
|
|
$('img').each((_, ele) => {
|
|
const $ele = $(ele);
|
|
|
|
// fix lazyload
|
|
if (!$ele.attr('src')) {
|
|
const lazySrc = $ele.attr('data-src') || $ele.attr('data-original');
|
|
if (lazySrc) {
|
|
$ele.attr('src', lazySrc);
|
|
} else {
|
|
for (const key in ele.attribs) {
|
|
const value = ele.attribs[key].trim();
|
|
if (['.gif', '.png', '.jpg', '.webp'].some((suffix) => value.includes(suffix))) {
|
|
$ele.attr('src', value);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// redundant attributes
|
|
for (const e of ['onclick', 'onerror', 'onload']) {
|
|
$ele.removeAttr(e);
|
|
}
|
|
});
|
|
|
|
// resolve relative link & fix referrer policy
|
|
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy
|
|
// https://www.w3schools.com/tags/att_href.asp
|
|
$('a, area').each((_, elem) => {
|
|
resolveRelativeLink($, elem, 'href', baseUrl);
|
|
// $(elem).attr('rel', 'noreferrer'); // currently no such a need
|
|
});
|
|
// https://www.w3schools.com/tags/att_src.asp
|
|
$('img, video, audio, source, iframe, embed, track').each((_, elem) => {
|
|
resolveRelativeLink($, elem, 'src', baseUrl);
|
|
});
|
|
$('video[poster]').each((_, elem) => {
|
|
resolveRelativeLink($, elem, 'poster', baseUrl);
|
|
});
|
|
$('img, iframe').each((_, elem) => {
|
|
if (!$(elem).attr('referrerpolicy')) {
|
|
$(elem).attr('referrerpolicy', 'no-referrer');
|
|
}
|
|
});
|
|
|
|
if (extractEnclosures) {
|
|
item.attachments = extractAttachments($, item, hideImages);
|
|
}
|
|
|
|
item.description = $('body').html() + '' + (config.suffix || '');
|
|
|
|
if (item._extra?.links && $('.rsshub-quote').length) {
|
|
item._extra?.links?.map((e) => {
|
|
e.content_html = $.html($('.rsshub-quote'));
|
|
return e;
|
|
});
|
|
}
|
|
}
|
|
|
|
// handle category
|
|
if (item.category) {
|
|
// convert single string to array, and filter only string type category
|
|
Array.isArray(item.category) || (item.category = [item.category]);
|
|
item.category = item.category.filter((e: unknown): e is string => typeof e === 'string');
|
|
}
|
|
return item;
|
|
};
|
|
|
|
data.item = await Promise.all(data.item.map((itm) => handleItem(itm)));
|
|
|
|
// filter
|
|
const engine = config.feature.filter_regex_engine;
|
|
const makeRegex = (str: string) => {
|
|
// default: case_senstivie = true
|
|
const insensitive = ctx.req.query('filter_case_sensitive') === 'false';
|
|
switch (engine) {
|
|
case 'regexp':
|
|
return new RegExp(str, insensitive ? 'i' : '');
|
|
case 're2':
|
|
return RE2JS.compile(str, insensitive ? RE2JS.CASE_INSENSITIVE : 0);
|
|
default:
|
|
throw new Error(`Invalid Engine Value: ${engine}, please check your config.`);
|
|
}
|
|
};
|
|
|
|
if (ctx.req.query('filter')) {
|
|
const regex = makeRegex(ctx.req.query('filter')!);
|
|
|
|
data.item = data.item.filter((item) => {
|
|
const title = item.title || '';
|
|
const description = item.description || title;
|
|
const author = getAuthorString(item);
|
|
const category = Array.isArray(item.category) ? item.category : [];
|
|
const isFilter =
|
|
regex instanceof RE2JS
|
|
? regex.matcher(title).find() || regex.matcher(description).find() || regex.matcher(author).find() || category.some((c) => regex.matcher(c).find())
|
|
: title.match(regex) || description.match(regex) || author.match(regex) || category.some((c) => c.match(regex));
|
|
|
|
return isFilter;
|
|
});
|
|
}
|
|
|
|
// 启用filter参数时，无效filter_title/description/author/category
|
|
if (!ctx.req.query('filter') && (ctx.req.query('filter_title') || ctx.req.query('filter_description') || ctx.req.query('filter_author') || ctx.req.query('filter_category'))) {
|
|
data.item = data.item.filter((item) => {
|
|
const title = item.title || '';
|
|
const description = item.description || title;
|
|
const author = getAuthorString(item);
|
|
const category = Array.isArray(item.category) ? item.category : [];
|
|
let isFilter = true;
|
|
|
|
if (ctx.req.query('filter_title')) {
|
|
const titleRegex = makeRegex(ctx.req.query('filter_title')!);
|
|
isFilter = titleRegex instanceof RE2JS ? titleRegex.matcher(title).find() : titleRegex.test(title);
|
|
}
|
|
if (ctx.req.query('filter_description')) {
|
|
const descriptionRegex = makeRegex(ctx.req.query('filter_description')!);
|
|
isFilter &&= descriptionRegex instanceof RE2JS ? descriptionRegex.matcher(description).find() : descriptionRegex.test(description);
|
|
}
|
|
if (ctx.req.query('filter_author')) {
|
|
const authorRegex = makeRegex(ctx.req.query('filter_author')!);
|
|
isFilter &&= authorRegex instanceof RE2JS ? authorRegex.matcher(author).find() : authorRegex.test(author);
|
|
}
|
|
if (ctx.req.query('filter_category')) {
|
|
const categoryRegex = makeRegex(ctx.req.query('filter_category')!);
|
|
isFilter &&= category.some((c) => (categoryRegex instanceof RE2JS ? categoryRegex.matcher(c).find() : c.match(categoryRegex)));
|
|
}
|
|
|
|
return isFilter;
|
|
});
|
|
}
|
|
|
|
if (ctx.req.query('filterout') || ctx.req.query('filterout_title') || ctx.req.query('filterout_description') || ctx.req.query('filterout_author') || ctx.req.query('filterout_category')) {
|
|
data.item = data.item.filter((item) => {
|
|
const title = item.title;
|
|
const description = item.description || title;
|
|
const author = getAuthorString(item);
|
|
const category = Array.isArray(item.category) ? item.category : [];
|
|
let isFilter = true;
|
|
|
|
if (ctx.req.query('filterout') || ctx.req.query('filterout_title')) {
|
|
const titleRegex = makeRegex(ctx.req.query('filterout_title') || ctx.req.query('filterout')!);
|
|
isFilter = titleRegex instanceof RE2JS ? !titleRegex.matcher(title).find() : !titleRegex.test(title);
|
|
}
|
|
if (ctx.req.query('filterout') || ctx.req.query('filterout_description')) {
|
|
const descriptionRegex = makeRegex(ctx.req.query('filterout_description') || ctx.req.query('filterout')!);
|
|
isFilter &&= descriptionRegex instanceof RE2JS ? !descriptionRegex.matcher(description).find() : !descriptionRegex.test(description);
|
|
}
|
|
if (ctx.req.query('filterout_author')) {
|
|
const authorRegex = makeRegex(ctx.req.query('filterout_author')!);
|
|
isFilter &&= authorRegex instanceof RE2JS ? !authorRegex.matcher(author).find() : !authorRegex.test(author);
|
|
}
|
|
if (ctx.req.query('filterout_category')) {
|
|
const categoryRegex = makeRegex(ctx.req.query('filterout_category')!);
|
|
isFilter &&= category.every((c) => !(categoryRegex instanceof RE2JS ? categoryRegex.matcher(c).find() : c.match(categoryRegex)));
|
|
}
|
|
|
|
return isFilter;
|
|
});
|
|
}
|
|
|
|
if (ctx.req.query('filter_time')) {
|
|
const now = Date.now();
|
|
data.item = data.item.filter(({ pubDate }) => {
|
|
let isFilter = true;
|
|
try {
|
|
isFilter = !pubDate || now - new Date(pubDate).getTime() <= Number.parseInt(ctx.req.query('filter_time')!) * 1000;
|
|
} catch {
|
|
// no-empty
|
|
}
|
|
return isFilter;
|
|
});
|
|
}
|
|
|
|
// limit
|
|
const limit = Number.parseInt(ctx.req.query('limit') ?? '');
|
|
if (limit >= 0) {
|
|
data.item = data.item.slice(0, limit);
|
|
}
|
|
|
|
// telegram instant view
|
|
if (ctx.req.query('tgiv')) {
|
|
data.item.map((item) => {
|
|
if (item.link) {
|
|
const encodedlink = encodeURIComponent(item.link);
|
|
item.link = `https://t.me/iv?url=${encodedlink}&rhash=${ctx.req.query('tgiv')}`;
|
|
return item;
|
|
}
|
|
return item;
|
|
});
|
|
}
|
|
|
|
// fulltext
|
|
if (ctx.req.query('mode')?.toLowerCase() === 'fulltext') {
|
|
const tasks = data.item.map(async (item) => {
|
|
const { link, author, description } = item;
|
|
const parsed_result: any = await cache.tryGet(`mercury-cache-${link}`, async () => {
|
|
if (link) {
|
|
// if parser failed, return default description and not report error
|
|
try {
|
|
const { default: Parser } = await import('@jocmp/mercury-parser');
|
|
const res = await ofetch(link);
|
|
const $ = load(res);
|
|
const result = await Parser.parse(link, {
|
|
html: $.html(),
|
|
});
|
|
return result;
|
|
} catch {
|
|
// no-empty
|
|
}
|
|
}
|
|
});
|
|
|
|
item.author = author || parsed_result?.author;
|
|
item.description = parsed_result && parsed_result.content.length > 40 ? decodeHTMLStrict(parsed_result.content) : description;
|
|
if (extractEnclosures && parsed_result?.content.length > 40 && item.description) {
|
|
item.attachments = extractAttachments(load(item.description), item, hideImages);
|
|
}
|
|
});
|
|
await Promise.all(tasks);
|
|
}
|
|
|
|
// openai
|
|
if (ctx.req.query('chatgpt') && config.openai.apiKey) {
|
|
data.item = await Promise.all(
|
|
data.item.map(async (item) => {
|
|
try {
|
|
// handle description
|
|
if (config.openai.inputOption === 'description' && item.description) {
|
|
const description = await cache.tryGet(`openai:description:${item.link}`, async () => {
|
|
const description = convert(item.description!);
|
|
const descriptionMd = await getAiCompletion(config.openai.promptDescription, description);
|
|
return md.render(descriptionMd);
|
|
});
|
|
// add it to the description
|
|
if (description !== '') {
|
|
item.description = description + '<hr/><br/>' + item.description;
|
|
}
|
|
}
|
|
// handle title
|
|
else if (config.openai.inputOption === 'title' && item.title) {
|
|
const title = await cache.tryGet(`openai:title:${item.link}`, async () => {
|
|
const title = convert(item.title!);
|
|
return await getAiCompletion(config.openai.promptTitle, title);
|
|
});
|
|
// replace the title
|
|
if (title !== '') {
|
|
item.title = title + '';
|
|
}
|
|
}
|
|
// handle both
|
|
else if (config.openai.inputOption === 'both' && item.title && item.description) {
|
|
const title = await cache.tryGet(`openai:title:${item.link}`, async () => {
|
|
const title = convert(item.title!);
|
|
return await getAiCompletion(config.openai.promptTitle, title);
|
|
});
|
|
// replace the title
|
|
if (title !== '') {
|
|
item.title = title + '';
|
|
}
|
|
|
|
const description = await cache.tryGet(`openai:description:${item.link}`, async () => {
|
|
const description = convert(item.description!);
|
|
const descriptionMd = await getAiCompletion(config.openai.promptDescription, description);
|
|
return md.render(descriptionMd);
|
|
});
|
|
// add it to the description
|
|
if (description !== '') {
|
|
item.description = description + '<hr/><br/>' + item.description;
|
|
}
|
|
}
|
|
} catch {
|
|
// when openai failed, return default content and not write cache
|
|
}
|
|
return item;
|
|
})
|
|
);
|
|
}
|
|
|
|
// scihub
|
|
if (ctx.req.query('scihub')) {
|
|
data.item.map((item) => {
|
|
item.link = item.doi ? `${config.scihub.host}${item.doi}` : `${config.scihub.host}${item.link}`;
|
|
return item;
|
|
});
|
|
}
|
|
|
|
// opencc
|
|
if (!isWorker && ctx.req.query('opencc')) {
|
|
const { simplecc } = await import('simplecc-wasm');
|
|
for (const item of data.item) {
|
|
item.title = simplecc(item.title ?? item.link, ctx.req.query('opencc')!);
|
|
item.description = simplecc(item.description ?? item.title ?? item.link, ctx.req.query('opencc')!);
|
|
}
|
|
}
|
|
|
|
// brief
|
|
if (ctx.req.query('brief')) {
|
|
const num = /[1-9]\d{2,}/;
|
|
if (num.test(ctx.req.query('brief')!)) {
|
|
const brief: number = Number.parseInt(ctx.req.query('brief')!);
|
|
for (const item of data.item) {
|
|
if (!item.description) {
|
|
continue;
|
|
}
|
|
|
|
const text = sanitizeHtml(item.description, { allowedTags: [], allowedAttributes: {} });
|
|
item.description = text.length > brief ? `<p>${text.slice(0, brief)}…</p>` : `<p>${text}</p>`;
|
|
}
|
|
} else {
|
|
throw new Error('Invalid parameter brief. Please check the doc https://docs.rsshub.app/guide/parameters#shu-chu-jian-xun');
|
|
}
|
|
}
|
|
// some parameters are processed in `anti-hotlink.js`
|
|
|
|
ctx.set('data', data);
|
|
} else {
|
|
// throw new Error('wrong path');
|
|
}
|
|
};
|
|
|
|
export default middleware;
|