GithubStar/RSSHub
1
0
Fork 0
mirror of https://github.com/DIYgod/RSSHub.git synced 2026-10-16 21:02:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(medium): recover authenticated feeds with source previews

Browse Source
This commit is contained in:
DIYgod
2026-10-15 19:36:37 +08:00
parent 70284f0574
commit c3151da81e
6 changed files with 275 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/routes/medium/browser.ts
+52
View File
@@ -0,0 +1,52 @@
import { getPlaywrightPage } from '@/utils/playwright';
type MediumBrowser = Awaited<ReturnType<typeof getPlaywrightPage>>;
export type GetMediumBrowser = () => Promise<MediumBrowser>;
export function isMediumChallenge(error: any) {
const body = error?.data ?? error?.response?._data;
const html = typeof body === 'string' ? body : body?.body;
return (error?.statusCode ?? error?.status ?? error?.response?.status) === 403 && typeof html === 'string' && /cf-chl|challenge-platform|Just a moment/i.test(html);
}
async function createMediumBrowser(cookie: string) {
const browser = await getPlaywrightPage('https://medium.com/', { noGoto: true, closeTimeout: 0 });
try {
await browser.context.route('**/*', async (route) => {
await (['document', 'script', 'stylesheet', 'xhr', 'fetch'].includes(route.request().resourceType()) ? route.continue() : route.abort());
});
const cookies = cookie
.split(';')
.map((part) => part.trim())
.filter((part) => part.indexOf('=') > 0)
.map((part) => {
const separator = part.indexOf('=');
return { name: part.slice(0, separator), value: part.slice(separator + 1), url: 'https://medium.com/', secure: true };
});
await browser.context.addCookies(cookies);
await browser.page.goto('https://medium.com/', { waitUntil: 'domcontentloaded', timeout: 20000 });
if (new URL(browser.page.url()).origin !== 'https://medium.com') {
throw new Error('The Medium browser source redirected outside its expected origin.');
}
return browser;
} catch (error) {
await browser.destroy();
throw error;
}
}
export async function withMediumBrowser<T>(cookie: string, callback: (getBrowser: GetMediumBrowser) => Promise<T>) {
let pending: Promise<MediumBrowser> | undefined;
let browser: MediumBrowser | undefined;
const getBrowser = async () => {
browser = await (pending ??= createMediumBrowser(cookie));
return browser;
};
try {
return await callback(getBrowser);
} finally {
if (browser) {
await browser.destroy();
}
}
}
lib/routes/medium/following.ts
+20 -17
View File
@@ -4,6 +4,7 @@ import { config } from '@/config';
import ConfigNotFoundError from '@/errors/types/config-not-found.js';
import type { Route } from '@/types';
import { withMediumBrowser } from './browser';
import { getFollowingFeedQuery } from './graphql.js';
import parseArticle from './parse-article.js';
@@ -19,7 +20,7 @@ export const route: Route = {
description: '',
},
],
requirePuppeteer: false,
requirePuppeteer: true,
antiCrawler: false,
supportBT: false,
supportPodcast: false,
@@ -30,10 +31,12 @@ export const route: Route = {
handler,
description: `::: warning
Personalized recommendations require the cookie value after logging in, so only self-hosting is supported. See the configuration module on the deployment page for details.
:::`,
:::
When Medium blocks article pages, their publicly visible previews may be returned.`,
};
async function handler(ctx) {
function handler(ctx) {
const user = ctx.req.param('user');
const cookie = config.medium.cookies[user];
@@ -41,21 +44,21 @@ async function handler(ctx) {
throw new ConfigNotFoundError(`缺少 Medium 用户 ${user} 登录后的 Cookie 值`);
}
const posts = await getFollowingFeedQuery(user, cookie);
ctx.set('json', posts);
return withMediumBrowser(cookie, async (getBrowser) => {
const posts = await getFollowingFeedQuery(user, cookie, undefined, getBrowser);
ctx.set('json', posts);
if (!posts) {
// login failed
throw new ConfigNotFoundError(`Medium 用户 ${user} 的 Cookie 无效或已过期`);
}
if (!posts) {
// login failed
throw new ConfigNotFoundError(`Medium 用户 ${user} 的 Cookie 无效或已过期`);
}
const urls: string[] = posts.items.map((data) => data.post.mediumUrl);
const parsedArticles = await pMap(posts.items, ({ post }) => parseArticle(ctx, post.mediumUrl, { cookie, post }), { concurrency: 3 });
const parsedArticles = await pMap(urls, (url) => parseArticle(ctx, url), { concurrency: 3 });
return {
title: `${user} Medium Following`,
link: 'https://medium.com/?feed=following',
item: parsedArticles,
};
return {
title: `${user} Medium Following`,
link: 'https://medium.com/?feed=following',
item: parsedArticles,
};
});
}
lib/routes/medium/for-you.ts
+20 -17
View File
@@ -4,6 +4,7 @@ import { config } from '@/config';
import ConfigNotFoundError from '@/errors/types/config-not-found.js';
import type { Route } from '@/types';
import { withMediumBrowser } from './browser';
import { getWebInlineRecommendedFeedQuery } from './graphql.js';
import parseArticle from './parse-article.js';
@@ -19,7 +20,7 @@ export const route: Route = {
description: '',
},
],
requirePuppeteer: false,
requirePuppeteer: true,
antiCrawler: false,
supportBT: false,
supportPodcast: false,
@@ -30,10 +31,12 @@ export const route: Route = {
handler,
description: `::: warning
Personalized recommendations require the cookie value after logging in, so only self-hosting is supported. See the configuration module on the deployment page for details.
:::`,
:::
When Medium blocks article pages, their publicly visible previews may be returned.`,
};
async function handler(ctx) {
function handler(ctx) {
const user = ctx.req.param('user');
const cookie = config.medium.cookies[user];
@@ -41,21 +44,21 @@ async function handler(ctx) {
throw new ConfigNotFoundError(`缺少 Medium 用户 ${user} 登录后的 Cookie 值`);
}
const posts = await getWebInlineRecommendedFeedQuery(user, cookie);
ctx.set('json', posts);
return withMediumBrowser(cookie, async (getBrowser) => {
const posts = await getWebInlineRecommendedFeedQuery(user, cookie, undefined, getBrowser);
ctx.set('json', posts);
if (!posts) {
// login failed
throw new ConfigNotFoundError(`Medium 用户 ${user} 的 Cookie 无效或已过期`);
}
if (!posts) {
// login failed
throw new ConfigNotFoundError(`Medium 用户 ${user} 的 Cookie 无效或已过期`);
}
const urls: string[] = posts.items.map((data) => data.post.mediumUrl);
const parsedArticles = await pMap(posts.items, ({ post }) => parseArticle(ctx, post.mediumUrl, { cookie, post }), { concurrency: 3 });
const parsedArticles = await pMap(urls, (url) => parseArticle(ctx, url), { concurrency: 3 });
return {
title: `${user} Medium For You`,
link: 'https://medium.com/',
item: parsedArticles,
};
return {
title: `${user} Medium For You`,
link: 'https://medium.com/',
item: parsedArticles,
};
});
}
lib/routes/medium/graphql.ts
+102 -33
View File
@@ -1,46 +1,112 @@
import got from '@/utils/got';
async function graphqlRequest(body, cookie) {
const { data } = await got('https://medium.com/_/graphql', {
method: 'POST',
headers: {
accept: '*/*',
'accept-language': 'en-US,en;q=0.9,zh;q=0.8,zh-CN;q=0.7',
'apollographql-client-name': 'lite',
'apollographql-client-version': 'main-20240329-011934-2370d8b72b',
'cache-control': 'no-cache',
'content-type': 'application/json',
'medium-frontend-app': 'lite/main-20240329-011934-2370d8b72b',
'medium-frontend-path': '/',
'medium-frontend-route': 'homepage',
'ot-tracer-sampled': 'true',
'ot-tracer-spanid': '26b843316dc9494d',
'ot-tracer-traceid': 'c84ea9154765033',
pragma: 'no-cache',
'sec-ch-ua': '"Chromium";v="123", "Not:A-Brand";v="8"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"macOS"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'graphql-operation': body.operationName,
cookie,
import { type GetMediumBrowser, isMediumChallenge } from './browser';
export type PostPreview = {
mediumUrl: string;
title: string;
firstPublishedAt: number;
creator?: { name?: string };
extendedPreviewContent?: { subtitle?: string };
};
type Feed = { items: Array<{ post: PostPreview }> };
const postPreviewFields = `
id
title
firstPublishedAt
creator { name }
extendedPreviewContent { isFullContent subtitle }
`;
async function browserGraphqlRequest(browser: Awaited<ReturnType<GetMediumBrowser>>, body, headers, retry = true): Promise<{ status: number; data: unknown; challenge: boolean }> {
const response = await browser.page.evaluate(
async ({ body, headers }) => {
const response = await fetch('https://medium.com/_/graphql', {
method: 'POST',
redirect: 'manual',
headers,
body,
});
return {
status: response.status,
data: response.ok ? await response.json() : undefined,
challenge: response.status === 403 && /cf-chl|challenge-platform|Just a moment/i.test(await response.text()),
};
},
body: JSON.stringify([body]),
});
{ body: JSON.stringify([body]), headers }
);
if (retry && response.status === 403 && response.challenge) {
await browser.page.reload({ waitUntil: 'domcontentloaded', timeout: 20000 });
if (new URL(browser.page.url()).origin !== 'https://medium.com') {
throw new Error('The Medium browser source redirected outside its expected origin.');
}
return browserGraphqlRequest(browser, body, headers, false);
}
return response;
}
async function graphqlRequest(body, cookie, getBrowser?: GetMediumBrowser) {
const headers = {
accept: '*/*',
'accept-language': 'en-US,en;q=0.9,zh;q=0.8,zh-CN;q=0.7',
'apollographql-client-name': 'lite',
'apollographql-client-version': 'main-20240329-011934-2370d8b72b',
'cache-control': 'no-cache',
'content-type': 'application/json',
'medium-frontend-app': 'lite/main-20240329-011934-2370d8b72b',
'medium-frontend-path': '/',
'medium-frontend-route': 'homepage',
'ot-tracer-sampled': 'true',
'ot-tracer-spanid': '26b843316dc9494d',
'ot-tracer-traceid': 'c84ea9154765033',
pragma: 'no-cache',
'sec-ch-ua': '"Chromium";v="123", "Not:A-Brand";v="8"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"macOS"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'graphql-operation': body.operationName,
cookie,
};
let data;
try {
({ data } = await got('https://medium.com/_/graphql', {
method: 'POST',
redirect: 'manual',
headers,
body: JSON.stringify([body]),
}));
} catch (error) {
if (!getBrowser || !isMediumChallenge(error)) {
throw error;
}
const browser = await getBrowser();
const publicHeaders = Object.fromEntries(Object.entries(headers).filter(([name]) => name !== 'cookie'));
const response = await browserGraphqlRequest(browser, body, publicHeaders);
if (response.status !== 200) {
throw new Error(`The Medium GraphQL source returned HTTP ${response.status}. Verify the session and try again later.`, { cause: error });
}
data = response.data;
}
if (!Array.isArray(data)) {
throw new TypeError('The Medium GraphQL source did not return its expected JSON response.');
}
return data[0].data;
}
async function getFollowingFeedQuery(user, cookie, pagingLimit = 20) {
return (await graphqlRequest(newFollowingFeedQuery(pagingLimit), cookie))?.followingFeed;
async function getFollowingFeedQuery(user, cookie, pagingLimit = 20, getBrowser?: GetMediumBrowser): Promise<Feed | undefined> {
return (await graphqlRequest(newFollowingFeedQuery(pagingLimit), cookie, getBrowser))?.followingFeed;
}
async function getWebInlineRecommendedFeedQuery(user, cookie, pagingLimit = 20) {
return (await graphqlRequest(newWebInlineRecommendedFeedQuery(pagingLimit), cookie))?.webRecommendedFeed;
async function getWebInlineRecommendedFeedQuery(user, cookie, pagingLimit = 20, getBrowser?: GetMediumBrowser): Promise<Feed | undefined> {
return (await graphqlRequest(newWebInlineRecommendedFeedQuery(pagingLimit), cookie, getBrowser))?.webRecommendedFeed;
}
async function getWebInlineTopicFeedQuery(user, tagSlug, cookie, pagingLimit = 20) {
return (await graphqlRequest(newWebInlineTopicFeedQuery(tagSlug, pagingLimit), cookie))?.personalisedTagFeed;
async function getWebInlineTopicFeedQuery(user, tagSlug, cookie, pagingLimit = 20, getBrowser?: GetMediumBrowser): Promise<Feed | undefined> {
return (await graphqlRequest(newWebInlineTopicFeedQuery(tagSlug, pagingLimit), cookie, getBrowser))?.personalisedTagFeed;
}
async function getUserCatalogMainContentQuery(user, catalogId, cookie, pagingLimit = 20) {
@@ -64,6 +130,7 @@ function newFollowingFeedQuery(pagingLimit = 5) {
feedId
post {
mediumUrl
${postPreviewFields}
__typename
}
__typename
@@ -101,6 +168,7 @@ function newWebInlineRecommendedFeedQuery(pagingLimit = 5) {
feedId
post {
mediumUrl
${postPreviewFields}
__typename
}
__typename
@@ -138,6 +206,7 @@ function newWebInlineTopicFeedQuery(tagSlug, pagingLimit = 5) {
feedId
post {
mediumUrl
${postPreviewFields}
__typename
}
__typename
lib/routes/medium/parse-article.ts
+61 -16
View File
@@ -1,16 +1,41 @@
import { load } from 'cheerio';
import { escapeUTF8 } from 'entities/escape';
import { config } from '@/config';
import cache from '@/utils/cache';
import got from '@/utils/got';
import md5 from '@/utils/md5';
import ofetch from '@/utils/ofetch';
import { parseDate } from '@/utils/parse-date';
async function parse(url, cookie = '') {
const { data } = await got(url, {
headers: {
cookie,
},
import { isMediumChallenge } from './browser';
import type { PostPreview } from './graphql';
function isMediumUrl(url: URL) {
return url.protocol === 'https:' && (url.hostname === 'medium.com' || url.hostname.endsWith('.medium.com'));
}
async function fetchArticle(url: string, cookie: string, redirects = 0): Promise<string> {
const response = await ofetch.raw<string, 'text'>(url, {
headers: cookie && isMediumUrl(new URL(url)) ? { cookie } : {},
redirect: 'manual',
responseType: 'text',
});
if (response.status >= 300 && response.status < 400) {
const location = response.headers.get('location');
if (!location || redirects >= 5) {
throw new Error('The Medium article source returned an invalid redirect.');
}
const next = new URL(location, url);
if (!['https:', 'http:'].includes(next.protocol)) {
throw new Error('The Medium article source returned an unsupported redirect.');
}
return fetchArticle(next.href, cookie, redirects + 1);
}
return response._data ?? '';
}
async function parse(url, cookie = '') {
const data = await fetchArticle(url, cookie);
const $ = load(data);
const publishedTime = $('meta[property="article:published_time"]').attr('content');
@@ -30,6 +55,10 @@ async function parse(url, cookie = '') {
const subtitle = article.find('.pw-subtitle-paragraph');
const subtitleText = subtitle.text();
if (!titleText.trim() || !article.length || (!article.text().trim() && !article.find('img[src], video[src], audio[src], iframe[src]').length)) {
throw new Error('The Medium article source did not contain a usable article.');
}
return {
title: titleText,
subtitle: subtitleText,
@@ -40,16 +69,32 @@ async function parse(url, cookie = '') {
};
}
export default function parseArticle(ctx, url) {
return cache.tryGet(`medium:article:${url}`, async () => {
const { title, author, publishedTime, html } = await parse(url, config.medium.articleCookie);
export default function parseArticle(ctx, url, options: { cookie?: string; post?: PostPreview } = {}) {
const sessionKey = md5(JSON.stringify([options.cookie ?? '', config.medium.articleCookie]));
return cache.tryGet(`medium:article:${sessionKey}:${url}`, async () => {
try {
const { title, author, publishedTime, html } = await parse(url, config.medium.articleCookie);
return {
title,
author,
link: url,
description: html,
pubDate: publishedTime ? parseDate(publishedTime) : undefined,
};
return {
title,
author,
link: url,
description: html,
pubDate: publishedTime ? parseDate(publishedTime) : undefined,
};
} catch (error) {
const post = options.post;
const preview = post?.extendedPreviewContent?.subtitle;
if (!post || !isMediumChallenge(error) || post.mediumUrl !== url || !post.title?.trim() || !preview?.trim() || !(post.firstPublishedAt > 0)) {
throw error;
}
return {
title: post.title,
author: post.creator?.name,
link: url,
description: `<p>${escapeUTF8(preview)}</p>`,
pubDate: parseDate(post.firstPublishedAt),
};
}
});
}
lib/routes/medium/tag.ts
+20 -17
View File
@@ -4,6 +4,7 @@ import { config } from '@/config';
import ConfigNotFoundError from '@/errors/types/config-not-found.js';
import type { Route } from '@/types';
import { withMediumBrowser } from './browser';
import { getWebInlineTopicFeedQuery } from './graphql.js';
import parseArticle from './parse-article.js';
@@ -19,7 +20,7 @@ export const route: Route = {
description: '',
},
],
requirePuppeteer: false,
requirePuppeteer: true,
antiCrawler: false,
supportBT: false,
supportPodcast: false,
@@ -32,10 +33,12 @@ export const route: Route = {
::: warning
Personalized recommendations require the cookie value after logging in, so only self-hosting is supported. See the configuration module on the deployment page for details.
:::`,
:::
When Medium blocks article pages, their publicly visible previews may be returned.`,
};
async function handler(ctx) {
function handler(ctx) {
const user = ctx.req.param('user');
const tag = ctx.req.param('tag');
@@ -44,21 +47,21 @@ async function handler(ctx) {
throw new ConfigNotFoundError(`缺少 Medium 用户 ${user} 登录后的 Cookie 值`);
}
const posts = await getWebInlineTopicFeedQuery(user, tag, cookie);
ctx.set('json', posts);
return withMediumBrowser(cookie, async (getBrowser) => {
const posts = await getWebInlineTopicFeedQuery(user, tag, cookie, undefined, getBrowser);
ctx.set('json', posts);
if (!posts) {
// login failed
throw new ConfigNotFoundError(`Medium 用户 ${user} 的 Cookie 无效或已过期`);
}
if (!posts) {
// login failed
throw new ConfigNotFoundError(`Medium 用户 ${user} 的 Cookie 无效或已过期`);
}
const urls: string[] = posts.items.map((data) => data.post.mediumUrl);
const parsedArticles = await pMap(posts.items, ({ post }) => parseArticle(ctx, post.mediumUrl, { cookie, post }), { concurrency: 3 });
const parsedArticles = await pMap(urls, (url) => parseArticle(ctx, url), { concurrency: 3 });
return {
title: `${user} Medium Following Tag ${tag}`,
link: `https://medium.com/?tag=${tag}`,
item: parsedArticles,
};
return {
title: `${user} Medium Following Tag ${tag}`,
link: `https://medium.com/?tag=${tag}`,
item: parsedArticles,
};
});
}