From c3151da81e5811c9066ffde599613e7671a2a732 Mon Sep 17 00:00:00 2001 From: DIYgod Date: Thu, 15 Oct 2026 19:36:37 +0800 Subject: [PATCH] fix(medium): recover authenticated feeds with source previews --- lib/routes/medium/browser.ts | 52 +++++++++++ lib/routes/medium/following.ts | 37 ++++---- lib/routes/medium/for-you.ts | 37 ++++---- lib/routes/medium/graphql.ts | 135 ++++++++++++++++++++++------- lib/routes/medium/parse-article.ts | 77 ++++++++++++---- lib/routes/medium/tag.ts | 37 ++++---- 6 files changed, 275 insertions(+), 100 deletions(-) create mode 100644 lib/routes/medium/browser.ts diff --git a/lib/routes/medium/browser.ts b/lib/routes/medium/browser.ts new file mode 100644 index 0000000000..1dee8ff722 --- /dev/null +++ b/lib/routes/medium/browser.ts @@ -0,0 +1,52 @@ +import { getPlaywrightPage } from '@/utils/playwright'; + +type MediumBrowser = Awaited>; +export type GetMediumBrowser = () => Promise; + +export function isMediumChallenge(error: any) { + const body = error?.data ?? error?.response?._data; + const html = typeof body === 'string' ? body : body?.body; + return (error?.statusCode ?? error?.status ?? error?.response?.status) === 403 && typeof html === 'string' && /cf-chl|challenge-platform|Just a moment/i.test(html); +} + +async function createMediumBrowser(cookie: string) { + const browser = await getPlaywrightPage('https://medium.com/', { noGoto: true, closeTimeout: 0 }); + try { + await browser.context.route('**/*', async (route) => { + await (['document', 'script', 'stylesheet', 'xhr', 'fetch'].includes(route.request().resourceType()) ? route.continue() : route.abort()); + }); + const cookies = cookie + .split(';') + .map((part) => part.trim()) + .filter((part) => part.indexOf('=') > 0) + .map((part) => { + const separator = part.indexOf('='); + return { name: part.slice(0, separator), value: part.slice(separator + 1), url: 'https://medium.com/', secure: true }; + }); + await browser.context.addCookies(cookies); + await browser.page.goto('https://medium.com/', { waitUntil: 'domcontentloaded', timeout: 20000 }); + if (new URL(browser.page.url()).origin !== 'https://medium.com') { + throw new Error('The Medium browser source redirected outside its expected origin.'); + } + return browser; + } catch (error) { + await browser.destroy(); + throw error; + } +} + +export async function withMediumBrowser(cookie: string, callback: (getBrowser: GetMediumBrowser) => Promise) { + let pending: Promise | undefined; + let browser: MediumBrowser | undefined; + const getBrowser = async () => { + browser = await (pending ??= createMediumBrowser(cookie)); + return browser; + }; + try { + return await callback(getBrowser); + } finally { + if (browser) { + await browser.destroy(); + } + } +} diff --git a/lib/routes/medium/following.ts b/lib/routes/medium/following.ts index 6924a69736..d385ff3722 100644 --- a/lib/routes/medium/following.ts +++ b/lib/routes/medium/following.ts @@ -4,6 +4,7 @@ import { config } from '@/config'; import ConfigNotFoundError from '@/errors/types/config-not-found.js'; import type { Route } from '@/types'; +import { withMediumBrowser } from './browser'; import { getFollowingFeedQuery } from './graphql.js'; import parseArticle from './parse-article.js'; @@ -19,7 +20,7 @@ export const route: Route = { description: '', }, ], - requirePuppeteer: false, + requirePuppeteer: true, antiCrawler: false, supportBT: false, supportPodcast: false, @@ -30,10 +31,12 @@ export const route: Route = { handler, description: `::: warning Personalized recommendations require the cookie value after logging in, so only self-hosting is supported. See the configuration module on the deployment page for details. -:::`, +::: + +When Medium blocks article pages, their publicly visible previews may be returned.`, }; -async function handler(ctx) { +function handler(ctx) { const user = ctx.req.param('user'); const cookie = config.medium.cookies[user]; @@ -41,21 +44,21 @@ async function handler(ctx) { throw new ConfigNotFoundError(`缺少 Medium 用户 ${user} 登录后的 Cookie 值`); } - const posts = await getFollowingFeedQuery(user, cookie); - ctx.set('json', posts); + return withMediumBrowser(cookie, async (getBrowser) => { + const posts = await getFollowingFeedQuery(user, cookie, undefined, getBrowser); + ctx.set('json', posts); - if (!posts) { - // login failed - throw new ConfigNotFoundError(`Medium 用户 ${user} 的 Cookie 无效或已过期`); - } + if (!posts) { + // login failed + throw new ConfigNotFoundError(`Medium 用户 ${user} 的 Cookie 无效或已过期`); + } - const urls: string[] = posts.items.map((data) => data.post.mediumUrl); + const parsedArticles = await pMap(posts.items, ({ post }) => parseArticle(ctx, post.mediumUrl, { cookie, post }), { concurrency: 3 }); - const parsedArticles = await pMap(urls, (url) => parseArticle(ctx, url), { concurrency: 3 }); - - return { - title: `${user} Medium Following`, - link: 'https://medium.com/?feed=following', - item: parsedArticles, - }; + return { + title: `${user} Medium Following`, + link: 'https://medium.com/?feed=following', + item: parsedArticles, + }; + }); } diff --git a/lib/routes/medium/for-you.ts b/lib/routes/medium/for-you.ts index 8eec4f048d..aa078deee7 100644 --- a/lib/routes/medium/for-you.ts +++ b/lib/routes/medium/for-you.ts @@ -4,6 +4,7 @@ import { config } from '@/config'; import ConfigNotFoundError from '@/errors/types/config-not-found.js'; import type { Route } from '@/types'; +import { withMediumBrowser } from './browser'; import { getWebInlineRecommendedFeedQuery } from './graphql.js'; import parseArticle from './parse-article.js'; @@ -19,7 +20,7 @@ export const route: Route = { description: '', }, ], - requirePuppeteer: false, + requirePuppeteer: true, antiCrawler: false, supportBT: false, supportPodcast: false, @@ -30,10 +31,12 @@ export const route: Route = { handler, description: `::: warning Personalized recommendations require the cookie value after logging in, so only self-hosting is supported. See the configuration module on the deployment page for details. -:::`, +::: + +When Medium blocks article pages, their publicly visible previews may be returned.`, }; -async function handler(ctx) { +function handler(ctx) { const user = ctx.req.param('user'); const cookie = config.medium.cookies[user]; @@ -41,21 +44,21 @@ async function handler(ctx) { throw new ConfigNotFoundError(`缺少 Medium 用户 ${user} 登录后的 Cookie 值`); } - const posts = await getWebInlineRecommendedFeedQuery(user, cookie); - ctx.set('json', posts); + return withMediumBrowser(cookie, async (getBrowser) => { + const posts = await getWebInlineRecommendedFeedQuery(user, cookie, undefined, getBrowser); + ctx.set('json', posts); - if (!posts) { - // login failed - throw new ConfigNotFoundError(`Medium 用户 ${user} 的 Cookie 无效或已过期`); - } + if (!posts) { + // login failed + throw new ConfigNotFoundError(`Medium 用户 ${user} 的 Cookie 无效或已过期`); + } - const urls: string[] = posts.items.map((data) => data.post.mediumUrl); + const parsedArticles = await pMap(posts.items, ({ post }) => parseArticle(ctx, post.mediumUrl, { cookie, post }), { concurrency: 3 }); - const parsedArticles = await pMap(urls, (url) => parseArticle(ctx, url), { concurrency: 3 }); - - return { - title: `${user} Medium For You`, - link: 'https://medium.com/', - item: parsedArticles, - }; + return { + title: `${user} Medium For You`, + link: 'https://medium.com/', + item: parsedArticles, + }; + }); } diff --git a/lib/routes/medium/graphql.ts b/lib/routes/medium/graphql.ts index 39d11328e7..9f75f9f8da 100644 --- a/lib/routes/medium/graphql.ts +++ b/lib/routes/medium/graphql.ts @@ -1,46 +1,112 @@ import got from '@/utils/got'; -async function graphqlRequest(body, cookie) { - const { data } = await got('https://medium.com/_/graphql', { - method: 'POST', - headers: { - accept: '*/*', - 'accept-language': 'en-US,en;q=0.9,zh;q=0.8,zh-CN;q=0.7', - 'apollographql-client-name': 'lite', - 'apollographql-client-version': 'main-20240329-011934-2370d8b72b', - 'cache-control': 'no-cache', - 'content-type': 'application/json', - 'medium-frontend-app': 'lite/main-20240329-011934-2370d8b72b', - 'medium-frontend-path': '/', - 'medium-frontend-route': 'homepage', - 'ot-tracer-sampled': 'true', - 'ot-tracer-spanid': '26b843316dc9494d', - 'ot-tracer-traceid': 'c84ea9154765033', - pragma: 'no-cache', - 'sec-ch-ua': '"Chromium";v="123", "Not:A-Brand";v="8"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"macOS"', - 'sec-fetch-dest': 'empty', - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-origin', - 'graphql-operation': body.operationName, - cookie, +import { type GetMediumBrowser, isMediumChallenge } from './browser'; + +export type PostPreview = { + mediumUrl: string; + title: string; + firstPublishedAt: number; + creator?: { name?: string }; + extendedPreviewContent?: { subtitle?: string }; +}; + +type Feed = { items: Array<{ post: PostPreview }> }; + +const postPreviewFields = ` + id + title + firstPublishedAt + creator { name } + extendedPreviewContent { isFullContent subtitle } +`; + +async function browserGraphqlRequest(browser: Awaited>, body, headers, retry = true): Promise<{ status: number; data: unknown; challenge: boolean }> { + const response = await browser.page.evaluate( + async ({ body, headers }) => { + const response = await fetch('https://medium.com/_/graphql', { + method: 'POST', + redirect: 'manual', + headers, + body, + }); + return { + status: response.status, + data: response.ok ? await response.json() : undefined, + challenge: response.status === 403 && /cf-chl|challenge-platform|Just a moment/i.test(await response.text()), + }; }, - body: JSON.stringify([body]), - }); + { body: JSON.stringify([body]), headers } + ); + if (retry && response.status === 403 && response.challenge) { + await browser.page.reload({ waitUntil: 'domcontentloaded', timeout: 20000 }); + if (new URL(browser.page.url()).origin !== 'https://medium.com') { + throw new Error('The Medium browser source redirected outside its expected origin.'); + } + return browserGraphqlRequest(browser, body, headers, false); + } + return response; +} + +async function graphqlRequest(body, cookie, getBrowser?: GetMediumBrowser) { + const headers = { + accept: '*/*', + 'accept-language': 'en-US,en;q=0.9,zh;q=0.8,zh-CN;q=0.7', + 'apollographql-client-name': 'lite', + 'apollographql-client-version': 'main-20240329-011934-2370d8b72b', + 'cache-control': 'no-cache', + 'content-type': 'application/json', + 'medium-frontend-app': 'lite/main-20240329-011934-2370d8b72b', + 'medium-frontend-path': '/', + 'medium-frontend-route': 'homepage', + 'ot-tracer-sampled': 'true', + 'ot-tracer-spanid': '26b843316dc9494d', + 'ot-tracer-traceid': 'c84ea9154765033', + pragma: 'no-cache', + 'sec-ch-ua': '"Chromium";v="123", "Not:A-Brand";v="8"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"macOS"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'graphql-operation': body.operationName, + cookie, + }; + let data; + try { + ({ data } = await got('https://medium.com/_/graphql', { + method: 'POST', + redirect: 'manual', + headers, + body: JSON.stringify([body]), + })); + } catch (error) { + if (!getBrowser || !isMediumChallenge(error)) { + throw error; + } + const browser = await getBrowser(); + const publicHeaders = Object.fromEntries(Object.entries(headers).filter(([name]) => name !== 'cookie')); + const response = await browserGraphqlRequest(browser, body, publicHeaders); + if (response.status !== 200) { + throw new Error(`The Medium GraphQL source returned HTTP ${response.status}. Verify the session and try again later.`, { cause: error }); + } + data = response.data; + } + if (!Array.isArray(data)) { + throw new TypeError('The Medium GraphQL source did not return its expected JSON response.'); + } return data[0].data; } -async function getFollowingFeedQuery(user, cookie, pagingLimit = 20) { - return (await graphqlRequest(newFollowingFeedQuery(pagingLimit), cookie))?.followingFeed; +async function getFollowingFeedQuery(user, cookie, pagingLimit = 20, getBrowser?: GetMediumBrowser): Promise { + return (await graphqlRequest(newFollowingFeedQuery(pagingLimit), cookie, getBrowser))?.followingFeed; } -async function getWebInlineRecommendedFeedQuery(user, cookie, pagingLimit = 20) { - return (await graphqlRequest(newWebInlineRecommendedFeedQuery(pagingLimit), cookie))?.webRecommendedFeed; +async function getWebInlineRecommendedFeedQuery(user, cookie, pagingLimit = 20, getBrowser?: GetMediumBrowser): Promise { + return (await graphqlRequest(newWebInlineRecommendedFeedQuery(pagingLimit), cookie, getBrowser))?.webRecommendedFeed; } -async function getWebInlineTopicFeedQuery(user, tagSlug, cookie, pagingLimit = 20) { - return (await graphqlRequest(newWebInlineTopicFeedQuery(tagSlug, pagingLimit), cookie))?.personalisedTagFeed; +async function getWebInlineTopicFeedQuery(user, tagSlug, cookie, pagingLimit = 20, getBrowser?: GetMediumBrowser): Promise { + return (await graphqlRequest(newWebInlineTopicFeedQuery(tagSlug, pagingLimit), cookie, getBrowser))?.personalisedTagFeed; } async function getUserCatalogMainContentQuery(user, catalogId, cookie, pagingLimit = 20) { @@ -64,6 +130,7 @@ function newFollowingFeedQuery(pagingLimit = 5) { feedId post { mediumUrl + ${postPreviewFields} __typename } __typename @@ -101,6 +168,7 @@ function newWebInlineRecommendedFeedQuery(pagingLimit = 5) { feedId post { mediumUrl + ${postPreviewFields} __typename } __typename @@ -138,6 +206,7 @@ function newWebInlineTopicFeedQuery(tagSlug, pagingLimit = 5) { feedId post { mediumUrl + ${postPreviewFields} __typename } __typename diff --git a/lib/routes/medium/parse-article.ts b/lib/routes/medium/parse-article.ts index 33eef48a8f..592e9c883c 100644 --- a/lib/routes/medium/parse-article.ts +++ b/lib/routes/medium/parse-article.ts @@ -1,16 +1,41 @@ import { load } from 'cheerio'; +import { escapeUTF8 } from 'entities/escape'; import { config } from '@/config'; import cache from '@/utils/cache'; -import got from '@/utils/got'; +import md5 from '@/utils/md5'; +import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; -async function parse(url, cookie = '') { - const { data } = await got(url, { - headers: { - cookie, - }, +import { isMediumChallenge } from './browser'; +import type { PostPreview } from './graphql'; + +function isMediumUrl(url: URL) { + return url.protocol === 'https:' && (url.hostname === 'medium.com' || url.hostname.endsWith('.medium.com')); +} + +async function fetchArticle(url: string, cookie: string, redirects = 0): Promise { + const response = await ofetch.raw(url, { + headers: cookie && isMediumUrl(new URL(url)) ? { cookie } : {}, + redirect: 'manual', + responseType: 'text', }); + if (response.status >= 300 && response.status < 400) { + const location = response.headers.get('location'); + if (!location || redirects >= 5) { + throw new Error('The Medium article source returned an invalid redirect.'); + } + const next = new URL(location, url); + if (!['https:', 'http:'].includes(next.protocol)) { + throw new Error('The Medium article source returned an unsupported redirect.'); + } + return fetchArticle(next.href, cookie, redirects + 1); + } + return response._data ?? ''; +} + +async function parse(url, cookie = '') { + const data = await fetchArticle(url, cookie); const $ = load(data); const publishedTime = $('meta[property="article:published_time"]').attr('content'); @@ -30,6 +55,10 @@ async function parse(url, cookie = '') { const subtitle = article.find('.pw-subtitle-paragraph'); const subtitleText = subtitle.text(); + if (!titleText.trim() || !article.length || (!article.text().trim() && !article.find('img[src], video[src], audio[src], iframe[src]').length)) { + throw new Error('The Medium article source did not contain a usable article.'); + } + return { title: titleText, subtitle: subtitleText, @@ -40,16 +69,32 @@ async function parse(url, cookie = '') { }; } -export default function parseArticle(ctx, url) { - return cache.tryGet(`medium:article:${url}`, async () => { - const { title, author, publishedTime, html } = await parse(url, config.medium.articleCookie); +export default function parseArticle(ctx, url, options: { cookie?: string; post?: PostPreview } = {}) { + const sessionKey = md5(JSON.stringify([options.cookie ?? '', config.medium.articleCookie])); + return cache.tryGet(`medium:article:${sessionKey}:${url}`, async () => { + try { + const { title, author, publishedTime, html } = await parse(url, config.medium.articleCookie); - return { - title, - author, - link: url, - description: html, - pubDate: publishedTime ? parseDate(publishedTime) : undefined, - }; + return { + title, + author, + link: url, + description: html, + pubDate: publishedTime ? parseDate(publishedTime) : undefined, + }; + } catch (error) { + const post = options.post; + const preview = post?.extendedPreviewContent?.subtitle; + if (!post || !isMediumChallenge(error) || post.mediumUrl !== url || !post.title?.trim() || !preview?.trim() || !(post.firstPublishedAt > 0)) { + throw error; + } + return { + title: post.title, + author: post.creator?.name, + link: url, + description: `

${escapeUTF8(preview)}

`, + pubDate: parseDate(post.firstPublishedAt), + }; + } }); } diff --git a/lib/routes/medium/tag.ts b/lib/routes/medium/tag.ts index 1df1857cd4..303e1d7aa1 100644 --- a/lib/routes/medium/tag.ts +++ b/lib/routes/medium/tag.ts @@ -4,6 +4,7 @@ import { config } from '@/config'; import ConfigNotFoundError from '@/errors/types/config-not-found.js'; import type { Route } from '@/types'; +import { withMediumBrowser } from './browser'; import { getWebInlineTopicFeedQuery } from './graphql.js'; import parseArticle from './parse-article.js'; @@ -19,7 +20,7 @@ export const route: Route = { description: '', }, ], - requirePuppeteer: false, + requirePuppeteer: true, antiCrawler: false, supportBT: false, supportPodcast: false, @@ -32,10 +33,12 @@ export const route: Route = { ::: warning Personalized recommendations require the cookie value after logging in, so only self-hosting is supported. See the configuration module on the deployment page for details. -:::`, +::: + +When Medium blocks article pages, their publicly visible previews may be returned.`, }; -async function handler(ctx) { +function handler(ctx) { const user = ctx.req.param('user'); const tag = ctx.req.param('tag'); @@ -44,21 +47,21 @@ async function handler(ctx) { throw new ConfigNotFoundError(`缺少 Medium 用户 ${user} 登录后的 Cookie 值`); } - const posts = await getWebInlineTopicFeedQuery(user, tag, cookie); - ctx.set('json', posts); + return withMediumBrowser(cookie, async (getBrowser) => { + const posts = await getWebInlineTopicFeedQuery(user, tag, cookie, undefined, getBrowser); + ctx.set('json', posts); - if (!posts) { - // login failed - throw new ConfigNotFoundError(`Medium 用户 ${user} 的 Cookie 无效或已过期`); - } + if (!posts) { + // login failed + throw new ConfigNotFoundError(`Medium 用户 ${user} 的 Cookie 无效或已过期`); + } - const urls: string[] = posts.items.map((data) => data.post.mediumUrl); + const parsedArticles = await pMap(posts.items, ({ post }) => parseArticle(ctx, post.mediumUrl, { cookie, post }), { concurrency: 3 }); - const parsedArticles = await pMap(urls, (url) => parseArticle(ctx, url), { concurrency: 3 }); - - return { - title: `${user} Medium Following Tag ${tag}`, - link: `https://medium.com/?tag=${tag}`, - item: parsedArticles, - }; + return { + title: `${user} Medium Following Tag ${tag}`, + link: `https://medium.com/?tag=${tag}`, + item: parsedArticles, + }; + }); }