var net = require('net'); var util = require('./index'); var logger = require('./logger'); var config = require('../config'); var socketMgr = require('../socket-mgr'); var version = config.version; var nodeVersion = process.version.substring(1); if (!(config.reqCacheSize > 0) || config.reqCacheSize < 600) { config.reqCacheSize = 600; } if (!(config.frameCacheSize > 0) || config.frameCacheSize < 720) { config.frameCacheSize = 600; } var CLEAR_INTERVAL = 6000; var CACHE_TIME = 1000 * 60 * 2; var MAX_CACHE_TIME = 1000 * 60 * 6; var MAX_LENGTH = config.reqCacheSize; var OVERFLOW_LENGTH = MAX_LENGTH * 3; var MAX_CACHE_SIZE = MAX_LENGTH * 2; var PRESERVE_LEN = 360; var MAX_FRAMES_LENGTH = config.frameCacheSize; var COUNT = 100; var count = 0; var ids = []; var reqData = {}; var framesCache = []; var framesMap = {}; var proxy, binded; var clearCount = 0; var MAX_COUNT = 120; function formatApiOptions(options) { var result = { count: MAX_COUNT }; if (options) { var count = options.count; if (count > 0 && count < MAX_COUNT) { result.count = count; } var startTime = options.startTime > 0 ? options.startTime + '' : null; var startId = util.isString(options.startId) ? options.startId : null; if (startId && startTime && startId < startTime) { startId = startTime; } result.startId = startId || startTime; result.latest = options.latest; } return result; } function getItemId(item, name) { return name ? item[name] : item; } function getIdIndex(arr, target, name) { if (!target) { return 0; } var left = 0; var right = arr.length - 1; while (left <= right) { if (getItemId(arr[left], name) > target) { return left; } var count = right - left; if (!count) { return -1; } if (count === 1) { return getItemId(arr[right], name) > target ? right : -1; } var mid = left + Math.floor(count / 2); var val = getItemId(arr[mid], name); if (val > target) { right = mid; } else { left = mid + 1; } } return -1; } function enable() { if (binded) { return; } binded = true; proxy.on('request', handleRequest); proxy.on('frame', handleFrame); setInterval(clearCache, CLEAR_INTERVAL); } /** * 如果超过最大缓存数,清理如下请求数据: * 1. 已经请求结束且结束时间超过10秒 * 2. 请求#1前面的未结束且未被ui读取过的请求 */ var MAX_BUF_LEN1 = 1024 * 512; var MAX_BUF_LEN2 = 1024 * 256; var MAX_BUF_LEN3 = 1024 * 128; var MAX_BUF_LEN4 = 1024 * 64; var MIN1 = 1000 * 60; var MIN2 = MIN1 * 2; var MIN3 = MIN1 * 6; var MIN4 = MIN1 * 12; function reduceFrameSize(frame, len, interval, now) { var id = frame.frameId; if (now - id.substring(0, id.indexOf('-')) < interval) { return; } frame.len = len; var bin = frame.bin; var base64 = frame.base64; if (base64) { frame.base64 = null; bin = Buffer.from(base64, 'base64'); } if (bin) { frame.bin = bin.slice(0, len); } } var clearFrames = function (frame, now) { var len = frame.len || frame.length; if (!len || len <= MAX_BUF_LEN4) { return; } if (len > MAX_BUF_LEN1) { return reduceFrameSize(frame, MAX_BUF_LEN1, MIN1, now); } if (len > MAX_BUF_LEN2) { return reduceFrameSize(frame, MAX_BUF_LEN2, MIN2, now); } if (len > MAX_BUF_LEN3) { return reduceFrameSize(frame, MAX_BUF_LEN3, MIN3, now); } reduceFrameSize(frame, MAX_BUF_LEN4, MIN4, now); }; function clearCache() { var overflow = framesCache.length - MAX_FRAMES_LENGTH; var now = Date.now(); // 1 分钟触发一次 ++clearCount; if (clearCount > 10) { clearCount = 0; } if (overflow > 0) { framesCache.splice(0, overflow + 60); framesMap = {}; framesCache.forEach(function (frame) { framesMap[frame.reqId] = frame; !clearCount && clearFrames(frame, now); }); } else if (!clearCount) { framesCache.forEach(function (frame) { clearFrames(frame, now); }); } var len = ids.length; if (len <= MAX_LENGTH) { return; } var _ids = []; var preserveLen = len; overflow = -1; if (len >= OVERFLOW_LENGTH) { overflow = len - MAX_CACHE_SIZE; preserveLen = len - PRESERVE_LEN; } var isTimeout = function (curData, i) { if (i < overflow) { return true; } return ( curData.endTime && now - curData.endTime > (i >= preserveLen ? MAX_CACHE_TIME : CACHE_TIME) ); }; for (var i = 0; i < len; i++) { var id = ids[i]; var curData = reqData[id]; if (isTimeout(curData, i)) { curData.abort && curData.abort(true); delete reqData[id]; } else { if (curData.abort && now - curData.startTime > MAX_CACHE_TIME) { curData.abort(true); } _ids.push(id); } } ids = _ids; } // 不存在startTime相等的id function getIndex(startTime, start, end) { var midIndex = Math.floor((start + end) / 2); if (midIndex == start) { return end; } if (ids[midIndex] < startTime) { return getIndex(startTime, midIndex, end); } return getIndex(startTime, start, midIndex); } function toBase64String(data) { if (Buffer.isBuffer(data.body)) { data.base64 = data.body.toString('base64'); data.body = ''; } } function getIds(startTime, count, lastRowId) { startTime = startTime || lastRowId; if (!startTime) { return ids.slice(-count); } var index = 0; if (startTime !== '0') { index = ids.indexOf(startTime) + 1; if (!index && startTime.length > 6) { var startId = ids[0]; if (startId && startId < startTime) { var end = ids.length - 1; if (ids[end] < startTime) { return []; } index = getIndex(startTime, 0, end); } } } return ids.slice(index, index + count); } function removeBase64(data) { var result = {}; Object.keys(data).forEach(function(key) { if (key !== 'headers' && key !== 'base64' && key !== 'rawHeaderNames') { result[key] = data[key]; } }); return result; } function getBase64Len(base64) { if (!base64) { return 0; } var len = base64.length; if (base64[len - 1] === '=') { len -= 2; if (base64[len] === '=') { --len; } } return len; } function setBody(item, len) { if (!(len > 0)) { return item; } var base64 = item.base64; var curLen = getBase64Len(base64); item = removeBase64(item); if (len < curLen) { item.preLen = len; item.base64 = base64.substring(len); } return item; } function getList(ids, status) { if (!Array.isArray(ids)) { return []; } return ids.map(function (id, i) { var data = reqData[id]; var opts = status && status[i]; if (data) { toBase64String(data.req); toBase64String(data.res); } if (!data || typeof opts !== 'string') { return data; } opts = opts.split('-'); var result = {}; Object.keys(data).forEach(function(key) { if (key === 'url' || key === 'rulesHeaders' || (key === 'rules' && opts[1] != '0')) { return; } var item = data[key]; if (key === 'req') { if (!opts[0]) { item = removeBase64(item); } else { item = setBody(item, +opts[0]); } } else if (key === 'res') { item = setBody(item, +opts[1]); } result[key] = item; }); return result; }); } function handleRequest(req, data) { var id = (data.id = data.id || data.startTime + '-' + ++count); var removeAbort = function () { if (data.abort) { delete data.abort; } }; util.onReadEnd(req, removeAbort); req.on('error', removeAbort); req.on('abort', removeAbort); data.version = version; data.nodeVersion = nodeVersion; reqData[id] = data; ids.indexOf(id) === -1 && ids.push(id); } function decodeData(frame) { if (frame.base64 == null) { frame.base64 = frame.bin ? frame.bin.toString('base64') : ''; frame.bin = ''; } return frame; } function handleFrame(data) { framesCache.push(data); framesMap[data.reqId] = data; } function getFrames(curReqId, lastFrameId) { if (!curReqId) { return; } var result = []; var lastFrame = framesMap[curReqId]; if (lastFrame && (!lastFrameId || lastFrame.frameId > lastFrameId)) { var count = 16; for (var i = 0, len = framesCache.length; i < len; i++) { var frame = framesCache[i]; if ( frame.reqId === curReqId && (!lastFrameId || frame.frameId > lastFrameId) ) { result.push(decodeData(frame)); if (--count <= 0) { return result; } } } } return result; } function getLastFrame(curReqId) { var frame = framesMap[curReqId]; if (frame && frame.reqId === curReqId) { return decodeData(frame); } } function getDomain(item) { var url = item && item.url; if (!url) { return ''; } var index = url.indexOf('://'); if (index === -1) { return ''; } index += 3; url = url.substring(index); index = url.indexOf('/'); if (index !== -1) { url = url.substring(0, index); } index = url.indexOf(':'); if (index !== -1) { url = url.substring(0, index); } return url; } module.exports = function init(_proxy) { proxy = _proxy; enable(); /** * options: { * startTime: timestamp || timestamp + '-' + count * count: 获取新数据的数量 * ids: 请未结束的id列表 * } * * @param options */ function formatFilter(filter, clientIp, clientId) { if (!filter.url && !filter.name && !filter.value && !filter.ip) { return; } var url = util.trimStr(filter.url).toLowerCase(); var ip = util.trimStr(filter.ip); var list = []; var cid; var result; if (ip === 'self') { ip = clientIp; cid = clientId; } else if (ip === 'clientIp') { ip = clientIp; } if (ip === 'clientId') { if (clientId) { cid = clientId; } else { result = { clientIp: clientIp }; } ip = null; } else if (ip && !net.isIP(ip)) { ip.split(',').forEach(function (item) { item = item.trim(); if (item === 'clientId') { cid = clientId; } else { if (item === 'self') { cid = clientId; item = clientIp; } else if (item === 'clientIp') { item = clientIp; } if (list.indexOf(item) === -1) { list.push(item); } } }); ip = null; } if (url) { result = result || {}; result.url = url; } var headers; for (var i = 0; i < 6; i++) { var key = 'name' + (i || ''); var name = util.trimStr(filter[key]).toLowerCase(); if (name) { result = result || {}; var value = util.trimStr(filter['value' + (i || '')]).toLowerCase(); if (i) { headers = headers || []; result.headers = headers; headers.push({ name: name, value: value }); } else { result.name = name; result.value = value; } } } if (ip) { result = result || {}; result.ip = ip; } if (cid) { result = result || {}; result.clientId = cid; } if (list.length) { result = result || {}; result.ipList = result.idList = list.slice(0, 16); } if (result && (result.name || headers) && filter.mtype == 1) { result.exact = 1; } return result; } function checkClientIp(item, filter) { var clientIp = item.req.ip; if (filter.clientIp) { return clientIp === filter.clientIp; } if (filter.ip && clientIp === filter.ip) { return true; } // 有 clientId 过滤条件时,必须匹配 clientId var clientId = getClientId(item); var ipList = filter.ipList; if (filter.clientId) { if (clientId === filter.clientId) { return true; } if (!ipList) { return false; } } else if (!ipList) { return true; } var len = ipList.length; if (len < 3) { if (ipList[0] === clientIp || ipList[0] === clientId) { return true; } if (ipList[1] && (ipList[1] === clientIp || ipList[1] === clientId)) { return true; } } else { for (var i = 0; i < len; i++) { var ip = ipList[i]; if (ip === clientIp || ip === clientId) { return true; } } } return false; } function checkHeader(text, keyword, exact) { if (!keyword) { return text != null; } if (!text || typeof text !== 'string') { if (!Array.isArray(text)) { return false; } text = text.join('\n'); } text = text.toLowerCase(); if (exact) { return ( text === keyword || text === util.encodeURIComponent(keyword).toLowerCase() ); } return ( text.indexOf(keyword) !== -1 || text.indexOf(util.encodeURIComponent(keyword).toLowerCase()) !== -1 ); } function getClientId(item) { return ( item._clientId || item.req.headers[config.CLIENT_ID_HEADER] || item.clientId ); } function checkItem(item, filter) { if (!item || !checkClientIp(item, filter)) { return false; } var h = item.req.headers; if (filter.filterKey && h[filter.filterKey] != filter.filterValue) { return false; } if (filter.filterClientId && getClientId(item) != filter.filterClientId) { return false; } if ( filter.name && !checkHeader(h[filter.name], filter.value, filter.exact) ) { return false; } if ( filter.url && !checkHeader((item.isHttps ? 'tunnel://' : '') + item.url, filter.url) ) { return false; } var headers = filter.headers; if (headers) { for (var i = 0, len = headers.length; i < len; i++) { var header = headers[i]; if (!checkHeader(h[header.name], header.value, filter.exact)) { return false; } } } return true; } proxy.getLastDataId = function () { return ids[ids.length - 1]; }; proxy.getCookiesByDomain = function(domain) { var result = []; if (!domain || typeof domain !== 'string') { return result; } for (var i = ids.length - 1; i >= 0; i--) { var item = reqData[ids[i]]; if (getDomain(item) === domain) { var cookie = item.req.headers.cookie; if (cookie && result.indexOf(cookie) === -1) { result.push(cookie); if (result.length >= 30) { return result; } } } } return result; }; proxy.getItem = function (id) { var item = reqData[id]; if (item) { toBase64String(item.req); toBase64String(item.res); } return item; }; proxy.getTestItem = function(id) { var list = Object.values(reqData); for (var i = 0, len = list.length; i < len; i++) { var item = list[i]; if (item.testId === id) { return item; } } }; proxy.abortRequest = function (id) { var item = id && reqData[id]; item && item.abort && item.abort(); }; proxy.getFrames = function (options) { return getFrames(options.curReqId, options.lastFrameId); }; proxy.getFramesForApi = function (options) { var reqId = options.reqId; var isServer = options.from === 'server'; var isClient = options.from === 'client'; var isAll = !isServer && !isClient; var params = formatApiOptions(options); var index = getIdIndex(framesCache, params.startId, 'frameId'); var result = []; if (index === -1) { return result; } var count = params.count; var len = framesCache.length; var addFrame = function(i) { var frame = framesCache[i]; if ((isAll || isClient === !!frame.isClient) && frame.reqId === reqId) { result.push(decodeData(frame)); if (--count <= 0) { return true; } } }; if (params.latest) { for (var i = len - 1; i >= index; i--) { if (addFrame(i)) { break; } } result.reverse(); } else { for (; index < len; index++) { if (addFrame(index)) { break; } } } return result; }; function checkUrl(session, subUrl) { if (!subUrl) { return true; } var url = session.isHttps ? 'tunnel://' + session.url : session.url.toLowerCase(); return url.indexOf(subUrl) !== -1; } var WS_RE = /^wss?:\/\//; var FONT_RE = /font\//i; var MEDIA_RE = /audio\/|video\/|application\/vnd.apple.mpegurl|application\/dash+xml/i; var WASM_RE = /application\/wasm/i; var TYPES = ['IMG', 'HTML', 'CSS', 'JS', 'JSON']; var ALIAS_TYPES = { WEBSOCKET: 'WS', WSS: 'WS', IMAGE: 'IMG', JAVASCRIPT: 'JS', STYLE: 'CSS', MATCHED: 'RULES', EDIT: 'COMPOSER' }; var RAW_TYPES = ['FONT', 'MEDIA', 'WASM']; function hasError(item) { var statusCode = item.res && item.res.statusCode; return item.reqError || item.resError || (item.customData && item.customData.error) || (statusCode && (!/^\d+$/.test(statusCode) || statusCode >= 400)); } function checkType(session, filterType) { if (!filterType) { return true; } filterType = ALIAS_TYPES[filterType] || filterType; if (filterType === 'CAPTUREERROR') { return session.captureError; } if (filterType === 'WS') { return WS_RE.test(session.url); } if (filterType === 'TUNNEL') { return session.isHttps; } if (filterType === 'RULES') { return session.rules && Object.keys(session.rules).length > 0; } var rawType = util.getRawType(session.res); var type; for (var i = 0, len = RAW_TYPES.length; i < len; i++) { type = RAW_TYPES[i]; if (filterType === type) { var p = type === 'FONT' ? FONT_RE : type === 'MEDIA' ? MEDIA_RE : WASM_RE; return p.test(rawType); } } if (filterType === 'SSE') { return rawType === 'text/event-stream'; } var curType = util.getContentType(rawType); for (i = 0, len = TYPES.length; i < len; i++) { type = TYPES[i]; if (filterType === type) { return curType === (i ? type : 'IMG'); } } if (filterType === 'ERROR') { return hasError(session); } if (filterType === 'MOCK') { var rules = session.rules || ''; return rules.rule && rules.rule.isLoc; } if (filterType === 'IMPORT') { return session.importedData; } if (filterType === 'COMPOSER') { return session.fc; } } function formatHeaderFilter(filter) { if (!filter) { return; } var name = filter.name || filter.key; if (typeof name !== 'string') { return; } return { name: name.trim(), subValue: filter.subValue == null ? null : String(filter.subValue).toLowerCase() }; } function filterHeader(headers, filter) { if (!filter) { return true; } if (!headers) { return false; } var value = headers[filter.name]; if (value == null) { return false; } var subValue = filter.subValue; var noSubValue = !subValue; if (noSubValue || !value) { return noSubValue; } value = Array.isArray(value) ? value.join('\n') : String(value); return value.toLowerCase().indexOf(subValue) !== -1; } function checkMethod(session, method) { if (!method) { return true; } return method.indexOf(session.req.method) !== -1; } function checkStatus(session, status) { return !status || status.indexOf(String(session.res.statusCode).toUpperCase()) !== -1; } proxy.getSessionsForApi = function (options) { var params = formatApiOptions(options); var index = getIdIndex(ids, params.startId); var result = []; if (index === -1) { return result; } var count = params.count; var len = ids.length; var subUrl = util.getString(options.subUrl).toLowerCase(); var type = util.getString(options.type).toUpperCase(); var method = options.method; var status = options.statusCode || options.status; var reqHeader = formatHeaderFilter(options.reqHeader); var resHeader = formatHeaderFilter(options.resHeader); if (Array.isArray(method)) { method = method.join(); } else { method = util.getString(method); } if (Array.isArray(status)) { status = status.join(); } else if (status) { status = String(status); } method = method && method.toUpperCase().split(','); status = status && status.toUpperCase().split(','); var addSession = function(i) { var session = reqData[ids[i]]; if (checkUrl(session, subUrl) && checkMethod(session, method) && checkStatus(session, status) && filterHeader(session.req.headers, reqHeader) && filterHeader(session.res.headers, resHeader) && checkType(session, type)) { toBase64String(session.req); toBase64String(session.res); result.push(session); if (--count <= 0) { return true; } } }; if (params.latest) { for (var i = len - 1; i >= index; i--) { if (addSession(i)) { break; } } result.reverse(); } else { for (; index < len; index++) { if (addSession(index)) { break; } } } return result; }; function getNetworkTime(id) { var item = id && reqData[id]; return item && { startTime: item.startTime, dnsTime: item.dnsTime, requestTime: item.requestTime, responseTime: item.responseTime, endTime: item.endTime }; } proxy.getData = function ( options, clientIp, key, value, filterClientId, clientId ) { options = options || {}; var filter = formatFilter(options, clientIp, clientId); var data = {}; var count = options.count; var startTime = options.startTime; var clearNetwork = !(options.dumpCount > 0) && (count == 0 || startTime == -2); if (!clearNetwork) { count = count > 0 && count < COUNT ? +count : COUNT; if (options.dumpCount > 0) { var len = ids.length; startTime = ids[len > options.dumpCount ? len - options.dumpCount : 0]; } } if (key && value) { filter = filter || {}; filter.filterKey = key; filter.filterValue = value; } if (filterClientId) { filter = filter || {}; filter.filterClientId = filterClientId; } var newIds = clearNetwork || startTime == -1 ? [] : getIds(startTime, count, options.lastRowId); var setData = function (item) { if (item) { if (config.secureFilter) { try { item = config.secureFilter(item, clientIp, filter) || item; } catch (e) { if (config.debugMode) { /* eslint-disable no-console */ console.log(e); } logger.error(e); } } data[item.id] = item; } }; if (newIds.length > 0) { if (filter) { var id = newIds[0]; var index = ids.indexOf(newIds[0]); newIds = []; while (id && count > 0) { var item = reqData[id]; if (checkItem(item, filter)) { toBase64String(item.req); toBase64String(item.res); setData(item); newIds.push(id); --count; } id = ids[++index]; } } else if (!filter) { getList(newIds).forEach(setData); } } getList(options.ids, options.status).forEach(setData); var endId = ids[ids.length - 1]; var lastFrameId, frames; if (options.lastFrameId == -3) { var lastFrame = getLastFrame(options.curReqId); if (lastFrame) { if (lastFrame.closed || lastFrame.err) { frames = [lastFrame]; } else { lastFrameId = lastFrame.frameId; } } } else { frames = getFrames(options.curReqId, options.lastFrameId); } var lastNewId = newIds[newIds.length - 1]; var lastNewFrameId = frames && frames[frames.length - 1]; var hasNew = (lastNewId && endId !== lastNewId) || (lastNewFrameId && lastNewFrameId !== lastFrameId); var tunnelIds = options.tunnelIds; var tunnelIps = {}; if (Array.isArray(tunnelIds) && tunnelIds.length > 0) { tunnelIds.forEach(function (id) { tunnelIps[id] = proxy.getTunnelIp(id); }); } return { ids: options.ids || [], composerTime: getNetworkTime(options.composerReqId), tunnelIps: tunnelIps, newIds: newIds, data: data, hasNew: hasNew, lastId: clearNetwork ? endId : lastNewId, endId: endId, frames: frames, lastFrameId: lastFrameId, socketStatus: socketMgr.getStatus(options.curReqId) }; }; };