mirror of
https://github.com/avwo/whistle.git
synced 2026-10-18 14:41:12 +08:00
avwo
713 lines
17 KiB
JavaScript
713 lines
17 KiB
JavaScript
var net = require('net');
|
|
var util = require('./index');
|
|
var logger = require('./logger');
|
|
var config = require('../config');
|
|
var socketMgr = require('../socket-mgr');
|
|
|
|
var version = config.version;
|
|
var nodeVersion = process.version.substring(1);
|
|
|
|
if (!(config.reqCacheSize > 0) || config.reqCacheSize < 600) {
|
|
config.reqCacheSize = 600;
|
|
}
|
|
|
|
if (!(config.frameCacheSize > 0) || config.frameCacheSize < 720) {
|
|
config.frameCacheSize = 600;
|
|
}
|
|
|
|
var CLEAR_INTERVAL = 6000;
|
|
var CACHE_TIME = 1000 * 60 * 2;
|
|
var MAX_CACHE_TIME = 1000 * 60 * 6;
|
|
var MAX_LENGTH = config.reqCacheSize;
|
|
var OVERFLOW_LENGTH = MAX_LENGTH * 3;
|
|
var MAX_CACHE_SIZE = MAX_LENGTH * 2;
|
|
var PRESERVE_LEN = 360;
|
|
var MAX_FRAMES_LENGTH = config.frameCacheSize;
|
|
var COUNT = 100;
|
|
var count = 0;
|
|
var ids = [];
|
|
var reqData = {};
|
|
var framesCache = [];
|
|
var framesMap = {};
|
|
var proxy, binded;
|
|
var clearCount = 0;
|
|
|
|
function enable() {
|
|
if (binded) {
|
|
return;
|
|
}
|
|
binded = true;
|
|
proxy.on('request', handleRequest);
|
|
proxy.on('frame', handleFrame);
|
|
setInterval(clearCache, CLEAR_INTERVAL);
|
|
}
|
|
|
|
/**
|
|
* 如果超过最大缓存数，清理如下请求数据：
|
|
* 1. 已经请求结束且结束时间超过10秒
|
|
* 2. 请求#1前面的未结束且未被ui读取过的请求
|
|
*/
|
|
|
|
var MAX_BUF_LEN1 = 1024 * 512;
|
|
var MAX_BUF_LEN2 = 1024 * 256;
|
|
var MAX_BUF_LEN3 = 1024 * 128;
|
|
var MAX_BUF_LEN4 = 1024 * 64;
|
|
var MIN1 = 1000 * 60;
|
|
var MIN2 = MIN1 * 2;
|
|
var MIN3 = MIN1 * 6;
|
|
var MIN4 = MIN1 * 12;
|
|
|
|
function reduceFrameSize(frame, len, interval, now) {
|
|
var id = frame.frameId;
|
|
if (now - id.substring(0, id.indexOf('-')) < interval) {
|
|
return;
|
|
}
|
|
frame.len = len;
|
|
var bin = frame.bin;
|
|
var base64 = frame.base64;
|
|
if (base64) {
|
|
frame.base64 = null;
|
|
bin = Buffer.from(base64, 'base64');
|
|
}
|
|
if (bin) {
|
|
frame.bin = bin.slice(0, len);
|
|
}
|
|
}
|
|
|
|
var clearFrames = function (frame, now) {
|
|
var len = frame.len || frame.length;
|
|
if (!len || len <= MAX_BUF_LEN4) {
|
|
return;
|
|
}
|
|
if (len > MAX_BUF_LEN1) {
|
|
return reduceFrameSize(frame, MAX_BUF_LEN1, MIN1, now);
|
|
}
|
|
if (len > MAX_BUF_LEN2) {
|
|
return reduceFrameSize(frame, MAX_BUF_LEN2, MIN2, now);
|
|
}
|
|
|
|
if (len > MAX_BUF_LEN3) {
|
|
return reduceFrameSize(frame, MAX_BUF_LEN3, MIN3, now);
|
|
}
|
|
reduceFrameSize(frame, MAX_BUF_LEN4, MIN4, now);
|
|
};
|
|
|
|
function clearCache() {
|
|
var overflow = framesCache.length - MAX_FRAMES_LENGTH;
|
|
var now = Date.now();
|
|
// 1 分钟触发一次
|
|
++clearCount;
|
|
if (clearCount > 10) {
|
|
clearCount = 0;
|
|
}
|
|
|
|
if (overflow > 0) {
|
|
framesCache.splice(0, overflow + 60);
|
|
framesMap = {};
|
|
framesCache.forEach(function (frame) {
|
|
framesMap[frame.reqId] = frame;
|
|
!clearCount && clearFrames(frame, now);
|
|
});
|
|
} else if (!clearCount) {
|
|
framesCache.forEach(function (frame) {
|
|
clearFrames(frame, now);
|
|
});
|
|
}
|
|
var len = ids.length;
|
|
if (len <= MAX_LENGTH) {
|
|
return;
|
|
}
|
|
|
|
var _ids = [];
|
|
var preserveLen = len;
|
|
overflow = -1;
|
|
if (len >= OVERFLOW_LENGTH) {
|
|
overflow = len - MAX_CACHE_SIZE;
|
|
preserveLen = len - PRESERVE_LEN;
|
|
}
|
|
var isTimeout = function (curData, i) {
|
|
if (i < overflow) {
|
|
return true;
|
|
}
|
|
return (
|
|
curData.endTime &&
|
|
now - curData.endTime > (i >= preserveLen ? MAX_CACHE_TIME : CACHE_TIME)
|
|
);
|
|
};
|
|
for (var i = 0; i < len; i++) {
|
|
var id = ids[i];
|
|
var curData = reqData[id];
|
|
if (isTimeout(curData, i)) {
|
|
curData.abort && curData.abort(true);
|
|
delete reqData[id];
|
|
} else {
|
|
if (curData.abort && now - curData.startTime > MAX_CACHE_TIME) {
|
|
curData.abort(true);
|
|
}
|
|
_ids.push(id);
|
|
}
|
|
}
|
|
ids = _ids;
|
|
}
|
|
// 不存在startTime相等的id
|
|
function getIndex(startTime, start, end) {
|
|
var midIndex = Math.floor((start + end) / 2);
|
|
if (midIndex == start) {
|
|
return end;
|
|
}
|
|
if (ids[midIndex] < startTime) {
|
|
return getIndex(startTime, midIndex, end);
|
|
}
|
|
return getIndex(startTime, start, midIndex);
|
|
}
|
|
|
|
function toBase64String(data) {
|
|
if (Buffer.isBuffer(data.body)) {
|
|
data.base64 = data.body.toString('base64');
|
|
data.body = '';
|
|
}
|
|
}
|
|
|
|
function getIds(startTime, count, lastRowId) {
|
|
startTime = startTime || lastRowId;
|
|
if (!startTime) {
|
|
return ids.slice(-count);
|
|
}
|
|
var index = 0;
|
|
if (startTime !== '0') {
|
|
index = ids.indexOf(startTime) + 1;
|
|
if (!index && startTime.length > 6) {
|
|
var startId = ids[0];
|
|
if (startId && startId < startTime) {
|
|
var end = ids.length - 1;
|
|
if (ids[end] < startTime) {
|
|
return [];
|
|
}
|
|
index = getIndex(startTime, 0, end);
|
|
}
|
|
}
|
|
}
|
|
return ids.slice(index, index + count);
|
|
}
|
|
|
|
function removeBase64(data) {
|
|
var result = {};
|
|
Object.keys(data).forEach(function(key) {
|
|
if (key !== 'headers' && key !== 'base64' && key !== 'rawHeaderNames') {
|
|
result[key] = data[key];
|
|
}
|
|
});
|
|
return result;
|
|
}
|
|
|
|
function getBase64Len(base64) {
|
|
if (!base64) {
|
|
return 0;
|
|
}
|
|
var len = base64.length;
|
|
if (base64[len - 1] === '=') {
|
|
len -= 2;
|
|
if (base64[len] === '=') {
|
|
--len;
|
|
}
|
|
}
|
|
return len;
|
|
}
|
|
|
|
function setBody(item, len) {
|
|
if (!(len > 0)) {
|
|
return item;
|
|
}
|
|
var base64 = item.base64;
|
|
var curLen = getBase64Len(base64);
|
|
item = removeBase64(item);
|
|
if (len < curLen) {
|
|
item.preLen = len;
|
|
item.base64 = base64.substring(len);
|
|
}
|
|
return item;
|
|
}
|
|
|
|
function getList(ids, status) {
|
|
if (!Array.isArray(ids)) {
|
|
return [];
|
|
}
|
|
return ids.map(function (id, i) {
|
|
var data = reqData[id];
|
|
var opts = status && status[i];
|
|
if (data) {
|
|
toBase64String(data.req);
|
|
toBase64String(data.res);
|
|
}
|
|
if (!data || typeof opts !== 'string') {
|
|
return data;
|
|
}
|
|
opts = opts.split('-');
|
|
var result = {};
|
|
Object.keys(data).forEach(function(key) {
|
|
if (key === 'url' || key === 'rulesHeaders' || (key === 'rules' && opts[1] != '0')) {
|
|
return;
|
|
}
|
|
var item = data[key];
|
|
if (key === 'req') {
|
|
if (!opts[0]) {
|
|
item = removeBase64(item);
|
|
} else {
|
|
item = setBody(item, +opts[0]);
|
|
}
|
|
} else if (key === 'res') {
|
|
item = setBody(item, +opts[1]);
|
|
}
|
|
result[key] = item;
|
|
});
|
|
return result;
|
|
});
|
|
}
|
|
|
|
function handleRequest(req, data) {
|
|
var id = (data.id = data.id || data.startTime + '-' + ++count);
|
|
var removeAbort = function () {
|
|
if (data.abort) {
|
|
delete data.abort;
|
|
}
|
|
};
|
|
req.on('end', removeAbort);
|
|
req.on('error', removeAbort);
|
|
req.on('abort', removeAbort);
|
|
data.version = version;
|
|
data.nodeVersion = nodeVersion;
|
|
reqData[id] = data;
|
|
ids.indexOf(id) === -1 && ids.push(id);
|
|
}
|
|
|
|
function decodeData(frame) {
|
|
if (frame.base64 == null) {
|
|
frame.base64 = frame.bin ? frame.bin.toString('base64') : '';
|
|
frame.bin = '';
|
|
}
|
|
return frame;
|
|
}
|
|
function handleFrame(data) {
|
|
framesCache.push(data);
|
|
framesMap[data.reqId] = data;
|
|
}
|
|
|
|
function getFrames(curReqId, lastFrameId) {
|
|
if (!curReqId) {
|
|
return;
|
|
}
|
|
var result = [];
|
|
var lastFrame = framesMap[curReqId];
|
|
if (lastFrame && (!lastFrameId || lastFrame.frameId > lastFrameId)) {
|
|
var count = 16;
|
|
for (var i = 0, len = framesCache.length; i < len; i++) {
|
|
var frame = framesCache[i];
|
|
if (
|
|
frame.reqId === curReqId &&
|
|
(!lastFrameId || frame.frameId > lastFrameId)
|
|
) {
|
|
result.push(decodeData(frame));
|
|
if (--count <= 0) {
|
|
return result;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function getLastFrame(curReqId) {
|
|
var frame = framesMap[curReqId];
|
|
if (frame && frame.reqId === curReqId) {
|
|
return decodeData(frame);
|
|
}
|
|
}
|
|
|
|
function getDomain(item) {
|
|
var url = item && item.url;
|
|
if (!url) {
|
|
return '';
|
|
}
|
|
var index = url.indexOf('://');
|
|
if (index === -1) {
|
|
return '';
|
|
}
|
|
index += 3;
|
|
url = url.substring(index);
|
|
index = url.indexOf('/');
|
|
if (index !== -1) {
|
|
url = url.substring(0, index);
|
|
}
|
|
index = url.indexOf(':');
|
|
if (index !== -1) {
|
|
url = url.substring(0, index);
|
|
}
|
|
return url;
|
|
}
|
|
|
|
|
|
module.exports = function init(_proxy) {
|
|
proxy = _proxy;
|
|
enable();
|
|
/**
|
|
* options: {
|
|
* startTime: timestamp || timestamp + '-' + count
|
|
* count: 获取新数据的数量
|
|
* ids: 请未结束的id列表
|
|
* }
|
|
*
|
|
* @param options
|
|
*/
|
|
function formatFilter(filter, clientIp, clientId) {
|
|
if (!filter.url && !filter.name && !filter.value && !filter.ip) {
|
|
return;
|
|
}
|
|
var url = util.trimStr(filter.url).toLowerCase();
|
|
var ip = util.trimStr(filter.ip);
|
|
var list = [];
|
|
var cid;
|
|
var result;
|
|
if (ip === 'self') {
|
|
ip = clientIp;
|
|
cid = clientId;
|
|
} else if (ip === 'clientIp') {
|
|
ip = clientIp;
|
|
}
|
|
if (ip === 'clientId') {
|
|
if (clientId) {
|
|
cid = clientId;
|
|
} else {
|
|
result = { clientIp: clientIp };
|
|
}
|
|
ip = null;
|
|
} else if (ip && !net.isIP(ip)) {
|
|
ip.split(',').forEach(function (item) {
|
|
item = item.trim();
|
|
if (item === 'clientId') {
|
|
cid = clientId;
|
|
} else {
|
|
if (item === 'self') {
|
|
cid = clientId;
|
|
item = clientIp;
|
|
} else if (item === 'clientIp') {
|
|
item = clientIp;
|
|
}
|
|
if (list.indexOf(item) === -1) {
|
|
list.push(item);
|
|
}
|
|
}
|
|
});
|
|
ip = null;
|
|
}
|
|
if (url) {
|
|
result = result || {};
|
|
result.url = url;
|
|
}
|
|
var headers;
|
|
for (var i = 0; i < 6; i++) {
|
|
var key = 'name' + (i || '');
|
|
var name = util.trimStr(filter[key]).toLowerCase();
|
|
if (name) {
|
|
result = result || {};
|
|
var value = util.trimStr(filter['value' + (i || '')]).toLowerCase();
|
|
if (i) {
|
|
headers = headers || [];
|
|
result.headers = headers;
|
|
headers.push({
|
|
name: name,
|
|
value: value
|
|
});
|
|
} else {
|
|
result.name = name;
|
|
result.value = value;
|
|
}
|
|
}
|
|
}
|
|
if (ip) {
|
|
result = result || {};
|
|
result.ip = ip;
|
|
}
|
|
if (cid) {
|
|
result = result || {};
|
|
result.clientId = cid;
|
|
}
|
|
if (list.length) {
|
|
result = result || {};
|
|
result.ipList = result.idList = list.slice(0, 16);
|
|
}
|
|
if (result && (result.name || headers) && filter.mtype == 1) {
|
|
result.exact = 1;
|
|
}
|
|
return result;
|
|
}
|
|
function checkClientIp(item, filter) {
|
|
var clientIp = item.req.ip;
|
|
if (filter.clientIp) {
|
|
return clientIp === filter.clientIp;
|
|
}
|
|
if (filter.ip && clientIp === filter.ip) {
|
|
return true;
|
|
}
|
|
// 有 clientId 过滤条件时，必须匹配 clientId
|
|
var clientId = getClientId(item);
|
|
var ipList = filter.ipList;
|
|
if (filter.clientId) {
|
|
if (clientId === filter.clientId) {
|
|
return true;
|
|
}
|
|
if (!ipList) {
|
|
return false;
|
|
}
|
|
} else if (!ipList) {
|
|
return true;
|
|
}
|
|
var len = ipList.length;
|
|
if (len < 3) {
|
|
if (ipList[0] === clientIp || ipList[0] === clientId) {
|
|
return true;
|
|
}
|
|
if (ipList[1] && (ipList[1] === clientIp || ipList[1] === clientId)) {
|
|
return true;
|
|
}
|
|
} else {
|
|
for (var i = 0; i < len; i++) {
|
|
var ip = ipList[i];
|
|
if (ip === clientIp || ip === clientId) {
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
function checkHeader(text, keyword, exact) {
|
|
if (!keyword) {
|
|
return text != null;
|
|
}
|
|
if (!text || typeof text !== 'string') {
|
|
if (!Array.isArray(text)) {
|
|
return false;
|
|
}
|
|
text = text.join('\n');
|
|
}
|
|
text = text.toLowerCase();
|
|
if (exact) {
|
|
return (
|
|
text === keyword ||
|
|
text === util.encodeURIComponent(keyword).toLowerCase()
|
|
);
|
|
}
|
|
return (
|
|
text.indexOf(keyword) !== -1 ||
|
|
text.indexOf(util.encodeURIComponent(keyword).toLowerCase()) !== -1
|
|
);
|
|
}
|
|
|
|
function getClientId(item) {
|
|
return (
|
|
item._clientId ||
|
|
item.req.headers[config.CLIENT_ID_HEADER] ||
|
|
item.clientId
|
|
);
|
|
}
|
|
|
|
function checkItem(item, filter) {
|
|
if (!item || !checkClientIp(item, filter)) {
|
|
return false;
|
|
}
|
|
var h = item.req.headers;
|
|
if (filter.filterKey && h[filter.filterKey] != filter.filterValue) {
|
|
return false;
|
|
}
|
|
if (filter.filterClientId && getClientId(item) != filter.filterClientId) {
|
|
return false;
|
|
}
|
|
if (
|
|
filter.name &&
|
|
!checkHeader(h[filter.name], filter.value, filter.exact)
|
|
) {
|
|
return false;
|
|
}
|
|
if (
|
|
filter.url &&
|
|
!checkHeader((item.isHttps ? 'tunnel://' : '') + item.url, filter.url)
|
|
) {
|
|
return false;
|
|
}
|
|
var headers = filter.headers;
|
|
if (headers) {
|
|
for (var i = 0, len = headers.length; i < len; i++) {
|
|
var header = headers[i];
|
|
if (!checkHeader(h[header.name], header.value, filter.exact)) {
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
proxy.getLastDataId = function () {
|
|
return ids[ids.length - 1];
|
|
};
|
|
|
|
proxy.getCookiesByDomain = function(domain) {
|
|
var result = [];
|
|
if (!domain || typeof domain !== 'string') {
|
|
return result;
|
|
}
|
|
for (var i = ids.length - 1; i >= 0; i--) {
|
|
var item = reqData[ids[i]];
|
|
if (getDomain(item) === domain) {
|
|
var cookie = item.req.headers.cookie;
|
|
if (cookie && result.indexOf(cookie) === -1) {
|
|
result.push(cookie);
|
|
if (result.length >= 30) {
|
|
return result;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return result;
|
|
};
|
|
|
|
proxy.getItem = function (id) {
|
|
var item = reqData[id];
|
|
if (item) {
|
|
toBase64String(item.req);
|
|
toBase64String(item.res);
|
|
}
|
|
return item;
|
|
};
|
|
proxy.abortRequest = function (id) {
|
|
var item = id && reqData[id];
|
|
item && item.abort && item.abort();
|
|
};
|
|
proxy.getFrames = function (options) {
|
|
return getFrames(options.curReqId, options.lastFrameId);
|
|
};
|
|
|
|
function getNetworkTime(id) {
|
|
var item = id && reqData[id];
|
|
return item && {
|
|
startTime: item.startTime,
|
|
dnsTime: item.dnsTime,
|
|
requestTime: item.requestTime,
|
|
responseTime: item.responseTime,
|
|
endTime: item.endTime
|
|
};
|
|
}
|
|
|
|
proxy.getData = function (
|
|
options,
|
|
clientIp,
|
|
key,
|
|
value,
|
|
filterClientId,
|
|
clientId
|
|
) {
|
|
options = options || {};
|
|
var filter = formatFilter(options, clientIp, clientId);
|
|
var data = {};
|
|
var count = options.count;
|
|
var startTime = options.startTime;
|
|
var clearNetwork =
|
|
!(options.dumpCount > 0) && (count == 0 || startTime == -2);
|
|
if (!clearNetwork) {
|
|
count = count > 0 && count < COUNT ? +count : COUNT;
|
|
if (options.dumpCount > 0) {
|
|
var len = ids.length;
|
|
startTime = ids[len > options.dumpCount ? len - options.dumpCount : 0];
|
|
}
|
|
}
|
|
if (key && value) {
|
|
filter = filter || {};
|
|
filter.filterKey = key;
|
|
filter.filterValue = value;
|
|
}
|
|
if (filterClientId) {
|
|
filter = filter || {};
|
|
filter.filterClientId = filterClientId;
|
|
}
|
|
var newIds =
|
|
clearNetwork || startTime == -1
|
|
? []
|
|
: getIds(startTime, count, options.lastRowId);
|
|
var setData = function (item) {
|
|
if (item) {
|
|
if (config.secureFilter) {
|
|
try {
|
|
item = config.secureFilter(item, clientIp, filter) || item;
|
|
} catch (e) {
|
|
if (config.debugMode) {
|
|
/* eslint-disable no-console */
|
|
console.log(e);
|
|
}
|
|
logger.error(e);
|
|
}
|
|
}
|
|
data[item.id] = item;
|
|
}
|
|
};
|
|
if (newIds.length > 0) {
|
|
if (filter) {
|
|
var id = newIds[0];
|
|
var index = ids.indexOf(newIds[0]);
|
|
newIds = [];
|
|
while (id && count > 0) {
|
|
var item = reqData[id];
|
|
if (checkItem(item, filter)) {
|
|
toBase64String(item.req);
|
|
toBase64String(item.res);
|
|
setData(item);
|
|
newIds.push(id);
|
|
--count;
|
|
}
|
|
id = ids[++index];
|
|
}
|
|
} else if (!filter) {
|
|
getList(newIds).forEach(setData);
|
|
}
|
|
}
|
|
getList(options.ids, options.status).forEach(setData);
|
|
var endId = ids[ids.length - 1];
|
|
var lastFrameId, frames;
|
|
if (options.lastFrameId == -3) {
|
|
var lastFrame = getLastFrame(options.curReqId);
|
|
if (lastFrame) {
|
|
if (lastFrame.closed || lastFrame.err) {
|
|
frames = [lastFrame];
|
|
} else {
|
|
lastFrameId = lastFrame.frameId;
|
|
}
|
|
}
|
|
} else {
|
|
frames = getFrames(options.curReqId, options.lastFrameId);
|
|
}
|
|
var lastNewId = newIds[newIds.length - 1];
|
|
var lastNewFrameId = frames && frames[frames.length - 1];
|
|
var hasNew =
|
|
(lastNewId && endId !== lastNewId) ||
|
|
(lastNewFrameId && lastNewFrameId !== lastFrameId);
|
|
var tunnelIds = options.tunnelIds;
|
|
var tunnelIps = {};
|
|
if (Array.isArray(tunnelIds) && tunnelIds.length > 0) {
|
|
tunnelIds.forEach(function (id) {
|
|
tunnelIps[id] = proxy.getTunnelIp(id);
|
|
});
|
|
}
|
|
|
|
return {
|
|
ids: options.ids || [],
|
|
composerTime: getNetworkTime(options.composerReqId),
|
|
tunnelIps: tunnelIps,
|
|
newIds: newIds,
|
|
data: data,
|
|
hasNew: hasNew,
|
|
lastId: clearNetwork ? endId : lastNewId,
|
|
endId: endId,
|
|
frames: frames,
|
|
lastFrameId: lastFrameId,
|
|
socketStatus: socketMgr.getStatus(options.curReqId)
|
|
};
|
|
};
|
|
};
|