mirror of
https://github.com/xboxeer/NScrapy.git
synced 2026-10-19 23:03:19 +08:00
Elendil
50 lines
1.7 KiB
JSON
50 lines
1.7 KiB
JSON
{
|
|
"AppSettings": {
|
|
//"ConfigProvider": "Zookeeper",
|
|
//"ZookeeperEndpoint": "192.168.0.103:2181",
|
|
//"Scheduler": {
|
|
// "SchedulerType": "NScrapy.Scheduler.RedisExt.RedisScheduler"
|
|
//},
|
|
"Scheduler.RedisExt": {
|
|
"RedisServer": "192.168.0.106",
|
|
"RedisPort": "6379",
|
|
"ReceiverQueue": "NScrapy.Downloader",
|
|
"ResponseQueue": "NScrapy.ResponseQueue"
|
|
},
|
|
"DownloaderMiddlewares": [
|
|
{
|
|
"Middleware": "NScrapy.Downloader.Middleware.HttpUserAgentPoolMiddleware"
|
|
},
|
|
{
|
|
"RemovedMiddleware": "test3"
|
|
}
|
|
],
|
|
"Pipelines": [
|
|
{ "Pipeline": "NScrapy.Project.MongoItemPipeline" },
|
|
{ "Pipeline": "NScrapy.Project.CSVItemPipeline" }
|
|
],
|
|
"DownloaderPoolCapbility": 1000,
|
|
"HttpHeader": {
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
|
|
"Accept-Encoding": "gzip,deflate",
|
|
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36"
|
|
},
|
|
"SpiderProject": "NScrapy.Project",
|
|
"UserAgentPool": [
|
|
{
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko"
|
|
},
|
|
{
|
|
"User-Agent": "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"
|
|
},
|
|
{
|
|
"User-Agent": "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)"
|
|
},
|
|
{
|
|
"User-Agent": "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)"
|
|
}
|
|
]
|
|
}
|
|
}
|