mirror of
https://github.com/xboxeer/NScrapy.git
synced 2026-10-20 23:49:33 +08:00
Elendil
46086ffaf9
Create a ConifgProviderFactory so thay we can get ConfigProvider from a single point Also enhenced the logic of refresh config, right now when zookeeper watcher dected the config is changed NScrapy will update its context, and during that time, NScrapy will lock the access to the Connection of Redis
44 lines
1.4 KiB
JSON
44 lines
1.4 KiB
JSON
{
|
|
"AppSettings": {
|
|
"ConfigProvider": "Zookeeper",
|
|
"ZookeeperEndpoint": "192.168.0.103:2181",
|
|
"Scheduler": {
|
|
"SchedulerType": "NScrapy.Scheduler.RedisExt.RedisScheduler"
|
|
},
|
|
"Scheduler.RedisExt": {
|
|
"RedisServer": "192.168.0.103",
|
|
"RedisPort": "6379",
|
|
"ReceiverQueue": "NScrapy.Downloader",
|
|
"ResponseQueue": "NScrapy.ResponseQueue"
|
|
},
|
|
"DownloaderMiddlewares": [
|
|
{
|
|
"Middleware": "NScrapy.Downloader.Middleware.HttpUserAgentPoolMiddleware"
|
|
},
|
|
{
|
|
"Middleware": "NScrapy.DownloaderShell.StatusUpdaterMiddleware"
|
|
},
|
|
{
|
|
"RemovedMiddleware": "test3"
|
|
}
|
|
],
|
|
"Pipelines": [
|
|
{ "Pipeline": "NScrapy.Project.MongoItemPipeline" },
|
|
{ "Pipeline": "NScrapy.Project.CSVItemPipeline" }
|
|
],
|
|
"DownloaderPoolCapbility": 1000,
|
|
"HttpHeader": {
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
|
|
"Accept-Encoding": "gzip,deflate",
|
|
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36"
|
|
},
|
|
"SpiderProject": "NScrapy.Project",
|
|
"UserAgentPool": [
|
|
{
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko"
|
|
}
|
|
]
|
|
}
|
|
}
|