mirror of
https://github.com/xboxeer/NScrapy.git
synced 2026-10-28 20:12:39 +08:00
zheng elendil
ef96e3aa45
- docker-compose.yml: 3-mode support (local-only, local-redis, managed-redis) - Dockerfile.Spider: multi-stage build (SDK 10.0 → runtime 10.0) - Dockerfile.Downloader: multi-stage build for DownloaderShell - appsettings.spider.json: example Spider config with env var override docs - appsettings.downloader.json: example Downloader config - README.md: add Docker Deployment section
91 lines
3.8 KiB
JSON
91 lines
3.8 KiB
JSON
{
|
|
// =============================================================================
|
|
// appsettings.spider.json - Example configuration for NScrapy Spider
|
|
// =============================================================================
|
|
// This file configures the Spider (NScrapy.Shell / NScrapy.Project) when running
|
|
// inside a Docker container. It is loaded from appsettings.json at runtime.
|
|
//
|
|
// ENVIRONMENT VARIABLE OVERRIDES
|
|
// ------------------------------
|
|
// All settings in this file can be overridden by environment variables at
|
|
// container startup. This is the recommended approach for Docker Compose.
|
|
//
|
|
// Mapping rule: JSON key path → environment variable name
|
|
// Section key → "Section__SubKey__SubSubKey" (double-underscore separator)
|
|
//
|
|
// Examples:
|
|
// AppSettings:Scheduler:SchedulerType → Scheduler__SchedulerType
|
|
// AppSettings:Scheduler:RedisHost → Scheduler__RedisHost
|
|
// AppSettings:SpiderEngine:SpiderEngineName → SpiderEngine__SpiderEngineName
|
|
//
|
|
// Docker Compose example:
|
|
// environment:
|
|
// - REDIS_ENABLED=true
|
|
// - REDIS_HOST=redis
|
|
// - REDIS_PORT=6379
|
|
// - Scheduler__SchedulerType=NScrapy.Scheduler.RedisExt.RedisScheduler
|
|
// =============================================================================
|
|
|
|
"AppSettings": {
|
|
|
|
// ---- Config Provider ----
|
|
// "ConfigProvider" values: "Zookeeper", "JsonFile" (default)
|
|
// When using Zookeeper, also set ZookeeperEndpoint below.
|
|
"ConfigProvider": "JsonFile",
|
|
"ZookeeperEndpoint": "",
|
|
|
|
// ---- Spider Engine ----
|
|
// Leave empty to use the default NScrapyEngine.
|
|
// Override via: SpiderEngine__SpiderEngineName
|
|
"SpiderEngine": {
|
|
"SpiderEngineName": "",
|
|
"SpiderEngineAssembly": ""
|
|
},
|
|
|
|
// ---- Scheduler Configuration ----
|
|
// SchedulerType options:
|
|
// - "NScrapy.Scheduler.InMemoryScheduler" (local mode, no Redis)
|
|
// - "NScrapy.Scheduler.RedisExt.RedisScheduler" (distributed mode, needs Redis)
|
|
//
|
|
// Override via: Scheduler__SchedulerType
|
|
"Scheduler": {
|
|
"SchedulerType": "NScrapy.Scheduler.InMemoryScheduler"
|
|
},
|
|
|
|
// ---- Redis Scheduler Extension ----
|
|
// Only used when SchedulerType = RedisScheduler
|
|
// Override via: Scheduler__RedisExt__RedisServer, Scheduler__RedisExt__RedisPort, etc.
|
|
"Scheduler.RedisExt": {
|
|
"RedisServer": "redis",
|
|
"RedisPort": "6379",
|
|
"ReceiverQueue": "NScrapy.Downloader",
|
|
"ResponseQueue": "NScrapy.ResponseQueue"
|
|
},
|
|
|
|
// ---- Downloader Pool ----
|
|
// Maximum number of concurrent downloader workers across all nodes.
|
|
"DownloaderPoolCapbility": 4,
|
|
|
|
// ---- Downloader Middlewares ----
|
|
"DownloaderMiddlewares": [
|
|
{ "Middleware": "NScrapy.DownloaderShell.StatusUpdaterMiddleware" }
|
|
],
|
|
|
|
// ---- HTTP Header Defaults ----
|
|
"HttpHeader": {
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
"Accept-Encoding": "gzip,deflate,br",
|
|
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
|
},
|
|
|
|
// ---- User-Agent Pool (rotated per request) ----
|
|
"UserAgentPool": [
|
|
{ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" },
|
|
{ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15" },
|
|
{ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0" },
|
|
{ "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" }
|
|
]
|
|
}
|
|
}
|