{ // ============================================================================= // appsettings.spider.json - Example configuration for NScrapy Spider // ============================================================================= // This file configures the Spider (NScrapy.Shell / NScrapy.Project) when running // inside a Docker container. It is loaded from appsettings.json at runtime. // // ENVIRONMENT VARIABLE OVERRIDES // ------------------------------ // All settings in this file can be overridden by environment variables at // container startup. This is the recommended approach for Docker Compose. // // Mapping rule: JSON key path → environment variable name // Section key → "Section__SubKey__SubSubKey" (double-underscore separator) // // Examples: // AppSettings:Scheduler:SchedulerType → Scheduler__SchedulerType // AppSettings:Scheduler:RedisHost → Scheduler__RedisHost // AppSettings:SpiderEngine:SpiderEngineName → SpiderEngine__SpiderEngineName // // Docker Compose example: // environment: // - REDIS_ENABLED=true // - REDIS_HOST=redis // - REDIS_PORT=6379 // - Scheduler__SchedulerType=NScrapy.Scheduler.RedisExt.RedisScheduler // ============================================================================= "AppSettings": { // ---- Config Provider ---- // "ConfigProvider" values: "Zookeeper", "JsonFile" (default) // When using Zookeeper, also set ZookeeperEndpoint below. "ConfigProvider": "JsonFile", "ZookeeperEndpoint": "", // ---- Spider Engine ---- // Leave empty to use the default NScrapyEngine. // Override via: SpiderEngine__SpiderEngineName "SpiderEngine": { "SpiderEngineName": "", "SpiderEngineAssembly": "" }, // ---- Scheduler Configuration ---- // SchedulerType options: // - "NScrapy.Scheduler.InMemoryScheduler" (local mode, no Redis) // - "NScrapy.Scheduler.RedisExt.RedisScheduler" (distributed mode, needs Redis) // // Override via: Scheduler__SchedulerType "Scheduler": { "SchedulerType": "NScrapy.Scheduler.InMemoryScheduler" }, // ---- Redis Scheduler Extension ---- // Only used when SchedulerType = RedisScheduler // Override via: Scheduler__RedisExt__RedisServer, Scheduler__RedisExt__RedisPort, etc. "Scheduler.RedisExt": { "RedisServer": "redis", "RedisPort": "6379", "ReceiverQueue": "NScrapy.Downloader", "ResponseQueue": "NScrapy.ResponseQueue" }, // ---- Downloader Pool ---- // Maximum number of concurrent downloader workers across all nodes. "DownloaderPoolCapbility": 4, // ---- Downloader Middlewares ---- "DownloaderMiddlewares": [ { "Middleware": "NScrapy.DownloaderShell.StatusUpdaterMiddleware" } ], // ---- HTTP Header Defaults ---- "HttpHeader": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "Accept-Encoding": "gzip,deflate,br", "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" }, // ---- User-Agent Pool (rotated per request) ---- "UserAgentPool": [ { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" }, { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15" }, { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0" }, { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" } ] } }