mirror of
https://github.com/xboxeer/NScrapy.git
synced 2026-10-18 13:42:20 +08:00
zheng elendil
f53d6b78dc
- Add NScrapy.Cli project using Spectre.Console.Cli - Implement 'nscrapy new' command for generating spider projects from templates - Support basic and distributed spider templates - Configurable output directory and force overwrite - Implement 'nscrapy run' command for spider execution - Support single, spider, and downloader roles - Distributed mode with Redis configuration via CLI args - Configuration priority: CLI args > ENV > JSON - Update Dockerfiles to package CLI instead of raw DLLs - Update docker-compose.yml to use CLI commands
68 lines
2.6 KiB
Docker
68 lines
2.6 KiB
Docker
# =============================================================================
|
|
# Dockerfile.Spider - Multi-stage build for NScrapy CLI
|
|
# =============================================================================
|
|
# Build stage: compile NScrapy.Cli and all dependencies
|
|
# Runtime stage: run with .NET 10.0 runtime
|
|
#
|
|
# Usage:
|
|
# docker build -f Dockerfile.Spider -t nscrapy-spider .
|
|
# docker run nscrapy-spider run MySpider --role spider --distributed
|
|
#
|
|
# Environment variables (override CLI args):
|
|
# REDIS_HOST - Redis host address
|
|
# REDIS_PORT - Redis port (default: 6379)
|
|
# REDIS_PASSWORD - Redis password
|
|
# REDIS_USESSL - Use SSL for Redis ("true" / "false")
|
|
# NSCRAPY_REDIS_HOST - Same as REDIS_HOST (preferred)
|
|
# NSCRAPY_REDIS_PORT - Same as REDIS_PORT (preferred)
|
|
# =============================================================================
|
|
|
|
# ---- Build Stage ----
|
|
FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
|
|
WORKDIR /src
|
|
|
|
# Copy solution and project files first (layer caching optimisation)
|
|
COPY NScrapy.sln ./
|
|
COPY NScrapy.Cli/NScrapy.Cli.csproj NScrapy.Cli/
|
|
COPY NScrapy.Core/NScrapy.Core.csproj NScrapy.Core/
|
|
COPY NScrapy.Engine/NScrapy.Engine.csproj NScrapy.Engine/
|
|
COPY NScrapy.Infra/NScrapy.Infra.csproj NScrapy.Infra/
|
|
COPY NScrapy.Scheduler/NScrapy.Scheduler.csproj NScrapy.Scheduler/
|
|
COPY NScrapy.Spider/NScrapy.Spider.csproj NScrapy.Spider/
|
|
COPY NScrapy.Downloader/NScrapy.Downloader.csproj NScrapy.Downloader/
|
|
COPY NScrapy.Spider/NScrapy.Spider.csproj NScrapy.Spider/
|
|
|
|
# Restore all NuGet packages
|
|
RUN dotnet restore NScrapy.Cli/NScrapy.Cli.csproj
|
|
|
|
# Copy source code
|
|
COPY NScrapy.Cli/ NScrapy.Cli/
|
|
COPY NScrapy.Core/ NScrapy.Core/
|
|
COPY NScrapy.Engine/ NScrapy.Engine/
|
|
COPY NScrapy.Infra/ NScrapy.Infra/
|
|
COPY NScrapy.Scheduler/ NScrapy.Scheduler/
|
|
COPY NScrapy.Spider/ NScrapy.Spider/
|
|
COPY NScrapy.Downloader/ NScrapy.Downloader/
|
|
|
|
# Build and publish CLI
|
|
WORKDIR /src/NScrapy.Cli
|
|
RUN dotnet publish -c Release --no-restore -o /app/publish -p:PublishSingleFile=false
|
|
|
|
# ---- Runtime Stage ----
|
|
FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS runtime
|
|
WORKDIR /app
|
|
|
|
# Copy published output from build stage
|
|
COPY --from=build /app/publish .
|
|
|
|
# Copy log4net config so logging works inside the container
|
|
COPY NScrapy.Infra/log4net.config ./ 2>/dev/null || true
|
|
|
|
# Environment defaults
|
|
ENV DOTNET_RUNNING_IN_CONTAINER=true
|
|
ENV ASPNETCORE_URLS=http://+:80
|
|
|
|
# Entry point: run the CLI
|
|
# Usage: docker run nscrapy-spider run MySpider --role spider --distributed
|
|
ENTRYPOINT ["nscrapy"]
|