diff --git a/src/DotnetSpider.Agent/Program.cs b/src/DotnetSpider.Agent/Program.cs index 14e92cd7..70fc8170 100644 --- a/src/DotnetSpider.Agent/Program.cs +++ b/src/DotnetSpider.Agent/Program.cs @@ -20,7 +20,7 @@ namespace DotnetSpider.Agent .MinimumLevel.Override("System", LogEventLevel.Warning) .MinimumLevel.Override("Microsoft.AspNetCore.Authentication", LogEventLevel.Warning) .Enrich.FromLogContext() - .WriteTo.Console().WriteTo.RollingFile("logs/agent.txt") + .WriteTo.Console().WriteTo.RollingFile("logs/agent.log") .CreateLogger(); var builder = Host.CreateDefaultBuilder(args); diff --git a/src/DotnetSpider.Agent/appsettings.json b/src/DotnetSpider.Agent/appsettings.json index dd4fb591..641fda54 100644 --- a/src/DotnetSpider.Agent/appsettings.json +++ b/src/DotnetSpider.Agent/appsettings.json @@ -1,7 +1,7 @@ { "RabbitMQ": { "Exchange": "DOTNET_SPIDER", - "Host": "192.168.124.200", + "Host": "192.168.1.27", "UserName": "user", "Password": "password" }, diff --git a/src/DotnetSpider.AgentRegister/Program.cs b/src/DotnetSpider.AgentRegister/Program.cs index 436ea68e..1173f7cf 100644 --- a/src/DotnetSpider.AgentRegister/Program.cs +++ b/src/DotnetSpider.AgentRegister/Program.cs @@ -23,7 +23,7 @@ namespace DotnetSpider.AgentRegister .MinimumLevel.Override("System", LogEventLevel.Warning) .MinimumLevel.Override("Microsoft.AspNetCore.Authentication", LogEventLevel.Warning) .Enrich.FromLogContext() - .WriteTo.Console().WriteTo.RollingFile("logs/agent-register.txt") + .WriteTo.Console().WriteTo.RollingFile("logs/agent-register.log") .CreateLogger(); var builder = Host.CreateDefaultBuilder(args); diff --git a/src/DotnetSpider.Portal/BackgroundService/QuartzJob.cs b/src/DotnetSpider.Portal/BackgroundService/QuartzJob.cs index 5fd684aa..fa4b8e96 100644 --- a/src/DotnetSpider.Portal/BackgroundService/QuartzJob.cs +++ b/src/DotnetSpider.Portal/BackgroundService/QuartzJob.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Docker.DotNet; using Docker.DotNet.Models; @@ -45,11 +46,10 @@ namespace DotnetSpider.Portal.BackgroundService new Uri(options.Docker)) .CreateClient(); var batch = Guid.NewGuid().ToString("N"); - var env = new List((spider.Environment ?? "").Split(new[] {" "}, - StringSplitOptions.RemoveEmptyEntries)) + var env = new List((spider.Environment ?? "").Split(new[] {" ", "\n"}, + StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim())) { $"DOTNET_SPIDER_ID={batch}", - $"DOTNET_SPIDER_TYPE={spider.Type}", $"DOTNET_SPIDER_NAME={spider.Name}" }; @@ -62,13 +62,26 @@ namespace DotnetSpider.Portal.BackgroundService { {"dotnetspider.spider.id", spider.Id.ToString()}, {"dotnetspider.spider.batch", batch}, - {"dotnetspider.spider.type", spider.Type}, {"dotnetspider.spider.name", spider.Name} }, Env = env, HostConfig = new HostConfig() }; - parameters.HostConfig.Binds = options.DockerVolumes; + var volumes = new HashSet(); + foreach (var volume in options.DockerVolumes) + { + volumes.Add(volume); + } + + var configVolumes = new List((spider.Volume ?? "").Split(new[] {" ", "\n"}, + StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim())); + foreach (var volume in configVolumes) + { + volumes.Add(volume); + } + + parameters.HostConfig.Binds = volumes.ToList(); + var result = await client.Containers.CreateContainerAsync(parameters); if (result.ID == null) diff --git a/src/DotnetSpider.Portal/Data/Spider.cs b/src/DotnetSpider.Portal/Data/Spider.cs index 205d8010..ae006ec1 100644 --- a/src/DotnetSpider.Portal/Data/Spider.cs +++ b/src/DotnetSpider.Portal/Data/Spider.cs @@ -35,14 +35,6 @@ namespace DotnetSpider.Portal.Data [Column("IMAGE")] public string Image { get; set; } - /// - /// 需要运行的爬虫名称 - /// - [Required] - [StringLength(255)] - [Column("TYPE")] - public string Type { get; set; } - /// /// 定时表达式 /// @@ -58,6 +50,13 @@ namespace DotnetSpider.Portal.Data [Column("ENVIRONMENT")] public string Environment { get; set; } + /// + /// docker 运行挂载的盘 + /// + [StringLength(2000)] + [Column("VOLUME")] + public string Volume { get; set; } + /// /// Creation time of this entity. /// diff --git a/src/DotnetSpider.Portal/Migrations/20200520010613_AddDockerVolume.Designer.cs b/src/DotnetSpider.Portal/Migrations/20200520010613_AddDockerVolume.Designer.cs new file mode 100644 index 00000000..ecd29912 --- /dev/null +++ b/src/DotnetSpider.Portal/Migrations/20200520010613_AddDockerVolume.Designer.cs @@ -0,0 +1,134 @@ +// +using System; +using DotnetSpider.Portal.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +namespace DotnetSpider.Portal.Migrations +{ + [DbContext(typeof(PortalDbContext))] + [Migration("20200520010613_AddDockerVolume")] + partial class AddDockerVolume + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "3.1.3") + .HasAnnotation("Relational:MaxIdentifierLength", 64); + + modelBuilder.Entity("DotnetSpider.Portal.Data.Spider", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnName("ID") + .HasColumnType("int"); + + b.Property("CreationTime") + .HasColumnName("CREATION_TIME") + .HasColumnType("datetime(6)"); + + b.Property("Cron") + .IsRequired() + .HasColumnName("CRON") + .HasColumnType("varchar(255) CHARACTER SET utf8mb4") + .HasMaxLength(255); + + b.Property("Enabled") + .HasColumnName("ENABLED") + .HasColumnType("tinyint(1)"); + + b.Property("Environment") + .HasColumnName("ENVIRONMENT") + .HasColumnType("varchar(2000) CHARACTER SET utf8mb4") + .HasMaxLength(2000); + + b.Property("Image") + .IsRequired() + .HasColumnName("IMAGE") + .HasColumnType("varchar(255) CHARACTER SET utf8mb4") + .HasMaxLength(255); + + b.Property("LastModificationTime") + .HasColumnName("LAST_MODIFICATION_TIME") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasColumnName("NAME") + .HasColumnType("varchar(255) CHARACTER SET utf8mb4") + .HasMaxLength(255); + + b.Property("Type") + .IsRequired() + .HasColumnName("TYPE") + .HasColumnType("varchar(255) CHARACTER SET utf8mb4") + .HasMaxLength(255); + + b.Property("Volume") + .HasColumnName("VOLUME") + .HasColumnType("varchar(2000) CHARACTER SET utf8mb4") + .HasMaxLength(2000); + + b.HasKey("Id"); + + b.HasIndex("CreationTime"); + + b.HasIndex("Name"); + + b.ToTable("SPIDER"); + }); + + modelBuilder.Entity("DotnetSpider.Portal.Data.SpiderHistory", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnName("ID") + .HasColumnType("int"); + + b.Property("Batch") + .HasColumnName("BATCH") + .HasColumnType("varchar(36) CHARACTER SET utf8mb4") + .HasMaxLength(36); + + b.Property("ContainerId") + .HasColumnName("CONTAINER_ID") + .HasColumnType("varchar(100) CHARACTER SET utf8mb4") + .HasMaxLength(100); + + b.Property("CreationTime") + .HasColumnName("CREATION_TIME") + .HasColumnType("datetime(6)"); + + b.Property("SpiderId") + .HasColumnName("SPIDER_ID") + .HasColumnType("int"); + + b.Property("SpiderName") + .IsRequired() + .HasColumnName("SPIDER_NAME") + .HasColumnType("varchar(255) CHARACTER SET utf8mb4") + .HasMaxLength(255); + + b.Property("Status") + .IsRequired() + .HasColumnName("STATUS") + .HasColumnType("varchar(20) CHARACTER SET utf8mb4") + .HasMaxLength(20); + + b.HasKey("Id"); + + b.HasIndex("Batch"); + + b.HasIndex("CreationTime"); + + b.HasIndex("SpiderId"); + + b.ToTable("SPIDER_HISTORIES"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/DotnetSpider.Portal/Migrations/20200520010613_AddDockerVolume.cs b/src/DotnetSpider.Portal/Migrations/20200520010613_AddDockerVolume.cs new file mode 100644 index 00000000..df1f1e35 --- /dev/null +++ b/src/DotnetSpider.Portal/Migrations/20200520010613_AddDockerVolume.cs @@ -0,0 +1,23 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +namespace DotnetSpider.Portal.Migrations +{ + public partial class AddDockerVolume : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "VOLUME", + table: "SPIDER", + maxLength: 2000, + nullable: true); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "VOLUME", + table: "SPIDER"); + } + } +} diff --git a/src/DotnetSpider.Portal/Migrations/20200521083459_RemoveTypeColumn.Designer.cs b/src/DotnetSpider.Portal/Migrations/20200521083459_RemoveTypeColumn.Designer.cs new file mode 100644 index 00000000..e2189b78 --- /dev/null +++ b/src/DotnetSpider.Portal/Migrations/20200521083459_RemoveTypeColumn.Designer.cs @@ -0,0 +1,128 @@ +// +using System; +using DotnetSpider.Portal.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +namespace DotnetSpider.Portal.Migrations +{ + [DbContext(typeof(PortalDbContext))] + [Migration("20200521083459_RemoveTypeColumn")] + partial class RemoveTypeColumn + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "3.1.3") + .HasAnnotation("Relational:MaxIdentifierLength", 64); + + modelBuilder.Entity("DotnetSpider.Portal.Data.Spider", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnName("ID") + .HasColumnType("int"); + + b.Property("CreationTime") + .HasColumnName("CREATION_TIME") + .HasColumnType("datetime(6)"); + + b.Property("Cron") + .IsRequired() + .HasColumnName("CRON") + .HasColumnType("varchar(255) CHARACTER SET utf8mb4") + .HasMaxLength(255); + + b.Property("Enabled") + .HasColumnName("ENABLED") + .HasColumnType("tinyint(1)"); + + b.Property("Environment") + .HasColumnName("ENVIRONMENT") + .HasColumnType("varchar(2000) CHARACTER SET utf8mb4") + .HasMaxLength(2000); + + b.Property("Image") + .IsRequired() + .HasColumnName("IMAGE") + .HasColumnType("varchar(255) CHARACTER SET utf8mb4") + .HasMaxLength(255); + + b.Property("LastModificationTime") + .HasColumnName("LAST_MODIFICATION_TIME") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasColumnName("NAME") + .HasColumnType("varchar(255) CHARACTER SET utf8mb4") + .HasMaxLength(255); + + b.Property("Volume") + .HasColumnName("VOLUME") + .HasColumnType("varchar(2000) CHARACTER SET utf8mb4") + .HasMaxLength(2000); + + b.HasKey("Id"); + + b.HasIndex("CreationTime"); + + b.HasIndex("Name"); + + b.ToTable("SPIDER"); + }); + + modelBuilder.Entity("DotnetSpider.Portal.Data.SpiderHistory", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnName("ID") + .HasColumnType("int"); + + b.Property("Batch") + .HasColumnName("BATCH") + .HasColumnType("varchar(36) CHARACTER SET utf8mb4") + .HasMaxLength(36); + + b.Property("ContainerId") + .HasColumnName("CONTAINER_ID") + .HasColumnType("varchar(100) CHARACTER SET utf8mb4") + .HasMaxLength(100); + + b.Property("CreationTime") + .HasColumnName("CREATION_TIME") + .HasColumnType("datetime(6)"); + + b.Property("SpiderId") + .HasColumnName("SPIDER_ID") + .HasColumnType("int"); + + b.Property("SpiderName") + .IsRequired() + .HasColumnName("SPIDER_NAME") + .HasColumnType("varchar(255) CHARACTER SET utf8mb4") + .HasMaxLength(255); + + b.Property("Status") + .IsRequired() + .HasColumnName("STATUS") + .HasColumnType("varchar(20) CHARACTER SET utf8mb4") + .HasMaxLength(20); + + b.HasKey("Id"); + + b.HasIndex("Batch"); + + b.HasIndex("CreationTime"); + + b.HasIndex("SpiderId"); + + b.ToTable("SPIDER_HISTORIES"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/DotnetSpider.Portal/Migrations/20200521083459_RemoveTypeColumn.cs b/src/DotnetSpider.Portal/Migrations/20200521083459_RemoveTypeColumn.cs new file mode 100644 index 00000000..04449d8b --- /dev/null +++ b/src/DotnetSpider.Portal/Migrations/20200521083459_RemoveTypeColumn.cs @@ -0,0 +1,25 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +namespace DotnetSpider.Portal.Migrations +{ + public partial class RemoveTypeColumn : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "TYPE", + table: "SPIDER"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "TYPE", + table: "SPIDER", + type: "varchar(255) CHARACTER SET utf8mb4", + maxLength: 255, + nullable: false, + defaultValue: ""); + } + } +} diff --git a/src/DotnetSpider.Portal/Migrations/PortalDbContextModelSnapshot.cs b/src/DotnetSpider.Portal/Migrations/PortalDbContextModelSnapshot.cs index 1253e349..8e5769e2 100644 --- a/src/DotnetSpider.Portal/Migrations/PortalDbContextModelSnapshot.cs +++ b/src/DotnetSpider.Portal/Migrations/PortalDbContextModelSnapshot.cs @@ -3,6 +3,7 @@ using System; using DotnetSpider.Portal.Data; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; namespace DotnetSpider.Portal.Migrations { @@ -58,11 +59,10 @@ namespace DotnetSpider.Portal.Migrations .HasColumnType("varchar(255) CHARACTER SET utf8mb4") .HasMaxLength(255); - b.Property("Type") - .IsRequired() - .HasColumnName("TYPE") - .HasColumnType("varchar(255) CHARACTER SET utf8mb4") - .HasMaxLength(255); + b.Property("Volume") + .HasColumnName("VOLUME") + .HasColumnType("varchar(2000) CHARACTER SET utf8mb4") + .HasMaxLength(2000); b.HasKey("Id"); @@ -73,7 +73,7 @@ namespace DotnetSpider.Portal.Migrations b.ToTable("SPIDER"); }); - modelBuilder.Entity("DotnetSpider.Portal.Data.SpiderContainer", b => + modelBuilder.Entity("DotnetSpider.Portal.Data.SpiderHistory", b => { b.Property("Id") .ValueGeneratedOnAdd() diff --git a/src/DotnetSpider.Portal/Program.cs b/src/DotnetSpider.Portal/Program.cs index e5c0c125..778b437c 100644 --- a/src/DotnetSpider.Portal/Program.cs +++ b/src/DotnetSpider.Portal/Program.cs @@ -28,7 +28,7 @@ namespace DotnetSpider.Portal Builder = WebHost.CreateDefaultBuilder(args) .UseStartup() .UseSerilog() - .UseUrls("http://0.0.0.0:7896"); + .UseUrls("http://localhost:7896"); await Builder.Build().RunAsync(); } } diff --git a/src/DotnetSpider.Portal/SeedData.cs b/src/DotnetSpider.Portal/SeedData.cs index 1dba9545..c71ebf9d 100644 --- a/src/DotnetSpider.Portal/SeedData.cs +++ b/src/DotnetSpider.Portal/SeedData.cs @@ -77,7 +77,6 @@ namespace DotnetSpider.Portal { Name = "cnblogs", Cron = "0 1 */1 * * ?", - Type = "DotnetSpider.Spiders.CnblogsSpider", Image = "dotnetspider/spiders.startup:latest", CreationTime = DateTimeOffset.Now, Enabled = true, diff --git a/src/DotnetSpider.Portal/ViewObject/ListSpiderViewObject.cs b/src/DotnetSpider.Portal/ViewObject/ListSpiderViewObject.cs index f1e59f46..83dc786d 100644 --- a/src/DotnetSpider.Portal/ViewObject/ListSpiderViewObject.cs +++ b/src/DotnetSpider.Portal/ViewObject/ListSpiderViewObject.cs @@ -8,6 +8,7 @@ namespace DotnetSpider.Portal.ViewObject public string Cron { get; set; } public string Type { get; set; } public string Environment { get; set; } + public string Volume { get; set; } public bool Enabled { get; set; } public string CreationTime { get; set; } public string LastModificationTime { get; set; } diff --git a/src/DotnetSpider.Portal/ViewObject/SpiderViewObject.cs b/src/DotnetSpider.Portal/ViewObject/SpiderViewObject.cs index 7a3416a3..00756ae1 100644 --- a/src/DotnetSpider.Portal/ViewObject/SpiderViewObject.cs +++ b/src/DotnetSpider.Portal/ViewObject/SpiderViewObject.cs @@ -37,5 +37,11 @@ namespace DotnetSpider.Portal.ViewObject /// [StringLength(2000)] public string Environment { get; set; } + + /// + /// 挂载目录 + /// + [StringLength(2000)] + public string Volume { get; set; } } } diff --git a/src/DotnetSpider.Portal/Views/Spider/Index.cshtml b/src/DotnetSpider.Portal/Views/Spider/Index.cshtml index 5e705ddb..b5d55080 100644 --- a/src/DotnetSpider.Portal/Views/Spider/Index.cshtml +++ b/src/DotnetSpider.Portal/Views/Spider/Index.cshtml @@ -8,6 +8,9 @@ .modal-header, .modal-footer{ border: 0; } + .modal-lg { + max-width: 1024px; + } } @@ -43,7 +46,6 @@ NAME IMAGE CRON - TYPE ENVIONMENT EDIT TIME ACTION @@ -59,7 +61,6 @@ - @@ -123,18 +124,20 @@ -
+
+
+
- - + +
- - + +
diff --git a/src/DotnetSpider.Portal/appsettings.json b/src/DotnetSpider.Portal/appsettings.json index 5b2ddc37..933cd3c5 100644 --- a/src/DotnetSpider.Portal/appsettings.json +++ b/src/DotnetSpider.Portal/appsettings.json @@ -1,13 +1,13 @@ { - "ConnectionString": "Database='dotnetspider2';Data Source=localhost;password=1qazZAQ!;User ID=root;Port=3306;", + "ConnectionString": "Database='dotnetspider2';Data Source=192.168.10.17;password=1qazZAQ!;User ID=root;Port=3306;", "DatabaseType": "MySql", "RabbitMQ": { "Exchange": "DOTNET_SPIDER", - "Host": "localhost", + "Host": "192.168.10.17", "UserName": "user", "Password": "password" }, "Database": "dotnetspider2", - "Docker": "http://192.168.1.105:2376", + "Docker": "http://192.168.10.17:2376", "DockerVolumes": "" } diff --git a/src/DotnetSpider.Portal/wwwroot/js/spider.js b/src/DotnetSpider.Portal/wwwroot/js/spider.js index 4567ba6c..a7a76842 100644 --- a/src/DotnetSpider.Portal/wwwroot/js/spider.js +++ b/src/DotnetSpider.Portal/wwwroot/js/spider.js @@ -37,8 +37,8 @@ $(function () { name: record.name, image: record.image, cron: record.cron, - type: record.type, - environment: record.environment + environment: record.environment, + volume: record.volume }; $('#modal').modal('show') }, diff --git a/src/DotnetSpider.Sample/samples/CnblogsSpider.cs b/src/DotnetSpider.Sample/samples/CnblogsSpider.cs index a25eef1f..f7e89b2a 100644 --- a/src/DotnetSpider.Sample/samples/CnblogsSpider.cs +++ b/src/DotnetSpider.Sample/samples/CnblogsSpider.cs @@ -1,4 +1,5 @@ using System; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using DotnetSpider.DataFlow; @@ -63,7 +64,13 @@ namespace DotnetSpider.Sample.samples { public ListNewsParser() { - AddRequiredValidator("news\\.cnblogs\\.com/n/page"); + // AddRequiredValidator("news\\.cnblogs\\.com/n/page"); + AddRequiredValidator((request => + { + var host = request.RequestUri.Host; + var regex = host + "/$"; + return Regex.IsMatch(request.RequestUri.ToString(), regex); + })); // if you want to collect every pages // AddFollowRequestQuerier(Selectors.XPath(".//div[@class='pager']")); }