mirror of
https://github.com/dotnetcore/DotnetSpider.git
synced 2026-10-29 06:47:00 +08:00
Merge branch 'master' of https://github.com/dotnetcore/DotnetSpider
This commit is contained in:
邹嵩
@@ -20,7 +20,7 @@ namespace DotnetSpider.Agent
|
||||
.MinimumLevel.Override("System", LogEventLevel.Warning)
|
||||
.MinimumLevel.Override("Microsoft.AspNetCore.Authentication", LogEventLevel.Warning)
|
||||
.Enrich.FromLogContext()
|
||||
.WriteTo.Console().WriteTo.RollingFile("logs/agent.txt")
|
||||
.WriteTo.Console().WriteTo.RollingFile("logs/agent.log")
|
||||
.CreateLogger();
|
||||
|
||||
var builder = Host.CreateDefaultBuilder(args);
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"RabbitMQ": {
|
||||
"Exchange": "DOTNET_SPIDER",
|
||||
"Host": "192.168.124.200",
|
||||
"Host": "192.168.1.27",
|
||||
"UserName": "user",
|
||||
"Password": "password"
|
||||
},
|
||||
|
||||
@@ -23,7 +23,7 @@ namespace DotnetSpider.AgentRegister
|
||||
.MinimumLevel.Override("System", LogEventLevel.Warning)
|
||||
.MinimumLevel.Override("Microsoft.AspNetCore.Authentication", LogEventLevel.Warning)
|
||||
.Enrich.FromLogContext()
|
||||
.WriteTo.Console().WriteTo.RollingFile("logs/agent-register.txt")
|
||||
.WriteTo.Console().WriteTo.RollingFile("logs/agent-register.log")
|
||||
.CreateLogger();
|
||||
|
||||
var builder = Host.CreateDefaultBuilder(args);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Docker.DotNet;
|
||||
using Docker.DotNet.Models;
|
||||
@@ -45,11 +46,10 @@ namespace DotnetSpider.Portal.BackgroundService
|
||||
new Uri(options.Docker))
|
||||
.CreateClient();
|
||||
var batch = Guid.NewGuid().ToString("N");
|
||||
var env = new List<string>((spider.Environment ?? "").Split(new[] {" "},
|
||||
StringSplitOptions.RemoveEmptyEntries))
|
||||
var env = new List<string>((spider.Environment ?? "").Split(new[] {" ", "\n"},
|
||||
StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()))
|
||||
{
|
||||
$"DOTNET_SPIDER_ID={batch}",
|
||||
$"DOTNET_SPIDER_TYPE={spider.Type}",
|
||||
$"DOTNET_SPIDER_NAME={spider.Name}"
|
||||
};
|
||||
|
||||
@@ -62,13 +62,26 @@ namespace DotnetSpider.Portal.BackgroundService
|
||||
{
|
||||
{"dotnetspider.spider.id", spider.Id.ToString()},
|
||||
{"dotnetspider.spider.batch", batch},
|
||||
{"dotnetspider.spider.type", spider.Type},
|
||||
{"dotnetspider.spider.name", spider.Name}
|
||||
},
|
||||
Env = env,
|
||||
HostConfig = new HostConfig()
|
||||
};
|
||||
parameters.HostConfig.Binds = options.DockerVolumes;
|
||||
var volumes = new HashSet<string>();
|
||||
foreach (var volume in options.DockerVolumes)
|
||||
{
|
||||
volumes.Add(volume);
|
||||
}
|
||||
|
||||
var configVolumes = new List<string>((spider.Volume ?? "").Split(new[] {" ", "\n"},
|
||||
StringSplitOptions.RemoveEmptyEntries).Select(x => x.Trim()));
|
||||
foreach (var volume in configVolumes)
|
||||
{
|
||||
volumes.Add(volume);
|
||||
}
|
||||
|
||||
parameters.HostConfig.Binds = volumes.ToList();
|
||||
|
||||
var result = await client.Containers.CreateContainerAsync(parameters);
|
||||
|
||||
if (result.ID == null)
|
||||
|
||||
@@ -35,14 +35,6 @@ namespace DotnetSpider.Portal.Data
|
||||
[Column("IMAGE")]
|
||||
public string Image { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// 需要运行的爬虫名称
|
||||
/// </summary>
|
||||
[Required]
|
||||
[StringLength(255)]
|
||||
[Column("TYPE")]
|
||||
public string Type { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// 定时表达式
|
||||
/// </summary>
|
||||
@@ -58,6 +50,13 @@ namespace DotnetSpider.Portal.Data
|
||||
[Column("ENVIRONMENT")]
|
||||
public string Environment { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// docker 运行挂载的盘
|
||||
/// </summary>
|
||||
[StringLength(2000)]
|
||||
[Column("VOLUME")]
|
||||
public string Volume { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Creation time of this entity.
|
||||
/// </summary>
|
||||
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
// <auto-generated />
|
||||
using System;
|
||||
using DotnetSpider.Portal.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||
|
||||
namespace DotnetSpider.Portal.Migrations
|
||||
{
|
||||
[DbContext(typeof(PortalDbContext))]
|
||||
[Migration("20200520010613_AddDockerVolume")]
|
||||
partial class AddDockerVolume
|
||||
{
|
||||
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
||||
{
|
||||
#pragma warning disable 612, 618
|
||||
modelBuilder
|
||||
.HasAnnotation("ProductVersion", "3.1.3")
|
||||
.HasAnnotation("Relational:MaxIdentifierLength", 64);
|
||||
|
||||
modelBuilder.Entity("DotnetSpider.Portal.Data.Spider", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnName("ID")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<DateTimeOffset>("CreationTime")
|
||||
.HasColumnName("CREATION_TIME")
|
||||
.HasColumnType("datetime(6)");
|
||||
|
||||
b.Property<string>("Cron")
|
||||
.IsRequired()
|
||||
.HasColumnName("CRON")
|
||||
.HasColumnType("varchar(255) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(255);
|
||||
|
||||
b.Property<bool>("Enabled")
|
||||
.HasColumnName("ENABLED")
|
||||
.HasColumnType("tinyint(1)");
|
||||
|
||||
b.Property<string>("Environment")
|
||||
.HasColumnName("ENVIRONMENT")
|
||||
.HasColumnType("varchar(2000) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(2000);
|
||||
|
||||
b.Property<string>("Image")
|
||||
.IsRequired()
|
||||
.HasColumnName("IMAGE")
|
||||
.HasColumnType("varchar(255) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(255);
|
||||
|
||||
b.Property<DateTimeOffset>("LastModificationTime")
|
||||
.HasColumnName("LAST_MODIFICATION_TIME")
|
||||
.HasColumnType("datetime(6)");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.IsRequired()
|
||||
.HasColumnName("NAME")
|
||||
.HasColumnType("varchar(255) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(255);
|
||||
|
||||
b.Property<string>("Type")
|
||||
.IsRequired()
|
||||
.HasColumnName("TYPE")
|
||||
.HasColumnType("varchar(255) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(255);
|
||||
|
||||
b.Property<string>("Volume")
|
||||
.HasColumnName("VOLUME")
|
||||
.HasColumnType("varchar(2000) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(2000);
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("CreationTime");
|
||||
|
||||
b.HasIndex("Name");
|
||||
|
||||
b.ToTable("SPIDER");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DotnetSpider.Portal.Data.SpiderHistory", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnName("ID")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<string>("Batch")
|
||||
.HasColumnName("BATCH")
|
||||
.HasColumnType("varchar(36) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(36);
|
||||
|
||||
b.Property<string>("ContainerId")
|
||||
.HasColumnName("CONTAINER_ID")
|
||||
.HasColumnType("varchar(100) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(100);
|
||||
|
||||
b.Property<DateTimeOffset>("CreationTime")
|
||||
.HasColumnName("CREATION_TIME")
|
||||
.HasColumnType("datetime(6)");
|
||||
|
||||
b.Property<int>("SpiderId")
|
||||
.HasColumnName("SPIDER_ID")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<string>("SpiderName")
|
||||
.IsRequired()
|
||||
.HasColumnName("SPIDER_NAME")
|
||||
.HasColumnType("varchar(255) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(255);
|
||||
|
||||
b.Property<string>("Status")
|
||||
.IsRequired()
|
||||
.HasColumnName("STATUS")
|
||||
.HasColumnType("varchar(20) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(20);
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("Batch");
|
||||
|
||||
b.HasIndex("CreationTime");
|
||||
|
||||
b.HasIndex("SpiderId");
|
||||
|
||||
b.ToTable("SPIDER_HISTORIES");
|
||||
});
|
||||
#pragma warning restore 612, 618
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
namespace DotnetSpider.Portal.Migrations
|
||||
{
|
||||
public partial class AddDockerVolume : Migration
|
||||
{
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.AddColumn<string>(
|
||||
name: "VOLUME",
|
||||
table: "SPIDER",
|
||||
maxLength: 2000,
|
||||
nullable: true);
|
||||
}
|
||||
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropColumn(
|
||||
name: "VOLUME",
|
||||
table: "SPIDER");
|
||||
}
|
||||
}
|
||||
}
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
// <auto-generated />
|
||||
using System;
|
||||
using DotnetSpider.Portal.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||
|
||||
namespace DotnetSpider.Portal.Migrations
|
||||
{
|
||||
[DbContext(typeof(PortalDbContext))]
|
||||
[Migration("20200521083459_RemoveTypeColumn")]
|
||||
partial class RemoveTypeColumn
|
||||
{
|
||||
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
||||
{
|
||||
#pragma warning disable 612, 618
|
||||
modelBuilder
|
||||
.HasAnnotation("ProductVersion", "3.1.3")
|
||||
.HasAnnotation("Relational:MaxIdentifierLength", 64);
|
||||
|
||||
modelBuilder.Entity("DotnetSpider.Portal.Data.Spider", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnName("ID")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<DateTimeOffset>("CreationTime")
|
||||
.HasColumnName("CREATION_TIME")
|
||||
.HasColumnType("datetime(6)");
|
||||
|
||||
b.Property<string>("Cron")
|
||||
.IsRequired()
|
||||
.HasColumnName("CRON")
|
||||
.HasColumnType("varchar(255) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(255);
|
||||
|
||||
b.Property<bool>("Enabled")
|
||||
.HasColumnName("ENABLED")
|
||||
.HasColumnType("tinyint(1)");
|
||||
|
||||
b.Property<string>("Environment")
|
||||
.HasColumnName("ENVIRONMENT")
|
||||
.HasColumnType("varchar(2000) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(2000);
|
||||
|
||||
b.Property<string>("Image")
|
||||
.IsRequired()
|
||||
.HasColumnName("IMAGE")
|
||||
.HasColumnType("varchar(255) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(255);
|
||||
|
||||
b.Property<DateTimeOffset>("LastModificationTime")
|
||||
.HasColumnName("LAST_MODIFICATION_TIME")
|
||||
.HasColumnType("datetime(6)");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.IsRequired()
|
||||
.HasColumnName("NAME")
|
||||
.HasColumnType("varchar(255) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(255);
|
||||
|
||||
b.Property<string>("Volume")
|
||||
.HasColumnName("VOLUME")
|
||||
.HasColumnType("varchar(2000) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(2000);
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("CreationTime");
|
||||
|
||||
b.HasIndex("Name");
|
||||
|
||||
b.ToTable("SPIDER");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DotnetSpider.Portal.Data.SpiderHistory", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnName("ID")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<string>("Batch")
|
||||
.HasColumnName("BATCH")
|
||||
.HasColumnType("varchar(36) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(36);
|
||||
|
||||
b.Property<string>("ContainerId")
|
||||
.HasColumnName("CONTAINER_ID")
|
||||
.HasColumnType("varchar(100) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(100);
|
||||
|
||||
b.Property<DateTimeOffset>("CreationTime")
|
||||
.HasColumnName("CREATION_TIME")
|
||||
.HasColumnType("datetime(6)");
|
||||
|
||||
b.Property<int>("SpiderId")
|
||||
.HasColumnName("SPIDER_ID")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<string>("SpiderName")
|
||||
.IsRequired()
|
||||
.HasColumnName("SPIDER_NAME")
|
||||
.HasColumnType("varchar(255) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(255);
|
||||
|
||||
b.Property<string>("Status")
|
||||
.IsRequired()
|
||||
.HasColumnName("STATUS")
|
||||
.HasColumnType("varchar(20) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(20);
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("Batch");
|
||||
|
||||
b.HasIndex("CreationTime");
|
||||
|
||||
b.HasIndex("SpiderId");
|
||||
|
||||
b.ToTable("SPIDER_HISTORIES");
|
||||
});
|
||||
#pragma warning restore 612, 618
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
namespace DotnetSpider.Portal.Migrations
|
||||
{
|
||||
public partial class RemoveTypeColumn : Migration
|
||||
{
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropColumn(
|
||||
name: "TYPE",
|
||||
table: "SPIDER");
|
||||
}
|
||||
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.AddColumn<string>(
|
||||
name: "TYPE",
|
||||
table: "SPIDER",
|
||||
type: "varchar(255) CHARACTER SET utf8mb4",
|
||||
maxLength: 255,
|
||||
nullable: false,
|
||||
defaultValue: "");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ using System;
|
||||
using DotnetSpider.Portal.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||
|
||||
namespace DotnetSpider.Portal.Migrations
|
||||
{
|
||||
@@ -58,11 +59,10 @@ namespace DotnetSpider.Portal.Migrations
|
||||
.HasColumnType("varchar(255) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(255);
|
||||
|
||||
b.Property<string>("Type")
|
||||
.IsRequired()
|
||||
.HasColumnName("TYPE")
|
||||
.HasColumnType("varchar(255) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(255);
|
||||
b.Property<string>("Volume")
|
||||
.HasColumnName("VOLUME")
|
||||
.HasColumnType("varchar(2000) CHARACTER SET utf8mb4")
|
||||
.HasMaxLength(2000);
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
@@ -73,7 +73,7 @@ namespace DotnetSpider.Portal.Migrations
|
||||
b.ToTable("SPIDER");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DotnetSpider.Portal.Data.SpiderContainer", b =>
|
||||
modelBuilder.Entity("DotnetSpider.Portal.Data.SpiderHistory", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
|
||||
@@ -28,7 +28,7 @@ namespace DotnetSpider.Portal
|
||||
Builder = WebHost.CreateDefaultBuilder(args)
|
||||
.UseStartup<Startup>()
|
||||
.UseSerilog()
|
||||
.UseUrls("http://0.0.0.0:7896");
|
||||
.UseUrls("http://localhost:7896");
|
||||
await Builder.Build().RunAsync();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,7 +77,6 @@ namespace DotnetSpider.Portal
|
||||
{
|
||||
Name = "cnblogs",
|
||||
Cron = "0 1 */1 * * ?",
|
||||
Type = "DotnetSpider.Spiders.CnblogsSpider",
|
||||
Image = "dotnetspider/spiders.startup:latest",
|
||||
CreationTime = DateTimeOffset.Now,
|
||||
Enabled = true,
|
||||
|
||||
@@ -8,6 +8,7 @@ namespace DotnetSpider.Portal.ViewObject
|
||||
public string Cron { get; set; }
|
||||
public string Type { get; set; }
|
||||
public string Environment { get; set; }
|
||||
public string Volume { get; set; }
|
||||
public bool Enabled { get; set; }
|
||||
public string CreationTime { get; set; }
|
||||
public string LastModificationTime { get; set; }
|
||||
|
||||
@@ -37,5 +37,11 @@ namespace DotnetSpider.Portal.ViewObject
|
||||
/// </summary>
|
||||
[StringLength(2000)]
|
||||
public string Environment { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// 挂载目录
|
||||
/// </summary>
|
||||
[StringLength(2000)]
|
||||
public string Volume { get; set; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,9 @@
|
||||
.modal-header, .modal-footer{
|
||||
border: 0;
|
||||
}
|
||||
.modal-lg {
|
||||
max-width: 1024px;
|
||||
}
|
||||
</style>
|
||||
}
|
||||
|
||||
@@ -43,7 +46,6 @@
|
||||
<th>NAME</th>
|
||||
<th>IMAGE</th>
|
||||
<th>CRON</th>
|
||||
<th>TYPE</th>
|
||||
<th>ENVIONMENT</th>
|
||||
<th>EDIT TIME</th>
|
||||
<th style="width: 360px">ACTION</th>
|
||||
@@ -59,7 +61,6 @@
|
||||
<td v-text="item.name"></td>
|
||||
<td v-text="item.image"></td>
|
||||
<td v-text="item.cron"></td>
|
||||
<td v-text="item.type"></td>
|
||||
<td v-text="item.environment"></td>
|
||||
<td v-text="item.lastModificationTime"></td>
|
||||
<td>
|
||||
@@ -123,18 +124,20 @@
|
||||
<input type="text" v-model="item.cron" class="form-control"/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="col-md-12">
|
||||
<div class="form-group">
|
||||
<label>Type</label>
|
||||
<input type="text" v-model="item.type" class="form-control"/>
|
||||
<label>Environment</label>
|
||||
<textarea style="height: 100px" type="text" v-model="item.environment" class="form-control"></textarea>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="col-md-12">
|
||||
<div class="form-group">
|
||||
<label>Environment</label>
|
||||
<textarea style="height: 80px" type="text" v-model="item.environment" class="form-control"></textarea>
|
||||
<label>Volume</label>
|
||||
<textarea style="height: 100px" type="text" v-model="item.volume" class="form-control"></textarea>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
{
|
||||
"ConnectionString": "Database='dotnetspider2';Data Source=localhost;password=1qazZAQ!;User ID=root;Port=3306;",
|
||||
"ConnectionString": "Database='dotnetspider2';Data Source=192.168.10.17;password=1qazZAQ!;User ID=root;Port=3306;",
|
||||
"DatabaseType": "MySql",
|
||||
"RabbitMQ": {
|
||||
"Exchange": "DOTNET_SPIDER",
|
||||
"Host": "localhost",
|
||||
"Host": "192.168.10.17",
|
||||
"UserName": "user",
|
||||
"Password": "password"
|
||||
},
|
||||
"Database": "dotnetspider2",
|
||||
"Docker": "http://192.168.1.105:2376",
|
||||
"Docker": "http://192.168.10.17:2376",
|
||||
"DockerVolumes": ""
|
||||
}
|
||||
|
||||
@@ -37,8 +37,8 @@ $(function () {
|
||||
name: record.name,
|
||||
image: record.image,
|
||||
cron: record.cron,
|
||||
type: record.type,
|
||||
environment: record.environment
|
||||
environment: record.environment,
|
||||
volume: record.volume
|
||||
};
|
||||
$('#modal').modal('show')
|
||||
},
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using DotnetSpider.DataFlow;
|
||||
@@ -63,7 +64,13 @@ namespace DotnetSpider.Sample.samples
|
||||
{
|
||||
public ListNewsParser()
|
||||
{
|
||||
AddRequiredValidator("news\\.cnblogs\\.com/n/page");
|
||||
// AddRequiredValidator("news\\.cnblogs\\.com/n/page");
|
||||
AddRequiredValidator((request =>
|
||||
{
|
||||
var host = request.RequestUri.Host;
|
||||
var regex = host + "/$";
|
||||
return Regex.IsMatch(request.RequestUri.ToString(), regex);
|
||||
}));
|
||||
// if you want to collect every pages
|
||||
// AddFollowRequestQuerier(Selectors.XPath(".//div[@class='pager']"));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user