GithubStar/DotnetSpider
1
0
Fork 0
mirror of https://github.com/dotnetcore/DotnetSpider.git synced 2026-10-29 14:50:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

upgrade depens & refactor logs inforamtion

Browse Source
This commit is contained in:
Lewis Zou
2020-05-12 09:54:24 +08:00
parent 5ae0bb6fa6
commit 62ce282bfe
12 changed files with 41 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/DotnetSpider.Mongo/DotnetSpider.Mongo.csproj
+1 -1
View File
@@ -6,7 +6,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="MongoDB.Driver" Version="2.10.3" />
<PackageReference Include="MongoDB.Driver" Version="2.10.4" />
</ItemGroup>
<ItemGroup>
src/DotnetSpider.MySql/DotnetSpider.MySql.csproj
+1 -1
View File
@@ -10,6 +10,6 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="MySqlConnector" Version="0.64.0" />
<PackageReference Include="MySqlConnector" Version="0.66.0" />
</ItemGroup>
</Project>
src/DotnetSpider.Portal/DotnetSpider.Portal.csproj
+1 -1
View File
@@ -14,7 +14,7 @@
<PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="3.1.1" />
<PackageReference Include="Quartz" Version="3.0.7" />
<PackageReference Include="Quartz.AspNetCore.MySqlConnector" Version="1.0.4" />
<PackageReference Include="MySqlConnector" Version="0.64.0" />
<PackageReference Include="MySqlConnector" Version="0.66.0" />
<PackageReference Include="Serilog.AspNetCore" Version="3.2.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
<PackageReference Include="Serilog.Sinks.RollingFile" Version="3.3.0" />
src/DotnetSpider.Sample/DotnetSpider.Sample.csproj
-1
View File
@@ -18,7 +18,6 @@
<PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
<PackageReference Include="Serilog.Sinks.RollingFile" Version="3.3.0" />
<PackageReference Include="Serilog.Sinks.PeriodicBatching" Version="2.3.0" />
<PackageReference Include="System.Text.Json" Version="4.7.1" />
</ItemGroup>
<ItemGroup>
src/DotnetSpider.Sample/Program.cs
+1 -1
View File
@@ -33,7 +33,7 @@ namespace DotnetSpider.Sample
.WriteTo.Console().WriteTo.RollingFile("logs/spider.txt")
.CreateLogger();
await JsonEntitySpider.RunAsync();
await BaseUsageSpider.RunAsync();
Console.WriteLine("Bye!");
Environment.Exit(0);
src/DotnetSpider.Tests/DotnetSpider.Tests.csproj
+2 -2
View File
@@ -7,12 +7,12 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
<PackageReference Include="System.Text.Json" Version="4.7.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />
<PackageReference Include="coverlet.collector" Version="1.2.1" />
<PackageReference Include="Moq" Version="4.13.1" />
<PackageReference Include="Moq" Version="4.14.1" />
</ItemGroup>
<ItemGroup>
src/DotnetSpider/Agent/AgentService.cs
+1 -1
View File
@@ -134,7 +134,7 @@ namespace DotnetSpider.Agent
response.Agent = _options.AgentId;
await _messageQueue.PublishAsBytesAsync(string.Format(TopicNames.Spider, request.Owner.ToUpper()),
response);
_logger.LogInformation($"{request.RequestUri} download success");
_logger.LogInformation($"{request.Owner} download {request.RequestUri}, {request.Hash} success");
}
else
{
src/DotnetSpider/Agent/PPPoEService.cs
+2 -2
View File
@@ -32,7 +32,7 @@ namespace DotnetSpider.Agent
/// <returns></returns>
public async Task<string> DetectAsync(Request request, HttpResponseMessage httpResponseMessage)
{
var redialRegExp = request.GetHeader(Consts.RedialRegExp);
var redialRegExp = request.GetHeader(Consts.RedialRegexExpression);
if (IsActive && !string.IsNullOrWhiteSpace(redialRegExp))
{
var text = await httpResponseMessage.Content.ReadAsStringAsync();
@@ -127,4 +127,4 @@ namespace DotnetSpider.Agent
}
}
}
}
}
src/DotnetSpider/Consts.cs
+2 -2
View File
@@ -3,7 +3,7 @@ namespace DotnetSpider
public static class Consts
{
public const string ProxyPrefix = "PROXY_";
public const string RedialRegExp = "DOTNET_SPIDER_REDIAL_REGEXP";
public const string RedialRegexExpression = "DOTNET_SPIDER_REDIAL_REGEXP";
public const string ResponseBytes = "DOTNET_SPIDER_RESPONSE_BYTES";
}
}
}
src/DotnetSpider/Http/Request.cs
+2 -2
View File
@@ -87,8 +87,8 @@ namespace DotnetSpider.Http
[IgnoreMember]
public string RedialRegExp
{
get => GetProperty(Consts.RedialRegExp);
set => SetProperty(Consts.RedialRegExp, value);
get => GetProperty(Consts.RedialRegexExpression);
set => SetProperty(Consts.RedialRegexExpression, value);
}
/// <summary>
src/DotnetSpider/Infrastructure/RequestedQueue.cs
+1 -6
View File
@@ -4,7 +4,6 @@ using System.Runtime.CompilerServices;
using System.Threading.Tasks;
using DotnetSpider.Http;
using HWT;
using Microsoft.Extensions.Logging;
namespace DotnetSpider.Infrastructure
{
@@ -18,14 +17,12 @@ namespace DotnetSpider.Infrastructure
private readonly List<Request> _queue;
private readonly SpiderOptions _options;
private readonly ILogger _logger;
public RequestedQueue(SpiderOptions options, ILogger logger)
public RequestedQueue(SpiderOptions options)
{
_dict = new Dictionary<string, Request>();
_queue = new List<Request>();
_options = options;
_logger = logger;
}
public int Count => _dict.Count;
@@ -36,7 +33,6 @@ namespace DotnetSpider.Infrastructure
if (!_dict.ContainsKey(request.Hash))
{
_dict.Add(request.Hash, request);
_logger.LogInformation($"Start {request.Hash} timer at {DateTime.Now:yyyyj-MM-dd HH:mm:ss}");
_timer.NewTimeout(new TimeoutTask(this, request.Hash),
TimeSpan.FromSeconds(_options.RequestTimeout));
return true;
@@ -67,7 +63,6 @@ namespace DotnetSpider.Infrastructure
if (_dict.ContainsKey(hash))
{
var request = _dict[hash];
_logger.LogInformation($"Timeout {request.Hash} timer at {DateTime.Now:yyyyj-MM-dd HH:mm:ss}");
_queue.Add(request);
_dict.Remove(hash);
}
src/DotnetSpider/Spider.cs
+27 -27
View File
@@ -57,7 +57,7 @@ namespace DotnetSpider
Logger = logger;
_services = services;
Options = options.Value;
_requestedQueue = new RequestedQueue(Options, logger);
_requestedQueue = new RequestedQueue(Options);
_requestSuppliers = new List<IRequestSupplier>();
_dataFlows = new List<IDataFlow>();
}
@@ -117,7 +117,7 @@ namespace DotnetSpider
public override async Task StopAsync(CancellationToken cancellationToken)
{
Logger.LogInformation($"{Id}, {Name} stopping");
Logger.LogInformation($"{Id} stopping");
_consumer?.Close();
await base.StopAsync(cancellationToken);
@@ -126,8 +126,8 @@ namespace DotnetSpider
{
dataFlow.Dispose();
}
Logger.LogInformation($"{Id}, {Name} stopped");
// _services.MessageQueue.
Logger.LogInformation($"{Id} stopped");
}
protected virtual void ConfigureRequest(Request request)
@@ -179,10 +179,10 @@ namespace DotnetSpider
foreach (var request in requests)
{
if (request.DownloaderType.Contains("ADSL") &&
string.IsNullOrWhiteSpace(request.GetHeader(Consts.RedialRegExp)))
string.IsNullOrWhiteSpace(request.GetHeader(Consts.RedialRegexExpression)))
{
throw new ArgumentException(
$"Request {request.RequestUri} set to use ADSL but RedialEgeExp is empty");
$"Request {request.RequestUri}, {request.Hash} set to use ADSL but RedialRegExp is empty");
}
request.RequestedTimes += 1;
@@ -221,7 +221,7 @@ namespace DotnetSpider
Id = tuple.Id;
Name = tuple.Name;
Logger.LogInformation($"Initialize {Id}, {Name}");
Logger.LogInformation($"Initialize {Id}");
await _services.StatisticsClient.StartAsync(Id, Name);
await InitializeAsync(stoppingToken);
await LoadRequestFromSuppliers(stoppingToken);
@@ -229,27 +229,27 @@ namespace DotnetSpider
await _services.StatisticsClient.IncreaseTotalAsync(Id, _services.Scheduler.Total);
await RegisterConsumerAsync(stoppingToken);
await RunAsync(stoppingToken);
Logger.LogInformation($"{Id}, {Name} started");
Logger.LogInformation($"{Id} started");
}
private async Task RegisterConsumerAsync(CancellationToken stoppingToken)
{
var topic = string.Format(TopicNames.Spider, Id.ToUpper());
Logger.LogInformation($"Register topic {topic}");
Logger.LogInformation($"{Id} register topic {topic}");
_consumer = new AsyncMessageConsumer<byte[]>(topic);
_consumer.Received += async bytes =>
{
var message = await bytes.DeserializeAsync(stoppingToken);
if (message == null)
{
Logger.LogWarning("Received empty message");
Logger.LogWarning("{Id} received empty message");
return;
}
if (message is Exit exit)
{
Logger.LogInformation($"Receive exit message {JsonConvert.SerializeObject(exit)}");
Logger.LogInformation($"{Id} receive exit message {JsonConvert.SerializeObject(exit)}");
if (exit.Id == Id.ToUpper())
{
await ExitAsync();
@@ -261,7 +261,7 @@ namespace DotnetSpider
var request = _requestedQueue.Dequeue(response.RequestHash);
if (request == null)
{
Logger.LogWarning($"Dequeue {response.RequestHash} failed");
Logger.LogWarning($"{Id} dequeue {response.RequestHash} failed");
}
else
{
@@ -269,7 +269,7 @@ namespace DotnetSpider
{
if (_services.IsDistributed)
{
Logger.LogInformation($"download {request.RequestUri} success");
Logger.LogInformation($"{Id} download {request.RequestUri}, {request.Hash} success");
}
request.Agent = response.Agent;
@@ -284,7 +284,7 @@ namespace DotnetSpider
var exception = Encoding.UTF8.GetString(response.Content.Data);
if (_services.IsDistributed)
{
Logger.LogError($"download {request.RequestUri} failed: {exception}");
Logger.LogError($"{Id} download {request.RequestUri}, {request.Hash} failed: {exception}");
}
// 每次调用添加会导致 Requested + 1, 因此失败多次的请求最终会被过滤不再加到调度队列
@@ -294,7 +294,7 @@ namespace DotnetSpider
}
else
{
Logger.LogError($"Receive error message {JsonConvert.SerializeObject(message)}");
Logger.LogError($"{Id} receive error message {JsonConvert.SerializeObject(message)}");
}
};
await _services.MessageQueue.ConsumeAsync(_consumer, stoppingToken);
@@ -322,7 +322,7 @@ namespace DotnetSpider
// if download correct content, parser or storage failed by network or something else
// retry it until trigger retryTimes limitation
await AddRequestsAsync(request);
Logger.LogError($"Handle {JsonConvert.SerializeObject(request)} failed: {e}");
Logger.LogError($"{Id} handle {JsonConvert.SerializeObject(request)} failed: {e}");
}
}
@@ -353,13 +353,13 @@ namespace DotnetSpider
if (pausedTime > sleepTimeLimit)
{
Logger.LogInformation(
$"{Id}, {Name} paused too much time");
$"{Id} paused too much time");
break;
}
pausedTime += tuple.Interval;
Logger.LogInformation(
$"{Id}, {Name} too much requests enqueued");
$"{Id} too much requests enqueued");
await Task.Delay(tuple.Interval, default);
continue;
}
@@ -371,7 +371,7 @@ namespace DotnetSpider
foreach (var request in timeoutRequests)
{
Logger.LogWarning(
$"{Id}, {Name} request {request.RequestUri} timeout");
$"{Id} request {request.RequestUri}, {request.Hash} timeout");
}
await AddRequestsAsync(timeoutRequests);
@@ -411,7 +411,7 @@ namespace DotnetSpider
}
catch (Exception e)
{
Logger.LogError($"{Id}, {Name} exited by exception: {e}");
Logger.LogError($"{Id} exited by exception: {e}");
}
finally
{
@@ -426,7 +426,7 @@ namespace DotnetSpider
_services.ApplicationLifetime.StopApplication();
}
private (int Interval, int Batch) ComputeIntervalAndDequeueBatch(double speed)
private static (int Interval, int Batch) ComputeIntervalAndDequeueBatch(double speed)
{
if (speed >= 1)
{
@@ -453,7 +453,7 @@ namespace DotnetSpider
var proxy = await _services.ProxyPool.GetAsync(70);
if (proxy == null)
{
Logger.LogError("Exit because there is no available proxy");
Logger.LogError("{Id} exit because there is no available proxy");
return false;
}
@@ -498,7 +498,7 @@ namespace DotnetSpider
}
else
{
Logger.LogWarning($"Enqueue request: {request.RequestUri} failed");
Logger.LogWarning($"{Id} enqueue request: {request.RequestUri}, {request.Hash} failed");
}
}
}
@@ -517,7 +517,7 @@ namespace DotnetSpider
}
Logger.LogInformation(
$"{Id}, {Name} load request from {requestSupplier.GetType().Name} {_requestSuppliers.IndexOf(requestSupplier)}/{_requestSuppliers.Count}");
$"{Id} load request from {requestSupplier.GetType().Name} {_requestSuppliers.IndexOf(requestSupplier)}/{_requestSuppliers.Count}");
}
}
@@ -525,12 +525,12 @@ namespace DotnetSpider
{
if (_dataFlows.Count == 0)
{
Logger.LogWarning("There is no any dataFlow");
Logger.LogWarning("{Id} there is no any dataFlow");
}
else
{
var dataFlowInfo = string.Join(" -> ", _dataFlows.Select(x => x.GetType().Name));
Logger.LogInformation($"{Id}, {Name} DataFlows: {dataFlowInfo}");
Logger.LogInformation($"{Id} DataFlows: {dataFlowInfo}");
foreach (var dataFlow in _dataFlows)
{
dataFlow.SetLogger(Logger);
@@ -540,7 +540,7 @@ namespace DotnetSpider
}
catch (Exception e)
{
Logger.LogError($"Initialize dataFlow {dataFlow.GetType().Name} failed: {e}");
Logger.LogError($"{Id} initialize dataFlow {dataFlow.GetType().Name} failed: {e}");
_services.ApplicationLifetime.StopApplication();
}
}