diff --git a/src/DotnetSpider.Mongo/DotnetSpider.Mongo.csproj b/src/DotnetSpider.Mongo/DotnetSpider.Mongo.csproj index f2c56cbb..b67a5fa5 100644 --- a/src/DotnetSpider.Mongo/DotnetSpider.Mongo.csproj +++ b/src/DotnetSpider.Mongo/DotnetSpider.Mongo.csproj @@ -6,7 +6,7 @@ - + diff --git a/src/DotnetSpider.MySql/DotnetSpider.MySql.csproj b/src/DotnetSpider.MySql/DotnetSpider.MySql.csproj index 72b7ae05..772eabb1 100644 --- a/src/DotnetSpider.MySql/DotnetSpider.MySql.csproj +++ b/src/DotnetSpider.MySql/DotnetSpider.MySql.csproj @@ -10,6 +10,6 @@ - + diff --git a/src/DotnetSpider.Portal/DotnetSpider.Portal.csproj b/src/DotnetSpider.Portal/DotnetSpider.Portal.csproj index d7c9c93f..d51fe8cf 100644 --- a/src/DotnetSpider.Portal/DotnetSpider.Portal.csproj +++ b/src/DotnetSpider.Portal/DotnetSpider.Portal.csproj @@ -14,7 +14,7 @@ - + diff --git a/src/DotnetSpider.Sample/DotnetSpider.Sample.csproj b/src/DotnetSpider.Sample/DotnetSpider.Sample.csproj index e6f92eb1..1e15e40c 100644 --- a/src/DotnetSpider.Sample/DotnetSpider.Sample.csproj +++ b/src/DotnetSpider.Sample/DotnetSpider.Sample.csproj @@ -18,7 +18,6 @@ - diff --git a/src/DotnetSpider.Sample/Program.cs b/src/DotnetSpider.Sample/Program.cs index 120af7e9..b81c61fa 100644 --- a/src/DotnetSpider.Sample/Program.cs +++ b/src/DotnetSpider.Sample/Program.cs @@ -33,7 +33,7 @@ namespace DotnetSpider.Sample .WriteTo.Console().WriteTo.RollingFile("logs/spider.txt") .CreateLogger(); - await JsonEntitySpider.RunAsync(); + await BaseUsageSpider.RunAsync(); Console.WriteLine("Bye!"); Environment.Exit(0); diff --git a/src/DotnetSpider.Tests/DotnetSpider.Tests.csproj b/src/DotnetSpider.Tests/DotnetSpider.Tests.csproj index 89f54f06..411cb4d5 100644 --- a/src/DotnetSpider.Tests/DotnetSpider.Tests.csproj +++ b/src/DotnetSpider.Tests/DotnetSpider.Tests.csproj @@ -7,12 +7,12 @@ - + - + diff --git a/src/DotnetSpider/Agent/AgentService.cs b/src/DotnetSpider/Agent/AgentService.cs index 88a03823..a036d945 100644 --- a/src/DotnetSpider/Agent/AgentService.cs +++ b/src/DotnetSpider/Agent/AgentService.cs @@ -134,7 +134,7 @@ namespace DotnetSpider.Agent response.Agent = _options.AgentId; await _messageQueue.PublishAsBytesAsync(string.Format(TopicNames.Spider, request.Owner.ToUpper()), response); - _logger.LogInformation($"{request.RequestUri} download success"); + _logger.LogInformation($"{request.Owner} download {request.RequestUri}, {request.Hash} success"); } else { diff --git a/src/DotnetSpider/Agent/PPPoEService.cs b/src/DotnetSpider/Agent/PPPoEService.cs index 0d43db13..af8eea7c 100644 --- a/src/DotnetSpider/Agent/PPPoEService.cs +++ b/src/DotnetSpider/Agent/PPPoEService.cs @@ -32,7 +32,7 @@ namespace DotnetSpider.Agent /// public async Task DetectAsync(Request request, HttpResponseMessage httpResponseMessage) { - var redialRegExp = request.GetHeader(Consts.RedialRegExp); + var redialRegExp = request.GetHeader(Consts.RedialRegexExpression); if (IsActive && !string.IsNullOrWhiteSpace(redialRegExp)) { var text = await httpResponseMessage.Content.ReadAsStringAsync(); @@ -127,4 +127,4 @@ namespace DotnetSpider.Agent } } } -} \ No newline at end of file +} diff --git a/src/DotnetSpider/Consts.cs b/src/DotnetSpider/Consts.cs index d557e881..a3afd582 100644 --- a/src/DotnetSpider/Consts.cs +++ b/src/DotnetSpider/Consts.cs @@ -3,7 +3,7 @@ namespace DotnetSpider public static class Consts { public const string ProxyPrefix = "PROXY_"; - public const string RedialRegExp = "DOTNET_SPIDER_REDIAL_REGEXP"; + public const string RedialRegexExpression = "DOTNET_SPIDER_REDIAL_REGEXP"; public const string ResponseBytes = "DOTNET_SPIDER_RESPONSE_BYTES"; } -} \ No newline at end of file +} diff --git a/src/DotnetSpider/Http/Request.cs b/src/DotnetSpider/Http/Request.cs index 2eea181a..d9c73626 100644 --- a/src/DotnetSpider/Http/Request.cs +++ b/src/DotnetSpider/Http/Request.cs @@ -87,8 +87,8 @@ namespace DotnetSpider.Http [IgnoreMember] public string RedialRegExp { - get => GetProperty(Consts.RedialRegExp); - set => SetProperty(Consts.RedialRegExp, value); + get => GetProperty(Consts.RedialRegexExpression); + set => SetProperty(Consts.RedialRegexExpression, value); } /// diff --git a/src/DotnetSpider/Infrastructure/RequestedQueue.cs b/src/DotnetSpider/Infrastructure/RequestedQueue.cs index a197fae7..80c5416b 100644 --- a/src/DotnetSpider/Infrastructure/RequestedQueue.cs +++ b/src/DotnetSpider/Infrastructure/RequestedQueue.cs @@ -4,7 +4,6 @@ using System.Runtime.CompilerServices; using System.Threading.Tasks; using DotnetSpider.Http; using HWT; -using Microsoft.Extensions.Logging; namespace DotnetSpider.Infrastructure { @@ -18,14 +17,12 @@ namespace DotnetSpider.Infrastructure private readonly List _queue; private readonly SpiderOptions _options; - private readonly ILogger _logger; - public RequestedQueue(SpiderOptions options, ILogger logger) + public RequestedQueue(SpiderOptions options) { _dict = new Dictionary(); _queue = new List(); _options = options; - _logger = logger; } public int Count => _dict.Count; @@ -36,7 +33,6 @@ namespace DotnetSpider.Infrastructure if (!_dict.ContainsKey(request.Hash)) { _dict.Add(request.Hash, request); - _logger.LogInformation($"Start {request.Hash} timer at {DateTime.Now:yyyyj-MM-dd HH:mm:ss}"); _timer.NewTimeout(new TimeoutTask(this, request.Hash), TimeSpan.FromSeconds(_options.RequestTimeout)); return true; @@ -67,7 +63,6 @@ namespace DotnetSpider.Infrastructure if (_dict.ContainsKey(hash)) { var request = _dict[hash]; - _logger.LogInformation($"Timeout {request.Hash} timer at {DateTime.Now:yyyyj-MM-dd HH:mm:ss}"); _queue.Add(request); _dict.Remove(hash); } diff --git a/src/DotnetSpider/Spider.cs b/src/DotnetSpider/Spider.cs index a7d43204..c1d60ee8 100644 --- a/src/DotnetSpider/Spider.cs +++ b/src/DotnetSpider/Spider.cs @@ -57,7 +57,7 @@ namespace DotnetSpider Logger = logger; _services = services; Options = options.Value; - _requestedQueue = new RequestedQueue(Options, logger); + _requestedQueue = new RequestedQueue(Options); _requestSuppliers = new List(); _dataFlows = new List(); } @@ -117,7 +117,7 @@ namespace DotnetSpider public override async Task StopAsync(CancellationToken cancellationToken) { - Logger.LogInformation($"{Id}, {Name} stopping"); + Logger.LogInformation($"{Id} stopping"); _consumer?.Close(); await base.StopAsync(cancellationToken); @@ -126,8 +126,8 @@ namespace DotnetSpider { dataFlow.Dispose(); } - - Logger.LogInformation($"{Id}, {Name} stopped"); + // _services.MessageQueue. + Logger.LogInformation($"{Id} stopped"); } protected virtual void ConfigureRequest(Request request) @@ -179,10 +179,10 @@ namespace DotnetSpider foreach (var request in requests) { if (request.DownloaderType.Contains("ADSL") && - string.IsNullOrWhiteSpace(request.GetHeader(Consts.RedialRegExp))) + string.IsNullOrWhiteSpace(request.GetHeader(Consts.RedialRegexExpression))) { throw new ArgumentException( - $"Request {request.RequestUri} set to use ADSL but RedialEgeExp is empty"); + $"Request {request.RequestUri}, {request.Hash} set to use ADSL but RedialRegExp is empty"); } request.RequestedTimes += 1; @@ -221,7 +221,7 @@ namespace DotnetSpider Id = tuple.Id; Name = tuple.Name; - Logger.LogInformation($"Initialize {Id}, {Name}"); + Logger.LogInformation($"Initialize {Id}"); await _services.StatisticsClient.StartAsync(Id, Name); await InitializeAsync(stoppingToken); await LoadRequestFromSuppliers(stoppingToken); @@ -229,27 +229,27 @@ namespace DotnetSpider await _services.StatisticsClient.IncreaseTotalAsync(Id, _services.Scheduler.Total); await RegisterConsumerAsync(stoppingToken); await RunAsync(stoppingToken); - Logger.LogInformation($"{Id}, {Name} started"); + Logger.LogInformation($"{Id} started"); } private async Task RegisterConsumerAsync(CancellationToken stoppingToken) { var topic = string.Format(TopicNames.Spider, Id.ToUpper()); - Logger.LogInformation($"Register topic {topic}"); + Logger.LogInformation($"{Id} register topic {topic}"); _consumer = new AsyncMessageConsumer(topic); _consumer.Received += async bytes => { var message = await bytes.DeserializeAsync(stoppingToken); if (message == null) { - Logger.LogWarning("Received empty message"); + Logger.LogWarning("{Id} received empty message"); return; } if (message is Exit exit) { - Logger.LogInformation($"Receive exit message {JsonConvert.SerializeObject(exit)}"); + Logger.LogInformation($"{Id} receive exit message {JsonConvert.SerializeObject(exit)}"); if (exit.Id == Id.ToUpper()) { await ExitAsync(); @@ -261,7 +261,7 @@ namespace DotnetSpider var request = _requestedQueue.Dequeue(response.RequestHash); if (request == null) { - Logger.LogWarning($"Dequeue {response.RequestHash} failed"); + Logger.LogWarning($"{Id} dequeue {response.RequestHash} failed"); } else { @@ -269,7 +269,7 @@ namespace DotnetSpider { if (_services.IsDistributed) { - Logger.LogInformation($"download {request.RequestUri} success"); + Logger.LogInformation($"{Id} download {request.RequestUri}, {request.Hash} success"); } request.Agent = response.Agent; @@ -284,7 +284,7 @@ namespace DotnetSpider var exception = Encoding.UTF8.GetString(response.Content.Data); if (_services.IsDistributed) { - Logger.LogError($"download {request.RequestUri} failed: {exception}"); + Logger.LogError($"{Id} download {request.RequestUri}, {request.Hash} failed: {exception}"); } // 每次调用添加会导致 Requested + 1, 因此失败多次的请求最终会被过滤不再加到调度队列 @@ -294,7 +294,7 @@ namespace DotnetSpider } else { - Logger.LogError($"Receive error message {JsonConvert.SerializeObject(message)}"); + Logger.LogError($"{Id} receive error message {JsonConvert.SerializeObject(message)}"); } }; await _services.MessageQueue.ConsumeAsync(_consumer, stoppingToken); @@ -322,7 +322,7 @@ namespace DotnetSpider // if download correct content, parser or storage failed by network or something else // retry it until trigger retryTimes limitation await AddRequestsAsync(request); - Logger.LogError($"Handle {JsonConvert.SerializeObject(request)} failed: {e}"); + Logger.LogError($"{Id} handle {JsonConvert.SerializeObject(request)} failed: {e}"); } } @@ -353,13 +353,13 @@ namespace DotnetSpider if (pausedTime > sleepTimeLimit) { Logger.LogInformation( - $"{Id}, {Name} paused too much time"); + $"{Id} paused too much time"); break; } pausedTime += tuple.Interval; Logger.LogInformation( - $"{Id}, {Name} too much requests enqueued"); + $"{Id} too much requests enqueued"); await Task.Delay(tuple.Interval, default); continue; } @@ -371,7 +371,7 @@ namespace DotnetSpider foreach (var request in timeoutRequests) { Logger.LogWarning( - $"{Id}, {Name} request {request.RequestUri} timeout"); + $"{Id} request {request.RequestUri}, {request.Hash} timeout"); } await AddRequestsAsync(timeoutRequests); @@ -411,7 +411,7 @@ namespace DotnetSpider } catch (Exception e) { - Logger.LogError($"{Id}, {Name} exited by exception: {e}"); + Logger.LogError($"{Id} exited by exception: {e}"); } finally { @@ -426,7 +426,7 @@ namespace DotnetSpider _services.ApplicationLifetime.StopApplication(); } - private (int Interval, int Batch) ComputeIntervalAndDequeueBatch(double speed) + private static (int Interval, int Batch) ComputeIntervalAndDequeueBatch(double speed) { if (speed >= 1) { @@ -453,7 +453,7 @@ namespace DotnetSpider var proxy = await _services.ProxyPool.GetAsync(70); if (proxy == null) { - Logger.LogError("Exit because there is no available proxy"); + Logger.LogError("{Id} exit because there is no available proxy"); return false; } @@ -498,7 +498,7 @@ namespace DotnetSpider } else { - Logger.LogWarning($"Enqueue request: {request.RequestUri} failed"); + Logger.LogWarning($"{Id} enqueue request: {request.RequestUri}, {request.Hash} failed"); } } } @@ -517,7 +517,7 @@ namespace DotnetSpider } Logger.LogInformation( - $"{Id}, {Name} load request from {requestSupplier.GetType().Name} {_requestSuppliers.IndexOf(requestSupplier)}/{_requestSuppliers.Count}"); + $"{Id} load request from {requestSupplier.GetType().Name} {_requestSuppliers.IndexOf(requestSupplier)}/{_requestSuppliers.Count}"); } } @@ -525,12 +525,12 @@ namespace DotnetSpider { if (_dataFlows.Count == 0) { - Logger.LogWarning("There is no any dataFlow"); + Logger.LogWarning("{Id} there is no any dataFlow"); } else { var dataFlowInfo = string.Join(" -> ", _dataFlows.Select(x => x.GetType().Name)); - Logger.LogInformation($"{Id}, {Name} DataFlows: {dataFlowInfo}"); + Logger.LogInformation($"{Id} DataFlows: {dataFlowInfo}"); foreach (var dataFlow in _dataFlows) { dataFlow.SetLogger(Logger); @@ -540,7 +540,7 @@ namespace DotnetSpider } catch (Exception e) { - Logger.LogError($"Initialize dataFlow {dataFlow.GetType().Name} failed: {e}"); + Logger.LogError($"{Id} initialize dataFlow {dataFlow.GetType().Name} failed: {e}"); _services.ApplicationLifetime.StopApplication(); } }