GithubStar/FastGateway
1
0
Fork 0
mirror of https://github.com/239573049/FastGateway.git synced 2026-10-17 05:20:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

优化性能

Browse Source
This commit is contained in:
token
2024-11-26 01:51:50 +08:00
parent cd03f9ca7b
commit 74139bb738
11 changed files with 135 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/FastGateway.Service/BackgroundTask/ClientRequestBackgroundTask.cs
+5 -21
View File
@@ -1,5 +1,4 @@
using System.Collections.Concurrent;
using System.Threading.Channels;
using FastGateway.Entities;
using FastGateway.Service.DataAccess;
using FastGateway.Service.Dto;
@@ -10,47 +9,40 @@ namespace FastGateway.Service.BackgroundTask;
public class ClientRequestBackgroundTask(IServiceProvider serviceProvider, ISearcher searcher) : BackgroundService
{
private static Channel<ClientRequestLoggerInput> ClientRequestChannel { get; } =
Channel.CreateUnbounded<ClientRequestLoggerInput>();
/// <summary>
/// 线程安全集合
/// </summary>
private readonly ConcurrentBag<ClientRequestLoggerInput> _loggerBag = new();
private static readonly ConcurrentBag<ClientRequestLoggerInput> LoggerBag = new();
public static void AddLogger(ClientRequestLoggerInput input)
{
ClientRequestChannel.Writer.TryWrite(input);
LoggerBag.Add(input);
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
// 暂停1分钟
await Task.Delay(1000 * 60, stoppingToken);
_ = Start(stoppingToken);
await RunLoggerSave(stoppingToken);
}
private async Task RunLoggerSave(CancellationToken stoppingToken)
{
await using var scope = serviceProvider.CreateAsyncScope();
var masterContext = scope.ServiceProvider.GetRequiredService<MasterContext>();
var loggerContext = scope.ServiceProvider.GetRequiredService<LoggerContext>();
var logger = scope.ServiceProvider.GetRequiredService<ILogger<LoggerBackgroundTask>>();
while (!stoppingToken.IsCancellationRequested)
{
if (_loggerBag.IsEmpty)
if (LoggerBag.IsEmpty)
{
// 等待5s
await Task.Delay(10000, stoppingToken);
continue;
}
var loggerList = new List<ClientRequestLoggerInput>();
while (_loggerBag.TryTake(out var loggerItem))
while (LoggerBag.TryTake(out var loggerItem))
{
loggerList.Add(loggerItem);
}
@@ -126,7 +118,7 @@ public class ClientRequestBackgroundTask(IServiceProvider serviceProvider, ISear
foreach (var item in updateLoggerList)
{
await loggerContext.ClientRequestLoggers.Where(x => x.RequestTime == now && x.Ip == item.Ip)
await loggerContext.ClientRequestLoggers.Where(x => x.Ip == item.Ip && x.RequestTime == now)
.ExecuteUpdateAsync(x => x.SetProperty(a => a.Total, a => a.Total + item.Total)
.SetProperty(a => a.Success, a => a.Success + item.Success)
.SetProperty(a => a.Fail, a => a.Fail),
@@ -143,12 +135,4 @@ public class ClientRequestBackgroundTask(IServiceProvider serviceProvider, ISear
}
}
}
private async Task Start(CancellationToken stoppingToken)
{
await foreach (var item in ClientRequestChannel.Reader.ReadAllAsync(stoppingToken))
{
_loggerBag.Add(item);
}
}
}
src/FastGateway.Service/BackgroundTask/LoggerBackgroundTask.cs
+22 -42
View File
@@ -1,5 +1,4 @@
using System.Collections.Concurrent;
using System.Threading.Channels;
using System.Threading.Channels;
using FastGateway.Entities;
using FastGateway.Service.DataAccess;
using IP2Region.Net.Abstractions;
@@ -8,52 +7,36 @@ namespace FastGateway.Service.BackgroundTask;
public sealed class LoggerBackgroundTask(IServiceProvider serviceProvider, ISearcher searcher) : BackgroundService
{
private static Channel<ApplicationLogger> LoggerChannel { get; } = Channel.CreateUnbounded<ApplicationLogger>();
public static void AddLogger(ApplicationLogger logger)
{
LoggerChannel.Writer.TryWrite(logger);
}
/// <summary>
/// 线程安全集合
/// </summary>
private readonly ConcurrentBag<ApplicationLogger> _loggerBag = new();
private static readonly Channel<ApplicationLogger> LoggerBag = Channel.CreateUnbounded<ApplicationLogger>();
public static void AddLogger(ApplicationLogger logger)
{
LoggerBag.Writer.TryWrite(logger);
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
// 暂停1分钟
_ = Start(stoppingToken);
await RunLoggerSave(stoppingToken);
}
private async Task RunLoggerSave(CancellationToken stoppingToken)
{
await using var scope = serviceProvider.CreateAsyncScope();
var masterContext = scope.ServiceProvider.GetRequiredService<MasterContext>();
var loggerContext = scope.ServiceProvider.GetRequiredService<LoggerContext>();
var logger = scope.ServiceProvider.GetRequiredService<ILogger<LoggerBackgroundTask>>();
await using var loggerContext = scope.ServiceProvider.GetRequiredService<LoggerContext>();
while (!stoppingToken.IsCancellationRequested)
{
if (_loggerBag.IsEmpty)
{
// 等待5s
await Task.Delay(5000, stoppingToken);
continue;
}
var loggerList = new List<ApplicationLogger>();
while (_loggerBag.TryTake(out var loggerItem))
{
loggerList.Add(loggerItem);
}
try
{
foreach (var item in loggerList)
int count = 0;
while (LoggerBag.Reader.TryRead(out var item))
{
if (string.IsNullOrWhiteSpace(item.Ip)) continue;
if (string.IsNullOrWhiteSpace(item.Ip))
continue;
var locations = searcher.Search(item.Ip)?.Split("|", StringSplitOptions.RemoveEmptyEntries);
locations = locations?.Where(x => x != "0").ToArray();
@@ -69,23 +52,20 @@ public sealed class LoggerBackgroundTask(IServiceProvider serviceProvider, ISear
.Replace("移动", "")
.TrimStart('|')
.TrimEnd('|');
}
await loggerContext.ApplicationLoggers.AddRangeAsync(loggerList, stoppingToken);
await loggerContext.SaveChangesAsync(stoppingToken);
await loggerContext.ApplicationLoggers.AddAsync(item, stoppingToken);
count++;
if (count >= 100)
{
await loggerContext.SaveChangesAsync(stoppingToken);
}
}
}
catch (Exception e)
{
logger.LogError(e, "日志保存失败");
Console.WriteLine(e);
await Task.Delay(500, stoppingToken);
}
}
}
private async Task Start(CancellationToken stoppingToken)
{
await foreach (var item in LoggerChannel.Reader.ReadAllAsync(stoppingToken))
{
_loggerBag.Add(item);
}
}
}
src/FastGateway.Service/BackgroundTask/RenewSSLBackgroundService.cs
+9 -2
View File
@@ -10,7 +10,7 @@ public class RenewSSLBackgroundService(ILogger<RenewSSLBackgroundService> logger
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
logger.LogInformation("证书自动续期服务已启动");
// 暂停1分钟
await Task.Delay(1000 * 60, stoppingToken);
@@ -27,6 +27,8 @@ public class RenewSSLBackgroundService(ILogger<RenewSSLBackgroundService> logger
.Where(x => x.NotAfter == null || x.NotAfter < DateTime.Now.AddDays(15) && x.AutoRenew)
.ToArrayAsync(stoppingToken);
var isRenew = false;
foreach (var cert in certs)
{
try
@@ -43,6 +45,8 @@ public class RenewSSLBackgroundService(ILogger<RenewSSLBackgroundService> logger
.SetProperty(y => y.Certs, cert.Certs), stoppingToken);
logger.LogInformation($"证书续期成功：{cert.Id} {cert.Domain}");
isRenew = true;
}
catch (Exception e)
{
@@ -57,7 +61,10 @@ public class RenewSSLBackgroundService(ILogger<RenewSSLBackgroundService> logger
}
}
CertService.InitCert(await dbContext.Certs.ToListAsync(cancellationToken: stoppingToken));
if (isRenew)
{
CertService.InitCert(await dbContext.Certs.ToArrayAsync(cancellationToken: stoppingToken));
}
// 每24小时检查一次
await Task.Delay(1000 * 60 * 60 * 24, stoppingToken);
src/FastGateway.Service/Gateway/Gateway.cs
+4 -5
View File
@@ -47,7 +47,7 @@ public static class Gateway
/// </summary>
/// <param name="server"></param>
/// <param name="domainNames"></param>
public static void ReloadGateway(Server server, List<DomainName> domainNames)
public static void ReloadGateway(Server server, DomainName[] domainNames)
{
if (GatewayWebApplications.TryGetValue(server.Id, out var webApplication))
{
@@ -107,8 +107,8 @@ public static class Gateway
/// <summary>
/// 构建网关
/// </summary>
public static async Task BuilderGateway(Server server, List<DomainName> domainNames,
List<BlacklistAndWhitelist> blacklistAndWhitelists, List<RateLimit> rateLimits)
public static async Task BuilderGateway(Server server, DomainName[] domainNames,
BlacklistAndWhitelist[] blacklistAndWhitelists, RateLimit[] rateLimits)
{
try
{
@@ -339,12 +339,11 @@ public static class Gateway
}
private static (IReadOnlyList<RouteConfig> routes,
IReadOnlyList<ClusterConfig> clusters) BuildConfig(List<DomainName> domainNames)
IReadOnlyList<ClusterConfig> clusters) BuildConfig(DomainName[] domainNames)
{
var routes = new List<RouteConfig>();
var clusters = new List<ClusterConfig>();
foreach (var domainName in domainNames)
{
if (string.IsNullOrWhiteSpace(domainName.Path))
src/FastGateway.Service/Infrastructure/ISystemUsage.cs
+52 -30
View File
@@ -130,41 +130,47 @@ public class LinuxSystemUsage : ISystemUsage
}
public (float memoryUsage, ulong totalMemory, ulong useMemory) GetMemoryUsage()
{
var memInfo = ExecuteCommand("cat /proc/meminfo", "/bin/bash");
var lines = memInfo.Split('\n');
ulong totalMemory = 0;
ulong availableMemory = 0;
{
var memInfo = ExecuteCommand("cat /proc/meminfo", "/bin/bash");
var lines = memInfo.Split('\n');
ulong totalMemory = 0;
ulong availableMemory = 0;
foreach (var line in lines)
{
var parts = line.Split(new[] { ':' }, StringSplitOptions.RemoveEmptyEntries);
if (parts.Length < 2) continue;
foreach (var line in lines)
{
var parts = line.Split(new[] { ':' }, StringSplitOptions.RemoveEmptyEntries);
if (parts.Length < 2) continue;
var key = parts[0].Trim();
var value = ulong.Parse(parts[1].Trim().Split(' ')[0]);
var key = parts[0].Trim();
var value = ulong.Parse(parts[1].Trim().Split(' ')[0]);
switch (key)
{
case "MemTotal":
totalMemory = value;
break;
case "MemAvailable":
availableMemory = value;
break;
}
}
switch (key)
{
case "MemTotal":
totalMemory = value;
break;
case "MemAvailable":
availableMemory = value;
break;
}
}
// 默认单位是KB，需要转换byte
totalMemory *= 1024;
availableMemory *= 1024;
// 默认单位是KB，需要转换byte
totalMemory *= 1024;
availableMemory *= 1024;
ulong usedMemory = totalMemory - availableMemory;
float memoryUsage = (float)usedMemory / totalMemory * 100;
ulong usedMemory = totalMemory - availableMemory;
float memoryUsage = (float)usedMemory / totalMemory * 100;
return (memoryUsage, totalMemory, usedMemory);
}
return (memoryUsage, totalMemory, usedMemory);
}
private long previousReads = 0;
private long previousSectorsRead = 0;
private long previousWrites = 0;
private long previousSectorsWritten = 0;
public (float read, float write) GetDiskUsage()
{
@@ -178,8 +184,24 @@ public class LinuxSystemUsage : ISystemUsage
var parts = line.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
if (parts.Length > 13)
{
readBytes += float.Parse(parts[5]) * 512 / 1024; // sectors read * 512 bytes per sector to KB
writeBytes += float.Parse(parts[9]) * 512 / 1024; // sectors written * 512 bytes per sector to KB
string deviceName = parts[2]; // 设备名，通常是第三列
long readsCompleted = long.Parse(parts[3]); // 完成的读操作
long sectorsRead = long.Parse(parts[5]); // 读取的扇区数
long writesCompleted = long.Parse(parts[7]); // 完成的写操作
long sectorsWritten = long.Parse(parts[9]); // 写入的扇区数
long byteRead = (sectorsRead - previousSectorsRead) * 512; // 每个扇区512字节
long byteWritten = (sectorsWritten - previousSectorsWritten) * 512;
previousReads = readsCompleted;
previousSectorsRead = sectorsRead;
previousWrites = writesCompleted;
previousSectorsWritten = sectorsWritten;
if (deviceName == "sda")
{
readBytes += byteRead;
writeBytes += byteWritten;
}
}
}
src/FastGateway.Service/Program.cs
+7 -9
View File
@@ -76,6 +76,7 @@ public static class Program
builder.Services.AddHostedService<LoggerBackgroundTask>();
builder.Services.AddHostedService<RenewSSLBackgroundService>();
builder.Services.AddHostedService<ClientRequestBackgroundTask>();
builder.Services.AddHostedService<LogCleaningBackgroundService>();
builder.Services.AddDbContext<MasterContext>(optionsBuilder =>
{
// 判断当前目录是否存在data文件夹
@@ -109,18 +110,15 @@ public static class Program
await dbContext.Database.MigrateAsync();
await loggerContext.Database.MigrateAsync();
var certs = await dbContext.Certs.Where(x => x.Expired == false).ToListAsync();
CertService.InitCert(await dbContext.Certs.ToArrayAsync());
CertService.InitCert(certs);
var server = await dbContext.Servers.ToListAsync();
var domainNames = await dbContext.DomainNames.ToListAsync();
var blacklistAndWhitelists = await dbContext.BlacklistAndWhitelists.ToListAsync();
var rateLimits = await dbContext.RateLimits.ToListAsync();
foreach (var item in server)
var domainNames = await dbContext.DomainNames.ToArrayAsync();
var blacklistAndWhitelists = await dbContext.BlacklistAndWhitelists.ToArrayAsync();
var rateLimits = await dbContext.RateLimits.ToArrayAsync();
foreach (var item in await dbContext.Servers.ToArrayAsync())
{
await Task.Factory.StartNew(async () =>
await Gateway.Gateway.BuilderGateway(item, domainNames.Where(x => x.ServerId == item.Id).ToList(),
await Gateway.Gateway.BuilderGateway(item, domainNames.Where(x => x.ServerId == item.Id).ToArray(),
blacklistAndWhitelists, rateLimits));
}
}
src/FastGateway.Service/Services/BlacklistAndWhitelistService.cs
+1 -1
View File
@@ -10,7 +10,7 @@ namespace FastGateway.Service.Services;
public static class BlacklistAndWhitelistService
{
public static WebApplication UseBlacklistMiddleware(this WebApplication app,
List<BlacklistAndWhitelist> blacklistAndWhitelists)
BlacklistAndWhitelist[] blacklistAndWhitelists)
{
var whitelist = blacklistAndWhitelists.Where(x => x.Enable && !x.IsBlacklist).ToArray();
src/FastGateway.Service/Services/CertService.cs
+2 -2
View File
@@ -43,7 +43,7 @@ public static class CertService
return CertWebApplications.TryGetValue(domain, out var cert) ? cert : null;
}
public static void InitCert(List<Cert> certs)
public static void InitCert(Cert[] certs)
{
foreach (var cert in certs)
{
@@ -111,7 +111,7 @@ public static class CertService
.SetProperty(x => x.RenewTime, x => cert.RenewTime)
);
InitCert(await masterContext.Certs.Where(x => !x.Expired).ToListAsync());
InitCert(await masterContext.Certs.ToArrayAsync());
}
return ResultDto.CreateSuccess();
src/FastGateway.Service/Services/QpsService.cs
+5 -5
View File
@@ -31,7 +31,7 @@ public sealed class QpsItem
public static class QpsService
{
private static readonly QpsItem qpsItem = new();
private static readonly QpsItem QpsItem = new();
private static bool _enable;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -42,7 +42,7 @@ public static class QpsService
return;
}
qpsItem.IncrementRequests();
QpsItem.IncrementRequests();
}
public static void CalculateServiceQps()
@@ -52,14 +52,14 @@ public static class QpsService
return;
}
qpsItem.CalculateQps();
QpsItem.CalculateQps();
}
public static int GetServiceQps()
{
var qps = qpsItem.GetQps();
var qps = QpsItem.GetQps();
qpsItem.CalculateQps();
QpsItem.CalculateQps();
return qps;
}
src/FastGateway.Service/Services/RateLimitService.cs
+11 -13
View File
@@ -10,10 +10,10 @@ namespace FastGateway.Service.Services;
public static class RateLimitService
{
public static IServiceCollection AddRateLimitService(this IServiceCollection services, List<RateLimit> rateLimits)
public static IServiceCollection AddRateLimitService(this IServiceCollection services, RateLimit[] rateLimits)
{
// 如果启用限流则添加限流中间件
if (rateLimits.Count != 0)
if (rateLimits.Length != 0)
{
services.AddMemoryCache();
services.Configure<IpRateLimitOptions>
@@ -48,9 +48,9 @@ public static class RateLimitService
return services;
}
public static IEndpointRouteBuilder UseRateLimitMiddleware(this WebApplication app, List<RateLimit> rateLimits)
public static IEndpointRouteBuilder UseRateLimitMiddleware(this WebApplication app, RateLimit[] rateLimits)
{
if (rateLimits.Count == 0)
if (rateLimits.Length == 0)
{
return app;
}
@@ -69,33 +69,31 @@ public static class RateLimitService
.AddEndpointFilter<ResultFilter>()
.WithDisplayName("限流");
rateLimit.MapPost(string.Empty, async (MasterContext dbContext, RateLimit rateLimit) =>
rateLimit.MapPost(string.Empty, async (MasterContext dbContext, RateLimit limit) =>
{
if (string.IsNullOrWhiteSpace(rateLimit.Name))
if (string.IsNullOrWhiteSpace(limit.Name))
{
throw new ValidationException("限流名称不能为空");
}
if (await dbContext.RateLimits.AnyAsync(x => x.Name == rateLimit.Name))
if (await dbContext.RateLimits.AnyAsync(x => x.Name == limit.Name))
{
throw new ValidationException("限流名称已存在");
}
dbContext.RateLimits.Add(rateLimit);
dbContext.RateLimits.Add(limit);
await dbContext.SaveChangesAsync();
}).WithDescription("创建限流").WithDisplayName("创建限流").WithTags("限流");
rateLimit.MapGet(string.Empty, async (MasterContext dbContext, int page, int pageSize) =>
{
var result = await dbContext.RateLimits
.Skip((page - 1) * pageSize)
.ToListAsync();
var total = await dbContext.RateLimits.CountAsync();
return new PagingDto<RateLimit>(total, result);
return new PagingDto<RateLimit>(total, await dbContext.RateLimits
.Skip((page - 1) * pageSize)
.ToListAsync());
})
.WithDescription("获取限流列表")
.WithDisplayName("获取限流列表")
src/FastGateway.Service/Services/ServerService.cs
+17 -8
View File
@@ -75,6 +75,7 @@ public static class ServerService
dbContext.Servers.Update(server);
await dbContext.SaveChangesAsync();
}).WithDescription("更新服务").WithDisplayName("更新服务").WithTags("服务");
server.MapPut("{id}/enable", async (MasterContext dbContext, string id) =>
@@ -82,6 +83,7 @@ public static class ServerService
await dbContext.Servers
.Where(x => x.Id == id)
.ExecuteUpdateAsync(i => i.SetProperty(a => a.Enable, a => !a.Enable));
}).WithDescription("启用/禁用服务").WithDisplayName("启用/禁用服务").WithTags("服务");
// 启用服务
@@ -90,13 +92,22 @@ public static class ServerService
if (!Gateway.Gateway.CheckServerOnline(id))
{
var server = await dbContext.Servers.FirstOrDefaultAsync(x => x.Id == id);
var domainNames = await dbContext.DomainNames.Where(x => x.ServerId == id).ToListAsync();
var blacklistAndWhitelists = await dbContext.BlacklistAndWhitelists.ToListAsync();
var rateLimits = await dbContext.RateLimits.ToListAsync();
var domainNames = await dbContext.DomainNames.Where(x => x.ServerId == id).ToArrayAsync();
var blacklistAndWhitelists = await dbContext.BlacklistAndWhitelists.ToArrayAsync();
var rateLimits = await dbContext.RateLimits.ToArrayAsync();
await Task.Factory.StartNew(async () =>
await Gateway.Gateway.BuilderGateway(server, domainNames, blacklistAndWhitelists, rateLimits));
await Task.Delay(1000);
for (int i = 0; i < 10; i++)
{
// 等待服务启动
if (Gateway.Gateway.CheckServerOnline(id))
{
break;
}
await Task.Delay(1000);
}
}
else
{
@@ -113,11 +124,9 @@ public static class ServerService
throw new ValidationException("服务不存在");
}
var domainNames = await dbContext.DomainNames.Where(x => x.ServerId == id).ToListAsync();
Gateway.Gateway.ReloadGateway(server,
await dbContext.DomainNames.Where(x => x.ServerId == id).ToArrayAsync());
Gateway.Gateway.ReloadGateway(server, domainNames);
await Task.Delay(1000);
}).WithDescription("重载服务").WithDisplayName("重载服务").WithTags("服务");
return app;