mirror of
https://github.com/xboxeer/NScrapy.git
synced 2026-10-23 00:39:02 +08:00
46086ffaf9
Create a ConifgProviderFactory so thay we can get ConfigProvider from a single point Also enhenced the logic of refresh config, right now when zookeeper watcher dected the config is changed NScrapy will update its context, and during that time, NScrapy will lock the access to the Connection of Redis
64 lines
2.2 KiB
C#
64 lines
2.2 KiB
C#
using NScrapy.Downloader;
|
|
using StackExchange.Redis;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Text;
|
|
using System.Threading;
|
|
|
|
namespace NScrapy.DownloaderShell
|
|
{
|
|
public static class RedisManager
|
|
{
|
|
private static ConnectionMultiplexer connection;
|
|
private static bool connectionChangeRequested = false;
|
|
private static object lockObj = new object();
|
|
public static ConnectionMultiplexer Connection
|
|
{
|
|
get
|
|
{
|
|
if (connectionChangeRequested)
|
|
{
|
|
Monitor.Wait(lockObj);
|
|
}
|
|
return connection;
|
|
}
|
|
private set { connection = value; }
|
|
}
|
|
|
|
public static string ReceiverQueue { get; private set; }
|
|
|
|
static RedisManager()
|
|
{
|
|
Connect();
|
|
}
|
|
|
|
private static void Connect()
|
|
{
|
|
var redisServer = DownloaderContext.CurrentContext.CurrentConfig["AppSettings:Scheduler.RedisExt:RedisServer"];
|
|
var redisPort = DownloaderContext.CurrentContext.CurrentConfig["AppSettings:Scheduler.RedisExt:RedisPort"];
|
|
ReceiverQueue = string.IsNullOrEmpty(DownloaderContext.CurrentContext.CurrentConfig["AppSettings:Scheduler.RedisExt:ReceiverQueue"]) ? "NScrapy.Downloader" : DownloaderContext.CurrentContext.CurrentConfig["AppSettings:Scheduler.RedisExt:ReceiverQueue"];
|
|
var options = new ConfigurationOptions()
|
|
{
|
|
EndPoints = { $"{redisServer}:{redisPort}" },
|
|
SyncTimeout = 60000,
|
|
ConnectTimeout = 60000,
|
|
};
|
|
Connection = ConnectionMultiplexer.Connect(options);
|
|
}
|
|
|
|
public static void GetLock(string lockKey, string keyToken)
|
|
{
|
|
while (!Connection.GetDatabase().LockTake(lockKey, keyToken,new TimeSpan(TimeSpan.TicksPerSecond)))
|
|
{
|
|
//Sleep until we got the lock
|
|
Thread.Sleep(10);
|
|
}
|
|
}
|
|
|
|
public static void ReleaseLock(string lockKey, string keyToken)
|
|
{
|
|
Connection.GetDatabase().LockRelease(lockKey, keyToken);
|
|
}
|
|
}
|
|
}
|