GithubStar/DHTSpider
1
0
Fork 0
mirror of https://github.com/zhangkong828/DHTSpider.git synced 2026-10-17 04:30:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

README修改

Browse Source
This commit is contained in:
ZhangKong
2017-07-31 20:16:00 +08:00
parent 2554081f1d
commit e92e99ea25
16 changed files with 123 additions and 565 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+44 -37
View File
@@ -1,61 +1,68 @@
# DHTSpider
>A Very Simple Spider With DHT Crawler, Written by C#.
* The bittorrent library bases on DhtWalker,MonoTorrent.
* The bittorrent library bases on MonoTorrent.
## 介绍
### 介绍
DHTSpider 是一个由C#编写的 DHT 爬虫, 从全球` DHT ` 网络里"`嗅探`"正在下载的资源, 并把资源`metadata`(种子信息)进行抓取下载.
## 相关
### 相关
使用 `MongoDB` 进行数据存储.
>Cache
使用 `ElasticSearch` 进行全文检索(待定).
+ 使用 `Memory` > UseDefaultCache ( 默认 )
+ 使用 `Redis` > UseRedisCache ( TODO )
>Queue
## 环境
+ 使用 `Memory` > UseDefaultQueue （ 默认 ）
+ 使用 `Redis` > UseRedisQueue ( TODO )
>Store
+ 使用 `MongoDB` > UseMongoDBStore （ TODO ）
+ 使用 `ElasticSearch` > UseElasticSearchStore ( TODO )
>Log > `NLog`
>Ioc > `Autofac`
### 环境
1. `.net` 版本 `>=4.5.2`
2. 运行的机器需要`独立IP` , 能够访问外网 ( `翻墙` )
2. 运行的机器需要`独立IP` , 内网机器需要做下`端口映射`
## 参数
### 使用
```c#
var spider = SpiderConfiguration.Create() //使用默认配置
.UseDefaultCache() //默认使用内存缓存
.UseDefaultQueue() //默认使用内存队列
.Start();
```
Setting.InitSetting();//初始化相关参数
```
```
### 配置
```c#
LocalPort = 6881;
IsSaveTorrent = true;
TorrentSavePath = @"torrent\";
IsUsingMongoDb = false;
InitResolverThreadCount = 10;
MaxResolverThreadCount = 50;
WaitSeedsCount = 100;
//DhtSpiderSetting
DhtSpiderSetting.MaxSendQueue = 50;
DhtSpiderSetting.MaxFindSendPer = 100;
DhtSpiderSetting.MaxWaitCount = 1000;
DhtSpiderSetting.MaxCacheCount = 5000;
var spider = SpiderConfiguration.Create(new SpiderSetting()
{
LocalPort = 6881, //使用端口
IsSaveTorrent = true, //是否保存torrent
TorrentSavePath = "", //torrent保存路径
MaxSpiderThreadCount = 1, //爬虫线程数
MaxDownLoadThreadCount = 20 //下载线程数
})
.UseRedisCache() //使用redis缓存
.UseRedisQueue() //使用redis队列
.UseMongoDBStore() //使用mongodb存储
.Start();
```
## 感谢
项目开发中，`bittorrent`相关类库参考了 `DhtWalker` 和 `MonoTorrent`, 在此感谢作者 →_→
SimpleClient/Program.cs
+9 -11
View File
@@ -13,18 +13,16 @@ namespace SimpleClient
{
var spider = SpiderConfiguration.Create(new SpiderSetting()
{
LocalPort = 6881,
IsSaveTorrent = true,
TorrentSavePath = "",
MaxSpiderThreadCount = 1,
MaxDownLoadThreadCount = 20
LocalPort = 6881,//使用端口
IsSaveTorrent = true,//是否保存torrent
TorrentSavePath = "",//torrent保存路径
MaxSpiderThreadCount = 1,//爬虫线程数
MaxDownLoadThreadCount = 20//下载线程数
})
.UseDefaultCache()
.UseDefaultQueue()
//.UseRedisQueue()
//.UseElasticSearchStore()
.UseMongoDBStore()
.Start();
.UseDefaultCache() //默认使用内存缓存
.UseDefaultQueue() //默认使用内存队列
.Start();
Console.ReadKey();
}
Spider/Cache/RedisCache.cs
+1
View File
@@ -9,6 +9,7 @@ namespace Spider.Cache
{
public class RedisCache : ICache
{
//TODO
public RedisCache()
{
}
Spider/Core/DHTSpider.cs
+10 -16
View File
@@ -1,35 +1,20 @@
using Spider.Core.IoSocket;
using Spider.Core.UdpServer;
using Spider.Log;
using Spider.Log;
using Spider.Queue;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Tancoder.Torrent;
using Tancoder.Torrent.BEncoding;
using Tancoder.Torrent.Client;
using Tancoder.Torrent.Client.Messages;
using Tancoder.Torrent.Dht;
using Tancoder.Torrent.Dht.Listeners;
using Tancoder.Torrent.Dht.Messages;
namespace Spider.Core
{
public class DHTSpider : IDhtEngine
{
private static List<IPEndPoint> BOOTSTRAP_NODES = new List<IPEndPoint>() {
new IPEndPoint(Dns.GetHostEntry("router.bittorrent.com").AddressList[0], 6881),
new IPEndPoint(Dns.GetHostEntry("dht.transmissionbt.com").AddressList[0], 6881)
};
private static int MaxNodesSize = 1000;
public DHTSpider(IPEndPoint localAddress, IQueue queue)
{
LocalAddress = localAddress;
@@ -40,6 +25,14 @@ namespace Spider.Core
Queue = queue;
MessageQueue = new ConcurrentQueue<KeyValuePair<IPEndPoint, byte[]>>();
}
private static List<IPEndPoint> BOOTSTRAP_NODES = new List<IPEndPoint>() {
new IPEndPoint(Dns.GetHostEntry("router.bittorrent.com").AddressList[0], 6881),
new IPEndPoint(Dns.GetHostEntry("dht.transmissionbt.com").AddressList[0], 6881)
};
private static int MaxNodesSize = 1000;
public ConcurrentQueue<KeyValuePair<IPEndPoint, byte[]>> MessageQueue;
private object locker = new object();
public IMetaDataFilter Filter { get; set; }
@@ -216,6 +209,7 @@ namespace Spider.Core
}
public void Stop()
{
udpSocketListener.Stop();
}
private void JoinDHTNetwork()
Spider/Core/IoSocket/IoContextPool.cs
-104
View File
@@ -1,104 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Sockets;
using System.Text;
using System.Threading.Tasks;
namespace Spider.Core.IoSocket
{
/// <summary>
/// 与每个客户Socket相关联，进行Send和Receive投递时所需要的参数
/// </summary>
internal sealed class IoContextPool
{
List<SocketAsyncEventArgs> pool; //为每一个Socket客户端分配一个SocketAsyncEventArgs，用一个List管理，在程序启动时建立。
Int32 capacity; //pool对象池的容量
Int32 boundary; //已分配和未分配对象的边界，大的是已经分配的，小的是未分配的
internal IoContextPool(Int32 capacity)
{
this.pool = new List<SocketAsyncEventArgs>(capacity);
this.boundary = 0;
this.capacity = capacity;
}
/// <summary>
/// 往pool对象池中增加新建立的对象，因为这个程序在启动时会建立好所有对象，
/// 故这个方法只在初始化时会被调用,因此，没有加锁。
/// </summary>
/// <param name="arg"></param>
/// <returns></returns>
internal bool Add(SocketAsyncEventArgs arg)
{
if (arg != null && pool.Count < capacity)
{
pool.Add(arg);
boundary++;
return true;
}
else
return false;
}
internal Int32 Count
{
get { return this.pool.Count; }
}
/// <summary>
/// 取出集合中指定对象，内部使用
/// </summary>
/// <param name="index"></param>
/// <returns></returns>
//internal SocketAsyncEventArgs Get(int index)
//{
// if (index >= 0 && index < capacity)
// return pool[index];
// else
// return null;
//}
/// <summary>
/// 从对象池中取出一个对象，交给一个socket来进行投递请求操作
/// </summary>
/// <returns></returns>
internal SocketAsyncEventArgs Pop()
{
lock (this.pool)
{
if (boundary > 0)
{
--boundary;
return pool[boundary];
}
else
return null;
}
}
/// <summary>
/// 一个socket客户断开，与其相关的IoContext被释放，重新投入Pool中，备用。
/// </summary>
/// <param name="arg"></param>
/// <returns></returns>
internal bool Push(SocketAsyncEventArgs arg)
{
if (arg != null)
{
lock (this.pool)
{
int index = this.pool.IndexOf(arg, boundary); //找出被断开的客户,此处一定能查到，因此index不可能为-1，必定要大于0。
if (index == boundary) //正好是边界元素
boundary++;
else
{
this.pool[index] = this.pool[boundary]; //将断开客户移到边界上，边界右移
this.pool[boundary++] = arg;
}
}
return true;
}
else
return false;
}
}
}
Spider/Core/IoSocket/IoServer.cs
-263
View File
@@ -1,263 +0,0 @@
using Spider.Log;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace Spider.Core.IoSocket
{
public delegate void MessageReceived(byte[] buffer, IPEndPoint endpoint);
/// <summary>
/// 基于SocketAsyncEventArgs 实现 IOCP 服务器
/// </summary>
internal sealed class IoServer
{
/// <summary>
/// 监听Socket，用于接受客户端的连接请求
/// </summary>
private Socket listenSocket;
/// <summary>
/// 用于服务器执行的互斥同步对象
/// </summary>
private static Mutex mutex = new Mutex();
/// <summary>
/// 用于每个I/O Socket操作的缓冲区大小
/// </summary>
private Int32 bufferSize;
/// <summary>
/// 服务器上连接的客户端总数
/// </summary>
private Int32 numConnectedSockets;
/// <summary>
/// 服务器能接受的最大连接数量
/// </summary>
private Int32 numConnections;
/// <summary>
/// 完成端口上进行投递所用的IoContext对象池
/// </summary>
private IoContextPool ioContextPool;
private IPEndPoint endpoint;
/// <summary>
/// 构造函数，建立一个未初始化的服务器实例
/// </summary>
/// <param name="numConnections">服务器的最大连接数据</param>
/// <param name="bufferSize"></param>
internal IoServer(IPEndPoint endpoint)
{
this.endpoint = endpoint;
this.numConnectedSockets = 0;
this.numConnections = 10000;
this.bufferSize = 1024;
this.ioContextPool = new IoContextPool(numConnections);
// 为IoContextPool预分配SocketAsyncEventArgs对象
for (Int32 i = 0; i < this.numConnections; i++)
{
SocketAsyncEventArgs ioContext = new SocketAsyncEventArgs();
ioContext.Completed += new EventHandler<SocketAsyncEventArgs>(OnIOCompleted);
ioContext.SetBuffer(new Byte[this.bufferSize], 0, this.bufferSize);
// 将预分配的对象加入SocketAsyncEventArgs对象池中
this.ioContextPool.Add(ioContext);
}
}
/// <summary>
/// 当Socket上的发送或接收请求被完成时，调用此函数
/// </summary>
/// <param name="sender">激发事件的对象</param>
/// <param name="e">与发送或接收完成操作相关联的SocketAsyncEventArg对象</param>
private void OnIOCompleted(object sender, SocketAsyncEventArgs e)
{
switch (e.LastOperation)
{
case SocketAsyncOperation.ReceiveFrom:
case SocketAsyncOperation.Receive:
this.ProcessReceive(e);
break;
}
ioContextPool.Add(e);
}
public event MessageReceived MessageReceived;
/// <summary>
///接收完成时处理函数
/// </summary>
/// <param name="e">与接收完成操作相关联的SocketAsyncEventArg对象</param>
private void ProcessReceive(SocketAsyncEventArgs e)
{
// 检查远程主机是否关闭连接
if (e.BytesTransferred > 0)
{
if (e.SocketError == SocketError.Success)
{
Socket s = (Socket)e.UserToken;
//判断所有需接收的数据是否已经完成
if (s.Available == 0)
{
//获取接收到的数据
byte[] ByteArray = new byte[e.BytesTransferred];
Array.Copy(e.Buffer, 0, ByteArray, 0, ByteArray.Length);
MessageReceived?.Invoke(ByteArray, (IPEndPoint)e.RemoteEndPoint);
}
else if (!s.ReceiveAsync(e)) //为接收下一段数据，投递接收请求，这个函数有可能同步完成，这时返回false，并且不会引发SocketAsyncEventArgs.Completed事件
{
// 同步接收时处理接收完成事件
this.ProcessReceive(e);
}
}
else
{
this.ProcessError(e);
}
}
else
{
this.CloseClientSocket(e);
}
}
/// <summary>
/// 处理socket错误
/// </summary>
/// <param name="e"></param>
private void ProcessError(SocketAsyncEventArgs e)
{
Socket s = e.UserToken as Socket;
IPEndPoint localEp = s.LocalEndPoint as IPEndPoint;
this.CloseClientSocket(s, e);
string outStr = String.Format("套接字错误 {0}, IP {1}, 操作 {2}。", (Int32)e.SocketError, localEp, e.LastOperation);
Logger.Fatal(outStr);
}
/// <summary>
/// 关闭socket连接
/// </summary>
/// <param name="e">SocketAsyncEventArg associated with the completed send/receive operation.</param>
private void CloseClientSocket(SocketAsyncEventArgs e)
{
Socket s = e.UserToken as Socket;
this.CloseClientSocket(s, e);
}
private void CloseClientSocket(Socket s, SocketAsyncEventArgs e)
{
Interlocked.Decrement(ref this.numConnectedSockets);
// SocketAsyncEventArg 对象被释放，压入可重用队列。
this.ioContextPool.Push(e);
string outStr = String.Format("客户 {0} 断开, 共有 {1} 个连接。", s.RemoteEndPoint.ToString(), this.numConnectedSockets);
Logger.Fatal(outStr);
try
{
s.Shutdown(SocketShutdown.Send);
}
catch (Exception)
{
// Throw if client has closed, so it is not necessary to catch.
}
finally
{
s.Close();
}
}
/// <summary>
/// 启动服务，开始监听
/// </summary>
/// <param name="port">Port where the server will listen for connection requests.</param>
internal void Start()
{
// 创建监听socket
this.listenSocket = new Socket(endpoint.AddressFamily, SocketType.Dgram, ProtocolType.Udp);
//this.listenSocket.ReceiveBufferSize = this.bufferSize;
//this.listenSocket.SendBufferSize = this.bufferSize;
listenSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.Broadcast, true);
listenSocket.DontFragment = true;
listenSocket.EnableBroadcast = true;
listenSocket.Bind(endpoint);
// Blocks the current thread to receive incoming messages.
//mutex.WaitOne();
Receive();
}
private void Receive()
{
var ReceiveThread = new Thread(new ThreadStart(() =>
{
while (true)
{
if (ioContextPool.Count > 0)
{
SocketAsyncEventArgs ioContext = this.ioContextPool.Pop();
if (!listenSocket.ReceiveFromAsync(ioContext))
{
ProcessReceive(ioContext);
}
}
}
}));
ReceiveThread.Start();
}
/// <summary>
/// 停止服务
/// </summary>
internal void Stop()
{
this.listenSocket.Close();
//mutex.ReleaseMutex();
}
internal void Send(IPEndPoint ipendpoint, byte[] data)
{
listenSocket.BeginSendTo(data, 0, data.Length, SocketFlags.None, ipendpoint, SendCallBack, listenSocket);
}
private void SendCallBack(IAsyncResult result)
{
try
{
Socket sock = result.AsyncState as Socket;
if (sock != null)
{
sock.EndSend(result);
Logger.Warn("SendCallBack");
}
}
catch
{
}
}
}
}
Spider/Core/UdpServer/AsyncUDPServer.cs
-99
View File
@@ -1,99 +0,0 @@
using Spider.Log;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Threading.Tasks;
namespace Spider.Core.UdpServer
{
public delegate void MessageReceived(byte[] buffer, IPEndPoint endpoint);
public class AsyncUDPServer
{
private UdpClient _server;
private IPEndPoint endpoint;
public IPEndPoint Endpoint
{
get { return endpoint; }
}
public bool IsRunning { get; private set; }
public AsyncUDPServer(IPEndPoint endpoint)
{
this.endpoint = endpoint;
_server = new UdpClient(endpoint);
const uint IOC_IN = 0x80000000;
int IOC_VENDOR = 0x18000000;
int SIO_UDP_CONNRESET = (int)(IOC_IN | IOC_VENDOR | 12);
_server.Client.IOControl((int)SIO_UDP_CONNRESET, new byte[] { Convert.ToByte(false) }, new byte[4]);
}
public void Start()
{
if (!IsRunning)
{
IsRunning = true;
_server.EnableBroadcast = true;
_server.BeginReceive(ReceiveDataAsync, null);
}
}
public void Stop()
{
if (IsRunning)
{
IsRunning = false;
_server.Close();
}
}
public event MessageReceived MessageReceived;
private void ReceiveDataAsync(IAsyncResult ar)
{
IPEndPoint remote = null;
byte[] buffer = null;
try
{
buffer = _server.EndReceive(ar, ref remote);
MessageReceived?.Invoke(buffer, remote);
if (IsRunning && _server != null)
_server.BeginReceive(ReceiveDataAsync, null);
}
catch (Exception ex)
{
Logger.Fatal(ex.ToString());
if (IsRunning && _server != null)
_server.BeginReceive(ReceiveDataAsync, null);
}
}
/// <summary>
/// 发送数据
/// </summary>
public void Send(byte[] buffer, IPEndPoint endpoint)
{
try
{
if (endpoint.Address != IPAddress.Any)
_server.Send(buffer, buffer.Length, endpoint);
}
catch (Exception)
{
//TODO 异常处理
}
}
}
}
Spider/Core/UdpSocketListener.cs
+4 -8
View File
@@ -75,9 +75,9 @@ namespace Spider.Core
var errorCode = (int)e.SocketError;
//The listen socket was closed
if (errorCode == 995 || errorCode == 10004 || errorCode == 10038)
//return;
Logger.Fatal($"The listen socket was closed errorCode:{errorCode}");
//if (errorCode == 995 || errorCode == 10004 || errorCode == 10038)
//return;
//Logger.Fatal($"The listen socket was closed errorCode:{errorCode}");
Logger.Fatal($"errorCode:{errorCode}");
}
@@ -88,8 +88,6 @@ namespace Spider.Core
//获取接收到的数据
byte[] ByteArray = new byte[e.BytesTransferred];
Array.Copy(e.Buffer, 0, ByteArray, 0, ByteArray.Length);
//测试
Logger.Debug($"buffer:{ByteArray.Length} remote:{(IPEndPoint)e.RemoteEndPoint}");
MessageReceived?.Invoke(ByteArray, (IPEndPoint)e.RemoteEndPoint);
}
catch (Exception exc)
@@ -159,9 +157,7 @@ namespace Spider.Core
if (endpoint.Address != IPAddress.Any)
{
var len = m_ListenSocket.SendTo(buffer, endpoint);
//测试
Logger.Warn($"Send :{len} {buffer.Length} {endpoint}");
//Logger.Warn($"Send :{len} {buffer.Length} {endpoint}");
}
else
{
Spider/Log/Logger.cs
+2 -2
View File
@@ -42,9 +42,9 @@ namespace Spider.Log
public static void ConsoleWrite(string msg)
public static void ConsoleWrite(string msg, ConsoleColor consoleColor = ConsoleColor.Green)
{
Console.ForegroundColor = ConsoleColor.Green;
Console.ForegroundColor = consoleColor;
Console.WriteLine("------------------------------------------------------");
Console.WriteLine($"[{DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")}]{msg}");
}
Spider/Queue/MemoryQueue.cs → Spider/Queue/DefaultQueue.cs
+2 -2
View File
@@ -9,10 +9,10 @@ using Tancoder.Torrent;
namespace Spider.Queue
{
public class MemoryQueue : IQueue
public class DefaultQueue : IQueue
{
public ConcurrentQueue<KeyValuePair<InfoHash, IPEndPoint>> _queue = null;
public MemoryQueue()
public DefaultQueue()
{
_queue = new ConcurrentQueue<KeyValuePair<InfoHash, IPEndPoint>>();
}
Spider/Queue/RedisQueue.cs
+1 -1
View File
@@ -10,7 +10,7 @@ namespace Spider.Queue
{
public class RedisQueue : IQueue
{
//TODO
public RedisQueue()
{
}
Spider/Spider.csproj
+3 -4
View File
@@ -57,9 +57,6 @@
<Compile Include="Cache\ICache.cs" />
<Compile Include="Cache\DefaultCache.cs" />
<Compile Include="Cache\RedisCache.cs" />
<Compile Include="Core\IoSocket\IoContextPool.cs" />
<Compile Include="Core\IoSocket\IoServer.cs" />
<Compile Include="Core\UdpServer\AsyncUDPServer.cs" />
<Compile Include="Core\UdpSocketListener.cs" />
<Compile Include="Helper\RedisHelper.cs" />
<Compile Include="Queue\IQueue.cs" />
@@ -67,11 +64,13 @@
<Compile Include="Core\EasyTokenManager.cs" />
<Compile Include="Log\Logger.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Queue\MemoryQueue.cs" />
<Compile Include="Queue\DefaultQueue.cs" />
<Compile Include="Queue\RedisQueue.cs" />
<Compile Include="SpiderConfiguration.cs" />
<Compile Include="SpiderSetting.cs" />
<Compile Include="Store\ElasticSearchStore.cs" />
<Compile Include="Store\IStore.cs" />
<Compile Include="Store\MongoDBStore.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Tancoder.Torrent\Tancoder.Torrent.csproj">
Spider/SpiderConfiguration.cs
+11 -17
View File
@@ -3,6 +3,7 @@ using Spider.Cache;
using Spider.Core;
using Spider.Log;
using Spider.Queue;
using Spider.Store;
using System;
using System.Collections.Generic;
using System.IO;
@@ -28,6 +29,7 @@ namespace Spider
public SpiderSetting _option { get; set; }
public ICache _cache { get; set; }
public IQueue _queue { get; set; }
public IStore _store { get; set; }
private SpiderConfiguration(SpiderSetting option)
{
@@ -73,7 +75,7 @@ namespace Spider
public SpiderConfiguration UseDefaultQueue()
{
_builder.RegisterType<MemoryQueue>().As<IQueue>().Named<IQueue>("Queue").SingleInstance();
_builder.RegisterType<DefaultQueue>().As<IQueue>().Named<IQueue>("Queue").SingleInstance();
return _instance;
}
@@ -85,12 +87,13 @@ namespace Spider
public SpiderConfiguration UseElasticSearchStore()
{
_builder.RegisterType<ElasticSearchStore>().As<IStore>().Named<IStore>("Store").SingleInstance();
return _instance;
}
public SpiderConfiguration UseMongoDBStore()
{
//TODO
_builder.RegisterType<MongoDBStore>().As<IStore>().Named<IStore>("Store").SingleInstance();
return _instance;
}
@@ -110,18 +113,15 @@ namespace Spider
{
throw new Exception("没有注册Queue");
}
//if (!_container.IsRegisteredWithName<IStore>("Store"))
//{
// throw new Exception("没有注册Store");
//}
_queue = _container.ResolveNamed<IQueue>("Queue");
_cache = _container.ResolveNamed<ICache>("Cache");
//_store = _container.ResolveNamed<IStore>("Store");
Task.Run(() =>
{
while (true)
{
Thread.Sleep(1000 * 60);
Logger.Info($"QueueCount:{_queue.Count()}");
}
});
for (var i = 0; i < _option.MaxSpiderThreadCount; i++)
{
@@ -197,13 +197,7 @@ namespace Spider
File.WriteAllBytes(filepath, metadata.Encode());
}
var list = new List<string>();
//foreach (var item in metadata)
//{
// list.Add($"[key]={item.Key}[val]={item.Value}");
//}
//var str = string.Join("&", list);
//Logger.Warn($"{hash} {name} {str}");
Logger.ConsoleWrite($"线程[{threadId}]下载完成 Name:{name} ");
Logger.ConsoleWrite($"线程[{threadId}]下载完成 Name:{name} ", ConsoleColor.Yellow);
}
}
Spider/Store/ElasticSearchStore.cs
+17
View File
@@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Spider.Store
{
public class ElasticSearchStore : IStore
{
//TODO
public bool Insert(object obj)
{
throw new NotImplementedException();
}
}
}
Spider/Store/IStore.cs
+2 -1
View File
@@ -5,7 +5,8 @@ using System.Text;
namespace Spider.Store
{
interface IStore
public interface IStore
{
bool Insert(object obj);
}
}
Spider/Store/MongoDBStore.cs
+17
View File
@@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Spider.Store
{
public class MongoDBStore : IStore
{
//TODO
public bool Insert(object obj)
{
throw new NotImplementedException();
}
}
}