first stable version

2026-10-16 21:20:38 +08:00 · 2012-12-23 14:22:43 +08:00
commit 64f0b5f4aa
39 changed files with 4519 additions and 0 deletions
@@ -0,0 +1,30 @@
+Thumbs.db
+*.obj
+*.exe
+*.pdb
+*.user
+*.aps
+*.pch
+*.vspscc
+*_i.c
+*_p.c
+*.ncb
+*.suo
+*.sln.docstates
+*.tlb
+*.tlh
+*.bak
+*.cache
+*.ilk
+*.log
+[Bb]in
+[Dd]ebug*/
+*.lib
+*.sbr
+obj/
+[Rr]elease*/
+_ReSharper*/
+[Tt]est[Rr]esult*
+*.vssscc
+$tf*/
+packages/
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <solution>
+    <add key="disableSourceControlIntegration" value="true" />
+  </solution>
+</configuration>
@@ -0,0 +1,153 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+    <PropertyGroup>
+        <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">$(MSBuildProjectDirectory)\..\</SolutionDir>
+        
+        <!-- Enable the restore command to run before builds -->
+        <RestorePackages Condition="  '$(RestorePackages)' == '' ">false</RestorePackages>
+
+        <!-- Property that enables building a package from a project -->
+        <BuildPackage Condition=" '$(BuildPackage)' == '' ">false</BuildPackage>
+
+        <!-- Determines if package restore consent is required to restore packages -->
+        <RequireRestoreConsent Condition=" '$(RequireRestoreConsent)' != 'false' ">true</RequireRestoreConsent>
+        
+        <!-- Download NuGet.exe if it does not already exist -->
+        <DownloadNuGetExe Condition=" '$(DownloadNuGetExe)' == '' ">false</DownloadNuGetExe>
+    </PropertyGroup>
+    
+    <ItemGroup Condition=" '$(PackageSources)' == '' ">
+        <!-- Package sources used to restore packages. By default, registered sources under %APPDATA%\NuGet\NuGet.Config will be used -->
+        <!-- The official NuGet package source (https://nuget.org/api/v2/) will be excluded if package sources are specified and it does not appear in the list -->
+        <!--
+            <PackageSource Include="https://nuget.org/api/v2/" />
+            <PackageSource Include="https://my-nuget-source/nuget/" />
+        -->
+    </ItemGroup>
+
+    <PropertyGroup Condition=" '$(OS)' == 'Windows_NT'">
+        <!-- Windows specific commands -->
+        <NuGetToolsPath>$([System.IO.Path]::Combine($(SolutionDir), ".nuget"))</NuGetToolsPath>
+        <PackagesConfig>$([System.IO.Path]::Combine($(ProjectDir), "packages.config"))</PackagesConfig>
+        <PackagesDir>$([System.IO.Path]::Combine($(SolutionDir), "packages"))</PackagesDir>
+    </PropertyGroup>
+    
+    <PropertyGroup Condition=" '$(OS)' != 'Windows_NT'">
+        <!-- We need to launch nuget.exe with the mono command if we're not on windows -->
+        <NuGetToolsPath>$(SolutionDir).nuget</NuGetToolsPath>
+        <PackagesConfig>packages.config</PackagesConfig>
+        <PackagesDir>$(SolutionDir)packages</PackagesDir>
+    </PropertyGroup>
+    
+    <PropertyGroup>
+        <!-- NuGet command -->
+        <NuGetExePath Condition=" '$(NuGetExePath)' == '' ">$(NuGetToolsPath)\nuget.exe</NuGetExePath>
+        <PackageSources Condition=" $(PackageSources) == '' ">@(PackageSource)</PackageSources>
+        
+        <NuGetCommand Condition=" '$(OS)' == 'Windows_NT'">"$(NuGetExePath)"</NuGetCommand>
+        <NuGetCommand Condition=" '$(OS)' != 'Windows_NT' ">mono --runtime=v4.0.30319 $(NuGetExePath)</NuGetCommand>
+
+        <PackageOutputDir Condition="$(PackageOutputDir) == ''">$(TargetDir.Trim('\\'))</PackageOutputDir>
+        
+        <RequireConsentSwitch Condition=" $(RequireRestoreConsent) == 'true' ">-RequireConsent</RequireConsentSwitch>
+        <!-- Commands -->
+        <RestoreCommand>$(NuGetCommand) install "$(PackagesConfig)" -source "$(PackageSources)"  $(RequireConsentSwitch) -o "$(PackagesDir)"</RestoreCommand>
+        <BuildCommand>$(NuGetCommand) pack "$(ProjectPath)" -p Configuration=$(Configuration) -o "$(PackageOutputDir)" -symbols</BuildCommand>
+
+        <!-- We need to ensure packages are restored prior to assembly resolve -->
+        <ResolveReferencesDependsOn Condition="$(RestorePackages) == 'true'">
+            RestorePackages;
+            $(ResolveReferencesDependsOn);
+        </ResolveReferencesDependsOn>
+
+        <!-- Make the build depend on restore packages -->
+        <BuildDependsOn Condition="$(BuildPackage) == 'true'">
+            $(BuildDependsOn);
+            BuildPackage;
+        </BuildDependsOn>
+    </PropertyGroup>
+
+    <Target Name="CheckPrerequisites">
+        <!-- Raise an error if we're unable to locate nuget.exe  -->
+        <Error Condition="'$(DownloadNuGetExe)' != 'true' AND !Exists('$(NuGetExePath)')" Text="Unable to locate '$(NuGetExePath)'" />
+        <SetEnvironmentVariable EnvKey="VisualStudioVersion" EnvValue="$(VisualStudioVersion)" Condition=" '$(VisualStudioVersion)' != '' AND '$(OS)' == 'Windows_NT' " />
+        <!--
+        Take advantage of MsBuild's build dependency tracking to make sure that we only ever download nuget.exe once.
+        This effectively acts as a lock that makes sure that the download operation will only happen once and all
+        parallel builds will have to wait for it to complete.
+        -->
+        <MsBuild Targets="_DownloadNuGet" Projects="$(MSBuildThisFileFullPath)" Properties="Configuration=NOT_IMPORTANT" />
+    </Target>
+
+    <Target Name="_DownloadNuGet">
+        <DownloadNuGet OutputFilename="$(NuGetExePath)" Condition=" '$(DownloadNuGetExe)' == 'true' AND !Exists('$(NuGetExePath)')" />
+    </Target>
+
+    <Target Name="RestorePackages" DependsOnTargets="CheckPrerequisites">
+        <Exec Command="$(RestoreCommand)"
+              Condition="'$(OS)' != 'Windows_NT' And Exists('$(PackagesConfig)')" />
+              
+        <Exec Command="$(RestoreCommand)"
+              LogStandardErrorAsError="true"
+              Condition="'$(OS)' == 'Windows_NT' And Exists('$(PackagesConfig)')" />
+    </Target>
+
+    <Target Name="BuildPackage" DependsOnTargets="CheckPrerequisites">
+        <Exec Command="$(BuildCommand)" 
+              Condition=" '$(OS)' != 'Windows_NT' " />
+              
+        <Exec Command="$(BuildCommand)"
+              LogStandardErrorAsError="true"
+              Condition=" '$(OS)' == 'Windows_NT' " />
+    </Target>
+    
+    <UsingTask TaskName="DownloadNuGet" TaskFactory="CodeTaskFactory" AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.v4.0.dll">
+        <ParameterGroup>
+            <OutputFilename ParameterType="System.String" Required="true" />
+        </ParameterGroup>
+        <Task>
+            <Reference Include="System.Core" />
+            <Using Namespace="System" />
+            <Using Namespace="System.IO" />
+            <Using Namespace="System.Net" />
+            <Using Namespace="Microsoft.Build.Framework" />
+            <Using Namespace="Microsoft.Build.Utilities" />
+            <Code Type="Fragment" Language="cs">
+                <![CDATA[
+                try {
+                    OutputFilename = Path.GetFullPath(OutputFilename);
+
+                    Log.LogMessage("Downloading latest version of NuGet.exe...");
+                    WebClient webClient = new WebClient();
+                    webClient.DownloadFile("https://nuget.org/nuget.exe", OutputFilename);
+
+                    return true;
+                }
+                catch (Exception ex) {
+                    Log.LogErrorFromException(ex);
+                    return false;
+                }
+            ]]>
+            </Code>
+        </Task>
+    </UsingTask>
+    
+     <UsingTask TaskName="SetEnvironmentVariable" TaskFactory="CodeTaskFactory" AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.v4.0.dll">
+        <ParameterGroup>
+            <EnvKey ParameterType="System.String" Required="true" />
+            <EnvValue ParameterType="System.String" Required="true" />
+        </ParameterGroup>
+        <Task>
+            <Using Namespace="System" />
+            <Code Type="Fragment" Language="cs">
+                <![CDATA[
+                try {
+                    Environment.SetEnvironmentVariable(EnvKey, EnvValue, System.EnvironmentVariableTarget.Process);
+                }
+                catch  {
+                }
+            ]]>
+            </Code>
+        </Task>
+    </UsingTask>
+</Project>
@@ -0,0 +1 @@
+\.git
@@ -0,0 +1,2 @@
+# ScrapingSpider
+> C#开发的支持多线程，关键字过滤，正文内容自解析的爬虫
@@ -0,0 +1,49 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2012
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ScrapingSpider", "src\ScrapingSpider\ScrapingSpider.csproj", "{947A6286-6AAB-4AEE-9AF0-D4474EAF1876}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{59AB22D8-0DF8-408E-8CEC-CCE6F5E9FCB2}"
+	ProjectSection(SolutionItems) = preProject
+		.nuget\NuGet.Config = .nuget\NuGet.Config
+		.nuget\NuGet.exe = .nuget\NuGet.exe
+		.nuget\NuGet.targets = .nuget\NuGet.targets
+	EndProjectSection
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ScrapingSpider.Core", "ScrapingSpider.Core\ScrapingSpider.Core.csproj", "{636E94A7-2F32-47DF-B358-7F9E1EA8BA0D}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Debug|Mixed Platforms = Debug|Mixed Platforms
+		Debug|x86 = Debug|x86
+		Release|Any CPU = Release|Any CPU
+		Release|Mixed Platforms = Release|Mixed Platforms
+		Release|x86 = Release|x86
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{947A6286-6AAB-4AEE-9AF0-D4474EAF1876}.Debug|Any CPU.ActiveCfg = Debug|x86
+		{947A6286-6AAB-4AEE-9AF0-D4474EAF1876}.Debug|Mixed Platforms.ActiveCfg = Debug|x86
+		{947A6286-6AAB-4AEE-9AF0-D4474EAF1876}.Debug|Mixed Platforms.Build.0 = Debug|x86
+		{947A6286-6AAB-4AEE-9AF0-D4474EAF1876}.Debug|x86.ActiveCfg = Debug|x86
+		{947A6286-6AAB-4AEE-9AF0-D4474EAF1876}.Debug|x86.Build.0 = Debug|x86
+		{947A6286-6AAB-4AEE-9AF0-D4474EAF1876}.Release|Any CPU.ActiveCfg = Release|x86
+		{947A6286-6AAB-4AEE-9AF0-D4474EAF1876}.Release|Mixed Platforms.ActiveCfg = Release|x86
+		{947A6286-6AAB-4AEE-9AF0-D4474EAF1876}.Release|Mixed Platforms.Build.0 = Release|x86
+		{947A6286-6AAB-4AEE-9AF0-D4474EAF1876}.Release|x86.ActiveCfg = Release|x86
+		{947A6286-6AAB-4AEE-9AF0-D4474EAF1876}.Release|x86.Build.0 = Release|x86
+		{636E94A7-2F32-47DF-B358-7F9E1EA8BA0D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{636E94A7-2F32-47DF-B358-7F9E1EA8BA0D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{636E94A7-2F32-47DF-B358-7F9E1EA8BA0D}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{636E94A7-2F32-47DF-B358-7F9E1EA8BA0D}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+		{636E94A7-2F32-47DF-B358-7F9E1EA8BA0D}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{636E94A7-2F32-47DF-B358-7F9E1EA8BA0D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{636E94A7-2F32-47DF-B358-7F9E1EA8BA0D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{636E94A7-2F32-47DF-B358-7F9E1EA8BA0D}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{636E94A7-2F32-47DF-B358-7F9E1EA8BA0D}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+		{636E94A7-2F32-47DF-B358-7F9E1EA8BA0D}.Release|x86.ActiveCfg = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal
@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace ScrapingSpider.Core.Events
+{
+    public class AddUrlEventArgs : EventArgs
+    {
+        public string Url { get; set; }
+        public int Depth { get; set; }
+        public string Title { get; set; }
+    }
+}
@@ -0,0 +1,9 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace ScrapingSpider.Core.Events
+{
+    public delegate bool AddUrlEventHandler(AddUrlEventArgs args);
+}
@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace ScrapingSpider.Core.Events
+{
+    /// <summary>
+    /// 爬取错误
+    /// </summary>
+    public class CrawlErrorEventArgs : EventArgs
+    {
+        public string Url { get; set; }
+        public Exception Exception { get; set; }
+    }
+}
@@ -0,0 +1,9 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace ScrapingSpider.Core.Events
+{
+    public delegate void CrawlErrorEventHandler(CrawlErrorEventArgs args);
+}
@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace ScrapingSpider.Core.Events
+{
+
+    public class DataReceivedEventArgs : EventArgs
+    {
+        public string Url { get; set; }
+        public int Depth { get; set; }
+        public string Html { get; set; }
+    }
+}
@@ -0,0 +1,9 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace ScrapingSpider.Core.Events
+{
+    public delegate void DataReceivedEventHandler(DataReceivedEventArgs args);
+}
@@ -0,0 +1,27 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace ScrapingSpider.Core.Logging
+{
+    public class EmptyLogger : ILogger
+    {
+        public void Info(string msg)
+        {
+        }
+
+        public void Warn(string msg)
+        {
+        }
+
+        public void Error(string msg)
+        {
+        }
+
+        public void Error(Exception ex)
+        {
+        }
+    }
+}
@@ -0,0 +1,41 @@
+//============================================================
+//
+//    Copyright (C) 2012 翟士丹@CYOU●监控部●CYINC.MOMain 版权所有
+//    文件名　：ILogger.cs
+//    功能描述：日志记录接口
+//    创建标识：StanZhai 2012/07/18
+//    文件版本：1.0.0.0
+//
+//============================================================
+
+using System;
+
+namespace ScrapingSpider.Core.Logging
+{
+    public interface ILogger
+    {
+        /// <summary>
+        /// 添加Info级别日志
+        /// </summary>
+        /// <param name="msg">日志信息</param>
+        void Info(string msg);
+
+        /// <summary>
+        /// 添加WARN级别日志
+        /// </summary>
+        /// <param name="msg">日志信息</param>
+        void Warn(string msg);
+
+        /// <summary>
+        /// 添加ERROR级别日志
+        /// </summary>
+        /// <param name="msg">日志信息</param>
+        void Error(string msg);
+
+        /// <summary>
+        /// 添加ERROR级别日志，记录异常信息
+        /// </summary>
+        /// <param name="ex">异常实例</param>
+        void Error(Exception ex);
+    }
+}
@@ -0,0 +1,76 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace ScrapingSpider.Core.Models
+{
+    /// <summary>
+    /// 爬虫设置
+    /// </summary>
+    [Serializable]
+    public class Settings
+    {
+        /// <summary>
+        /// 初始化种子地址
+        /// </summary>
+        public string InitSeeds { get; set; }
+
+        private int _threads = 1;
+        /// <summary>
+        /// 启用的线程数量, 默认为1
+        /// </summary>
+        public int Threads
+        {
+            get { return _threads; }
+            set { _threads = value; }
+        }
+
+        /// <summary>
+        /// 要排除的链接:.png|.jpg|.gif|.rar
+        /// </summary>
+        public string EscapeLinks { get; set; }
+
+        /// <summary>
+        /// 指获取包含指定关键字的链接，多个关键字用回车分割
+        /// </summary>
+        public string Keywords { get; set; }
+
+        private int _crawlDepth = 3;
+        /// <summary>
+        /// 抓取深度，小于0表示不限制深度, 默认为3
+        /// </summary>
+        public int CrawlDepth
+        {
+            get { return _crawlDepth; }
+            set { _crawlDepth = value; }
+        }
+
+        private int _timeout = 15000;
+        /// <summary>
+        /// 抓取超时时间
+        /// </summary>
+        public int Timeout
+        {
+            get { return _timeout; }
+            set { _timeout = value; }
+        }
+
+        private string _userAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11";
+        public string UserAgent
+        {
+            get { return _userAgent; }
+            set { _userAgent = value; }
+        }
+
+        /// <summary>
+        /// 是否保留Cookie
+        /// </summary>
+        public bool KeepCookie { get; set; }
+
+        /// <summary>
+        /// 抓取过程中是否锁定Host
+        /// </summary>
+        public bool LockHost { get; set; }
+    }
+}
@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace ScrapingSpider.Core.Models
+{
+    /// <summary>
+    /// Url信息
+    /// </summary>
+    public class UrlInfo
+    {
+        /// <summary>
+        /// Url地址
+        /// </summary>
+        public string Url { get; set; }
+
+        /// <summary>
+        /// 相对于种子地址的深度
+        /// </summary>
+        public int Depth { get; set; }
+    }
+}
@@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// 有关程序集的常规信息通过以下
+// 特性集控制。更改这些特性值可修改
+// 与程序集关联的信息。
+[assembly: AssemblyTitle("ScrapingSpider.Core")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("ScrapingSpider.Core")]
+[assembly: AssemblyCopyright("Copyright ©  2012")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// 将 ComVisible 设置为 false 使此程序集中的类型
+// 对 COM 组件不可见。如果需要从 COM 访问此程序集中的类型，
+// 则将该类型上的 ComVisible 特性设置为 true。
+[assembly: ComVisible(false)]
+
+// 如果此项目向 COM 公开，则下列 GUID 用于类型库的 ID
+[assembly: Guid("14f9aa96-e22d-4452-9339-082c77811ab7")]
+
+// 程序集的版本信息由下面四个值组成:
+//
+//      主版本
+//      次版本 
+//      生成号
+//      修订号
+//
+// 可以指定所有这些值，也可以使用“生成号”和“修订号”的默认值，
+// 方法是按如下所示使用“*”:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{636E94A7-2F32-47DF-B358-7F9E1EA8BA0D}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>ScrapingSpider.Core</RootNamespace>
+    <AssemblyName>ScrapingSpider.Core</AssemblyName>
+    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
+    <RestorePackages>true</RestorePackages>
+    <TargetFrameworkProfile>Client</TargetFrameworkProfile>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="HtmlAgilityPack">
+      <HintPath>..\packages\HtmlAgilityPack.1.4.6\lib\Net40-client\HtmlAgilityPack.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Data.DataSetExtensions" />
+    <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="Events\AddUrlEventArgs.cs" />
+    <Compile Include="Events\AddUrlEventHandler.cs" />
+    <Compile Include="Events\CrawlErrorEventArgs.cs" />
+    <Compile Include="Events\CrawlErrorEventHandler.cs" />
+    <Compile Include="Events\DataReceivedEventArgs.cs" />
+    <Compile Include="Events\DataReceivedEventHandler.cs" />
+    <Compile Include="Logging\EmptyLogger.cs" />
+    <Compile Include="Logging\ILogger.cs" />
+    <Compile Include="Models\Settings.cs" />
+    <Compile Include="Models\UrlInfo.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Spider.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <Folder Include="Exceptions\" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="packages.config" />
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="$(SolutionDir)\.nuget\nuget.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>
@@ -0,0 +1,284 @@
+using System.IO.Compression;
+using ScrapingSpider.Core.Logging;
+using ScrapingSpider.Core.Models;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading;
+using ScrapingSpider.Core.Events;
+using HtmlAgilityPack;
+
+namespace ScrapingSpider.Core
+{
+    /// <summary>
+    /// 主爬虫类
+    /// Author: 翟士丹 StanZhai，2012/12/21
+    /// </summary>
+    public class Spider
+    {
+        private Settings _settings;
+        private Queue<UrlInfo> _urlQueue;
+        private Thread[] _crawlThreads;
+        private string[] _escapeLinks;
+        private string[] _keywords;
+        private CookieContainer _cookieContainer;
+        private ILogger _log;
+
+        #region 事件
+        public event DataReceivedEventHandler DataReceivedEvent;
+        public event CrawlErrorEventHandler CrawErrorEvent;
+        /// <summary>
+        /// 如果事件的执行结果为false，则不会将链接添加到队列
+        /// </summary>
+        public event AddUrlEventHandler AddUrlEvent;
+        #endregion
+
+        public Spider(Settings settings, ILogger logger, IEnumerable<UrlInfo> continueLinks = null)
+        {
+            _settings = settings;
+            _cookieContainer = new CookieContainer();
+            _crawlThreads = new Thread[_settings.Threads];
+            _urlQueue = new Queue<UrlInfo>();
+            _log = logger ?? new EmptyLogger();
+
+            // 将初始种子加入队列
+            foreach (var seed in _settings.InitSeeds.Split(Environment.NewLine.ToCharArray()))
+            {
+                if (Regex.IsMatch(seed, @"^(http|https)://([\w-]+\.)+[\w-]+(/[\w- ./?%&=]*)?", RegexOptions.IgnoreCase))
+                    _urlQueue.Enqueue(new UrlInfo { Url = seed, Depth = 1 });
+            }
+            // 将待继续爬取的链接加到队列
+            if (continueLinks != null)
+            {
+                foreach (var link in continueLinks)
+                {
+                    _urlQueue.Enqueue(link);
+                }
+            }
+            // 初始化每个爬取线程
+            for (int i = 0; i < _settings.Threads; i++)
+            {
+                _crawlThreads[i] = new Thread(new ThreadStart(CrawlProc));
+            }
+            // 设置排除的链接
+            if (!String.IsNullOrEmpty(_settings.EscapeLinks))
+                _escapeLinks = _settings.EscapeLinks.Split('|');
+            // 设置关键字
+            if (!String.IsNullOrEmpty(_settings.Keywords))
+                _keywords = _settings.Keywords.Replace("\r", "").Split('\n');
+        }
+
+        /// <summary>
+        /// 开始抓取
+        /// </summary>
+        public void Crawl()
+        {
+            foreach (var thread in _crawlThreads)
+            {
+                thread.Start();
+            }
+            _log.Info(String.Format("爬虫已启动，开启线程数：{0}", _crawlThreads.Length));
+        }
+
+        // 爬取过程
+        private void CrawlProc()
+        {
+            while (true)
+            {
+                if (_urlQueue.Count == 0)
+                {
+                    Thread.Sleep(2000);
+                    continue;
+                }
+
+                UrlInfo urlInfo = null;
+                lock (this)
+                {
+                    if (_urlQueue.Count == 0)
+                        continue;
+                    urlInfo = _urlQueue.Dequeue();
+                }
+                try
+                {
+                    if (urlInfo == null)
+                        continue;
+                    HttpWebRequest request = WebRequest.Create(urlInfo.Url) as HttpWebRequest;
+                    ConfigRequest(request);
+                    HttpWebResponse response = request.GetResponse() as HttpWebResponse;
+                    ParseCookie(response);
+                    // 获取网页数据量，如果页面压缩，则解压数据流
+                    using (Stream stream = response.ContentEncoding == "gzip" ? 
+                        new GZipStream(response.GetResponseStream(), CompressionMode.Decompress) : 
+                        response.GetResponseStream())
+                    {
+                        _log.Info(String.Format("{0}-{1}-{2}-{3}", 
+                            (int)response.StatusCode, 
+                            response.StatusDescription, 
+                            urlInfo.Depth,
+                            urlInfo.Url));
+
+                        string html = ParseContent(stream, response.CharacterSet);
+                        ParseLinks(urlInfo, html);
+                        if (DataReceivedEvent != null)
+                            DataReceivedEvent(new DataReceivedEventArgs { Url = urlInfo.Url, Depth = urlInfo.Depth, Html = html});
+                    }                   
+                }
+                catch (Exception ex)
+                {
+                    _log.Error(String.Format("{0}：{1}", urlInfo.Url, ex.Message));
+                    if (CrawErrorEvent != null)
+                        CrawErrorEvent(new CrawlErrorEventArgs { Url = urlInfo.Url, Exception = ex });
+                }
+            }
+        }
+
+        /// <summary>
+        /// 配置请求设置
+        /// </summary>
+        /// <param name="request"></param>
+        private void ConfigRequest(HttpWebRequest request)
+        {
+            request.UserAgent = _settings.UserAgent;
+            request.CookieContainer = _cookieContainer;
+            request.AllowAutoRedirect = true;
+            request.MediaType = "text/html";
+            if (_settings.Timeout > 0)
+                request.Timeout = _settings.Timeout;
+        }
+
+        /// <summary>
+        /// 解析Cookie
+        /// </summary>
+        /// <param name="response"></param>
+        private void ParseCookie(HttpWebResponse response)
+        {
+            if (_settings.KeepCookie)
+            {
+                string cookiesExpression = response.Headers["Set-Cookie"];
+                if (!string.IsNullOrEmpty(cookiesExpression))
+                {
+                    Uri cookieUrl = new Uri(string.Format("{0}://{1}:{2}/", response.ResponseUri.Scheme, response.ResponseUri.Host, response.ResponseUri.Port));
+                    _cookieContainer.SetCookies(cookieUrl, cookiesExpression);
+                }
+            }
+        }
+
+        /// <summary>
+        /// 解析页面
+        /// </summary>
+        /// <param name="response"></param>
+        private string ParseContent(Stream stream, string responseEncode)
+        {
+            if (_settings.Timeout > 0)
+                stream.ReadTimeout = _settings.Timeout;
+            MemoryStream ms = new MemoryStream();
+            stream.CopyTo(ms);
+            byte[] buffer = ms.ToArray();
+
+            Encoding encode = Encoding.ASCII;
+            string html = encode.GetString(buffer);
+            string cs = responseEncode;
+            Match match = Regex.Match(html, "<meta([^<]*)charset=([^<]*)\"", RegexOptions.IgnoreCase);
+            if (match.Success)
+            {
+                cs = match.Groups[2].Value;
+                cs = cs.Replace("\"", "");
+            }
+
+            if (String.IsNullOrEmpty(cs))
+                cs = responseEncode;
+            if (!String.IsNullOrEmpty(cs))
+                encode = Encoding.GetEncoding(cs);
+
+            return encode.GetString(buffer);
+        }
+
+        /// <summary>
+        /// 解析页面链接
+        /// </summary>
+        /// <param name="html"></param>
+        private void ParseLinks(UrlInfo currentUrl, string html)
+        {
+            if (_settings.CrawlDepth > 0 && currentUrl.Depth >= _settings.CrawlDepth)
+                return;
+
+            HtmlDocument doc = new HtmlDocument();
+            doc.LoadHtml(html);
+            HtmlNodeCollection linkNodes = doc.DocumentNode.SelectNodes("//a");
+            if (linkNodes == null)
+                return;
+            foreach (var linkNode in linkNodes)
+            {
+                HtmlAttribute href = linkNode.Attributes["href"];
+                string linkText = linkNode.InnerText;
+                if (href != null)
+                {
+                    bool canBeAdd = true;
+                    if (_escapeLinks != null)
+                    {
+                        foreach (var node in _escapeLinks)
+                        {
+                            if (href.Value.EndsWith(node, StringComparison.OrdinalIgnoreCase))
+                            {
+                                canBeAdd = false;
+                                break;
+                            }
+                        }                       
+                    }
+                    if (_keywords != null)
+                    {
+                        if (!_keywords.Any(t => linkText.Contains(t)))
+                            canBeAdd = false;
+                    }
+
+                    if (canBeAdd)
+                    {
+                        string url = href.Value
+                            .Replace("%3f", "?")
+                            .Replace("%3d", "=")
+                            .Replace("%2f", "/")
+                            .Replace("&amp;", "&");
+
+                        if (String.IsNullOrEmpty(url) || 
+                            url.StartsWith("#") || 
+                            url.StartsWith("mailto:", StringComparison.OrdinalIgnoreCase) || 
+                            url.StartsWith("javascript:", StringComparison.OrdinalIgnoreCase))
+                        {
+                            continue;
+                        }
+
+                        Uri uri = new Uri(currentUrl.Url); 
+                        Uri thisUri = url.StartsWith("http", StringComparison.OrdinalIgnoreCase) ? new Uri(url) : new Uri(uri, url);
+                        url = GetUrlAddess(thisUri);
+                        if (_settings.LockHost)
+                        {
+                            if (uri.Host != thisUri.Host)
+                                continue;
+                        }
+
+                        if (AddUrlEvent != null && !AddUrlEvent(new AddUrlEventArgs { Title = linkText, Depth = currentUrl.Depth + 1, Url = url }))
+                            continue;
+
+                        lock (this)
+                        {
+                            // 如果队列中包含了此Url，则不会添加到队列
+                            if (_urlQueue.Any(t => t.Url == url))
+                                continue;
+                            _urlQueue.Enqueue(new UrlInfo { Url = url, Depth = currentUrl.Depth + 1 });
+                        }
+                    }
+                }
+            }
+        }
+
+        private string GetUrlAddess(Uri url)
+        {
+            return url.Scheme + "://" + url.Authority + "/" + url.LocalPath.Trim('/');
+        }
+
+    }
+}
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="HtmlAgilityPack" version="1.4.6" targetFramework="net40-Client" />
+</packages>
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<configuration>
+  <configSections>
+  </configSections>
+  <connectionStrings>
+    <!--<add name="DefaultConnection" connectionString="Data Source = 10.6.154.251; database=master;Initial Catalog=WebCrawler;user id=sa;pwd=MOcyou0543_" providerName="System.Data.SqlClient"/>-->
+    <add name="DefaultConnection" connectionString="Data Source = localhost; database=master;Initial Catalog=Test;user id=sa;pwd=MOcyou0543_" providerName="System.Data.SqlClient"/>
+  </connectionStrings>
+</configuration>
@@ -0,0 +1,52 @@
+<?xml version="1.0"?>
+<configuration>
+
+  <configSections>
+    <section name="log4net" type="System.Configuration.IgnoreSectionHandler" />
+  </configSections>
+  <!--log4net日志配置-->
+  <log4net debug="false">
+    <root>
+      <level value="INFO" />
+      <appender-ref ref="LogFileAppender" />
+      <appender-ref ref="ErrorAppender" />
+      <appender-ref ref="ConsoleAppender" />
+    </root>
+    <appender name="LogFileAppender" type="log4net.Appender.RollingFileAppender">
+      <param name="File" value="Logs/Opt" />
+      <param name="AppendToFile" value="true" />
+      <param name="RollingStyle" value="Date" />
+      <param name="DatePattern" value="yyyy_MM_dd" />
+      <layout type="log4net.Layout.PatternLayout">
+        <param name="ConversionPattern" value="%d [%t] %-5p %c - %m%n" />
+      </layout>
+    </appender>
+    <!--专门记录错误的Appender-->
+    <appender name="ErrorAppender" type="log4net.Appender.RollingFileAppender">
+      <filter type="log4net.Filter.LevelRangeFilter">
+          <param name="LevelMin" value="ERROR" />
+          <param name="LevelMax" value="ERROR" />
+      </filter>
+      <param name="File" value="Logs/Error" />
+      <param name="AppendToFile" value="true" />
+      <param name="RollingStyle" value="Date" />
+      <param name="DatePattern" value="yyyy_MM_dd" />
+      <layout type="log4net.Layout.PatternLayout">
+        <param name="ConversionPattern" value="%d [%t] %-5p %c - %m%n" />
+      </layout>
+    </appender>
+    <appender name="ConsoleAppender" type="log4net.Appender.ColoredConsoleAppender">
+      <mapping>
+        <level value="ERROR" />
+        <foreColor value="Red, HighIntensity" />
+      </mapping>
+      <mapping>
+        <level value="INFO" />
+        <foreColor value="Green, HighIntensity" />
+      </mapping>
+      <layout type="log4net.Layout.PatternLayout">
+        <conversionPattern value="[%t]%d{MM-dd hh:mm:ss}=> %m%n" />
+      </layout>
+    </appender>
+  </log4net>
+</configuration>
@@ -0,0 +1,71 @@
+using System;
+using System.Collections;
+using System.Data;
+using ScrapingSpider.Models;
+
+namespace ScrapingSpider.DataAccess
+{
+    public class WebPageDao
+    {
+        public static int SaveOrUpdateWebPage(string hashId, string url)
+        {
+            return SaveOrUpdateWebPage(new WebPage
+                                     {
+                                         Id = hashId,
+                                         Url = url,
+                                         InsertDate = DateTime.Now
+                                     });
+        }
+
+
+        /// <summary>
+        /// 保存或更新WebPage
+        /// </summary>
+        /// <param name="webPage"></param>
+        /// <returns></returns>
+        public static int SaveOrUpdateWebPage(WebPage webPage)
+        {
+            string connStr = SqlHelper.ConnectionString();
+            string tableName = webPage.GetType().Name;
+
+            Hashtable hashtable = new Hashtable();
+            foreach (var type in webPage.GetType().GetProperties())
+            {
+                var value = type.GetValue(webPage, null);
+                if (value != null)
+                    hashtable.Add(type.Name, value);
+            }
+
+            if (IsIdExisted(webPage.Id))
+            {
+                return SqlHelper.Update(connStr, tableName, hashtable, "where Id='" + webPage.Id + "'");
+            }
+
+            return SqlHelper.Insert(connStr, tableName, hashtable);
+        }
+
+        public static bool IsIdExisted(string id)
+        {
+            return IsExisted("Id='" + id + "'");
+        }
+
+        public static bool IsUrlExisted(string Url)
+        {
+            return IsExisted("Url='" + Url + "'");
+        }
+
+        /// <summary>
+        /// 判断记录是否存在
+        /// </summary>
+        /// <param name="where">查询条件，如：Url='http://www.baidu.com' and Title='百度'</param>
+        /// <returns></returns>
+        public static bool IsExisted(string where)
+        {
+            int count = (int)SqlHelper.ExecuteScalar(
+                SqlHelper.ConnectionString(), 
+                CommandType.Text,
+                "select count(*) from WebPage where " + where);
+            return count != 0;
+        }
+    }
+}
@@ -0,0 +1,83 @@
+using System;
+using System.Linq;
+using System.Text.RegularExpressions;
+using HtmlAgilityPack;
+using ScrapingSpider.Models;
+
+namespace ScrapingSpider.Extensions
+{
+    public class ArticleParse
+    {
+        private static float _maxT;
+        private static HtmlNode _articleNode;
+        private static string _publishDate;
+        private static readonly string[] EscapeNode = { "#comment", "#text", "script", "style", "title", "head", "compress" };
+        private static Regex _dateTimeRegex = new Regex(@"((\d{4}|\d{2})(\-|\/|\.)\d{1,2}\3\d{1,2})|(\d{4}年\d{1,2}月\d{1,2}日)");
+
+        public static WebPage GetArticleWebPage(string rawHtml)
+        {
+            HtmlDocument doc = new HtmlDocument();
+            doc.LoadHtml(rawHtml);
+
+            WebPage webPage = new WebPage();
+            webPage.Raw = rawHtml.Replace("'", "''");
+            var contentNode = GetArticleNode(doc.DocumentNode);
+            if (contentNode != null)
+                webPage.Content = contentNode.InnerText.Replace("'", "''");
+            var titleNode = doc.DocumentNode.SelectSingleNode("//title");
+            if (titleNode != null)
+                webPage.Title = titleNode.InnerText.Replace("'", "''");
+            // 提取发布日期
+            if (!String.IsNullOrEmpty(_publishDate))
+                webPage.PublishDate = Convert.ToDateTime(_publishDate);
+
+            return webPage;
+        }
+
+        public static HtmlNode GetArticleNode(HtmlNode root)
+        {
+            _maxT = 0.0F;
+            _articleNode = null;
+            _publishDate = null;
+            // 过滤节点内的属性值
+            HtmlDocument doc = new HtmlDocument();
+            string clearnHtml = Regex.Replace(root.OuterHtml, "(([^\\s=]+)=(\"|\')(.*?)(\"|\'))*>", ">");
+            doc.LoadHtml(clearnHtml);
+            root = doc.DocumentNode;
+            ParseAllNodes(root);
+            return _articleNode;
+        }
+
+        private static void ParseAllNodes(HtmlNode root)
+        {
+            if (EscapeNode.Any(t => t == root.Name))
+                return;
+            foreach (var childNode in root.ChildNodes)
+            {
+                if (EscapeNode.Any(t => t == childNode.Name))
+                    continue;
+
+                float currentT = childNode.InnerText.Length / (float)(childNode.OuterHtml.Length);
+                // 分析正文节点
+                if (currentT > _maxT && currentT > 0.7)
+                {
+                    _maxT = currentT;
+                    {
+                        _articleNode = childNode.ParentNode;
+                    }
+                }
+                // 分析发布日期
+                if (String.IsNullOrEmpty(_publishDate) && childNode.InnerText.Length < 30)
+                {
+                    var match = _dateTimeRegex.Match(childNode.InnerText);
+                    if (match.Success)
+                    {
+                        _publishDate = match.Result("$1");
+                    }
+                }
+                ParseAllNodes(childNode);
+            }
+        }
+
+    }
+}
@@ -0,0 +1,17 @@
+using System;
+using System.Security.Cryptography;
+using System.Text;
+
+namespace ScrapingSpider.Extensions
+{
+    public class MD5Helper
+    {
+        public static string GetMD5HashCode(string text)
+        {
+            var md5 = new MD5CryptoServiceProvider();
+            var bytes = UnicodeEncoding.Default.GetBytes(text);
+            var md5HashByte = md5.ComputeHash(bytes);
+            return BitConverter.ToInt64(md5HashByte, 0).ToString();
+        }
+    }
+}
@@ -0,0 +1,293 @@
+namespace ScrapingSpider.Forms
+{
+    partial class FrmSettings
+    {
+        /// <summary>
+        /// Required designer variable.
+        /// </summary>
+        private System.ComponentModel.IContainer components = null;
+
+        /// <summary>
+        /// Clean up any resources being used.
+        /// </summary>
+        /// <param name="disposing">true if managed resources should be disposed; otherwise, false.</param>
+        protected override void Dispose(bool disposing)
+        {
+            if (disposing && (components != null))
+            {
+                components.Dispose();
+            }
+            base.Dispose(disposing);
+        }
+
+        #region Windows Form Designer generated code
+
+        /// <summary>
+        /// Required method for Designer support - do not modify
+        /// the contents of this method with the code editor.
+        /// </summary>
+        private void InitializeComponent()
+        {
+            this.components = new System.ComponentModel.Container();
+            System.Windows.Forms.Label crawlDepthLabel;
+            System.Windows.Forms.Label escapeLinksLabel;
+            System.Windows.Forms.Label initSeedsLabel;
+            System.Windows.Forms.Label keepCookieLabel;
+            System.Windows.Forms.Label lockHostLabel;
+            System.Windows.Forms.Label threadsLabel;
+            System.Windows.Forms.Label timeoutLabel;
+            System.Windows.Forms.Label userAgentLabel;
+            System.Windows.Forms.Label label1;
+            this.crawlDepthTextBox = new System.Windows.Forms.TextBox();
+            this.escapeLinksTextBox = new System.Windows.Forms.TextBox();
+            this.initSeedsTextBox = new System.Windows.Forms.TextBox();
+            this.keepCookieCheckBox = new System.Windows.Forms.CheckBox();
+            this.lockHostCheckBox = new System.Windows.Forms.CheckBox();
+            this.threadsTextBox = new System.Windows.Forms.TextBox();
+            this.timeoutTextBox = new System.Windows.Forms.TextBox();
+            this.userAgentTextBox = new System.Windows.Forms.TextBox();
+            this.btnOK = new System.Windows.Forms.Button();
+            this.keywordsTextbox = new System.Windows.Forms.TextBox();
+            this.settingsBindingSource = new System.Windows.Forms.BindingSource(this.components);
+            crawlDepthLabel = new System.Windows.Forms.Label();
+            escapeLinksLabel = new System.Windows.Forms.Label();
+            initSeedsLabel = new System.Windows.Forms.Label();
+            keepCookieLabel = new System.Windows.Forms.Label();
+            lockHostLabel = new System.Windows.Forms.Label();
+            threadsLabel = new System.Windows.Forms.Label();
+            timeoutLabel = new System.Windows.Forms.Label();
+            userAgentLabel = new System.Windows.Forms.Label();
+            label1 = new System.Windows.Forms.Label();
+            ((System.ComponentModel.ISupportInitialize)(this.settingsBindingSource)).BeginInit();
+            this.SuspendLayout();
+            // 
+            // crawlDepthLabel
+            // 
+            crawlDepthLabel.AutoSize = true;
+            crawlDepthLabel.Location = new System.Drawing.Point(287, 33);
+            crawlDepthLabel.Name = "crawlDepthLabel";
+            crawlDepthLabel.Size = new System.Drawing.Size(77, 12);
+            crawlDepthLabel.TabIndex = 1;
+            crawlDepthLabel.Text = "Crawl Depth:";
+            // 
+            // escapeLinksLabel
+            // 
+            escapeLinksLabel.AutoSize = true;
+            escapeLinksLabel.Location = new System.Drawing.Point(287, 60);
+            escapeLinksLabel.Name = "escapeLinksLabel";
+            escapeLinksLabel.Size = new System.Drawing.Size(83, 12);
+            escapeLinksLabel.TabIndex = 3;
+            escapeLinksLabel.Text = "Escape Links:";
+            // 
+            // initSeedsLabel
+            // 
+            initSeedsLabel.AutoSize = true;
+            initSeedsLabel.Location = new System.Drawing.Point(21, 18);
+            initSeedsLabel.Name = "initSeedsLabel";
+            initSeedsLabel.Size = new System.Drawing.Size(71, 12);
+            initSeedsLabel.TabIndex = 5;
+            initSeedsLabel.Text = "Init Seeds:";
+            // 
+            // keepCookieLabel
+            // 
+            keepCookieLabel.AutoSize = true;
+            keepCookieLabel.Location = new System.Drawing.Point(287, 89);
+            keepCookieLabel.Name = "keepCookieLabel";
+            keepCookieLabel.Size = new System.Drawing.Size(77, 12);
+            keepCookieLabel.TabIndex = 7;
+            keepCookieLabel.Text = "Keep Cookie:";
+            // 
+            // lockHostLabel
+            // 
+            lockHostLabel.AutoSize = true;
+            lockHostLabel.Location = new System.Drawing.Point(287, 119);
+            lockHostLabel.Name = "lockHostLabel";
+            lockHostLabel.Size = new System.Drawing.Size(65, 12);
+            lockHostLabel.TabIndex = 9;
+            lockHostLabel.Text = "Lock Host:";
+            // 
+            // threadsLabel
+            // 
+            threadsLabel.AutoSize = true;
+            threadsLabel.Location = new System.Drawing.Point(287, 147);
+            threadsLabel.Name = "threadsLabel";
+            threadsLabel.Size = new System.Drawing.Size(53, 12);
+            threadsLabel.TabIndex = 11;
+            threadsLabel.Text = "Threads:";
+            // 
+            // timeoutLabel
+            // 
+            timeoutLabel.AutoSize = true;
+            timeoutLabel.Location = new System.Drawing.Point(287, 174);
+            timeoutLabel.Name = "timeoutLabel";
+            timeoutLabel.Size = new System.Drawing.Size(53, 12);
+            timeoutLabel.TabIndex = 13;
+            timeoutLabel.Text = "Timeout:";
+            // 
+            // userAgentLabel
+            // 
+            userAgentLabel.AutoSize = true;
+            userAgentLabel.Location = new System.Drawing.Point(287, 201);
+            userAgentLabel.Name = "userAgentLabel";
+            userAgentLabel.Size = new System.Drawing.Size(71, 12);
+            userAgentLabel.TabIndex = 15;
+            userAgentLabel.Text = "User Agent:";
+            // 
+            // crawlDepthTextBox
+            // 
+            this.crawlDepthTextBox.DataBindings.Add(new System.Windows.Forms.Binding("Text", this.settingsBindingSource, "CrawlDepth", true));
+            this.crawlDepthTextBox.Location = new System.Drawing.Point(376, 30);
+            this.crawlDepthTextBox.Name = "crawlDepthTextBox";
+            this.crawlDepthTextBox.Size = new System.Drawing.Size(104, 21);
+            this.crawlDepthTextBox.TabIndex = 2;
+            // 
+            // escapeLinksTextBox
+            // 
+            this.escapeLinksTextBox.DataBindings.Add(new System.Windows.Forms.Binding("Text", this.settingsBindingSource, "EscapeLinks", true));
+            this.escapeLinksTextBox.Location = new System.Drawing.Point(376, 57);
+            this.escapeLinksTextBox.Name = "escapeLinksTextBox";
+            this.escapeLinksTextBox.Size = new System.Drawing.Size(104, 21);
+            this.escapeLinksTextBox.TabIndex = 4;
+            // 
+            // initSeedsTextBox
+            // 
+            this.initSeedsTextBox.AcceptsReturn = true;
+            this.initSeedsTextBox.DataBindings.Add(new System.Windows.Forms.Binding("Text", this.settingsBindingSource, "InitSeeds", true));
+            this.initSeedsTextBox.Location = new System.Drawing.Point(23, 42);
+            this.initSeedsTextBox.Multiline = true;
+            this.initSeedsTextBox.Name = "initSeedsTextBox";
+            this.initSeedsTextBox.ScrollBars = System.Windows.Forms.ScrollBars.Both;
+            this.initSeedsTextBox.Size = new System.Drawing.Size(243, 104);
+            this.initSeedsTextBox.TabIndex = 6;
+            // 
+            // keepCookieCheckBox
+            // 
+            this.keepCookieCheckBox.DataBindings.Add(new System.Windows.Forms.Binding("CheckState", this.settingsBindingSource, "KeepCookie", true));
+            this.keepCookieCheckBox.Location = new System.Drawing.Point(376, 84);
+            this.keepCookieCheckBox.Name = "keepCookieCheckBox";
+            this.keepCookieCheckBox.Size = new System.Drawing.Size(104, 24);
+            this.keepCookieCheckBox.TabIndex = 8;
+            this.keepCookieCheckBox.Text = "Keep";
+            this.keepCookieCheckBox.UseVisualStyleBackColor = true;
+            // 
+            // lockHostCheckBox
+            // 
+            this.lockHostCheckBox.DataBindings.Add(new System.Windows.Forms.Binding("CheckState", this.settingsBindingSource, "LockHost", true));
+            this.lockHostCheckBox.Location = new System.Drawing.Point(376, 114);
+            this.lockHostCheckBox.Name = "lockHostCheckBox";
+            this.lockHostCheckBox.Size = new System.Drawing.Size(104, 24);
+            this.lockHostCheckBox.TabIndex = 10;
+            this.lockHostCheckBox.Text = "Lock";
+            this.lockHostCheckBox.UseVisualStyleBackColor = true;
+            // 
+            // threadsTextBox
+            // 
+            this.threadsTextBox.DataBindings.Add(new System.Windows.Forms.Binding("Text", this.settingsBindingSource, "Threads", true));
+            this.threadsTextBox.Location = new System.Drawing.Point(376, 144);
+            this.threadsTextBox.Name = "threadsTextBox";
+            this.threadsTextBox.Size = new System.Drawing.Size(104, 21);
+            this.threadsTextBox.TabIndex = 12;
+            // 
+            // timeoutTextBox
+            // 
+            this.timeoutTextBox.DataBindings.Add(new System.Windows.Forms.Binding("Text", this.settingsBindingSource, "Timeout", true));
+            this.timeoutTextBox.Location = new System.Drawing.Point(376, 171);
+            this.timeoutTextBox.Name = "timeoutTextBox";
+            this.timeoutTextBox.Size = new System.Drawing.Size(104, 21);
+            this.timeoutTextBox.TabIndex = 14;
+            // 
+            // userAgentTextBox
+            // 
+            this.userAgentTextBox.DataBindings.Add(new System.Windows.Forms.Binding("Text", this.settingsBindingSource, "UserAgent", true));
+            this.userAgentTextBox.Location = new System.Drawing.Point(376, 198);
+            this.userAgentTextBox.Name = "userAgentTextBox";
+            this.userAgentTextBox.Size = new System.Drawing.Size(104, 21);
+            this.userAgentTextBox.TabIndex = 16;
+            // 
+            // btnOK
+            // 
+            this.btnOK.Location = new System.Drawing.Point(398, 244);
+            this.btnOK.Name = "btnOK";
+            this.btnOK.Size = new System.Drawing.Size(81, 28);
+            this.btnOK.TabIndex = 17;
+            this.btnOK.Text = "开始爬取";
+            this.btnOK.UseVisualStyleBackColor = true;
+            this.btnOK.Click += new System.EventHandler(this.btnOK_Click);
+            // 
+            // keywordsTextbox
+            // 
+            this.keywordsTextbox.AcceptsReturn = true;
+            this.keywordsTextbox.DataBindings.Add(new System.Windows.Forms.Binding("Text", this.settingsBindingSource, "Keywords", true));
+            this.keywordsTextbox.Location = new System.Drawing.Point(23, 168);
+            this.keywordsTextbox.Multiline = true;
+            this.keywordsTextbox.Name = "keywordsTextbox";
+            this.keywordsTextbox.ScrollBars = System.Windows.Forms.ScrollBars.Both;
+            this.keywordsTextbox.Size = new System.Drawing.Size(243, 104);
+            this.keywordsTextbox.TabIndex = 6;
+            // 
+            // label1
+            // 
+            label1.AutoSize = true;
+            label1.Location = new System.Drawing.Point(21, 144);
+            label1.Name = "label1";
+            label1.Size = new System.Drawing.Size(59, 12);
+            label1.TabIndex = 5;
+            label1.Text = "Keywords:";
+            // 
+            // settingsBindingSource
+            // 
+            this.settingsBindingSource.AllowNew = false;
+            this.settingsBindingSource.DataSource = typeof(ScrapingSpider.Core.Models.Settings);
+            // 
+            // FrmSettings
+            // 
+            this.AcceptButton = this.btnOK;
+            this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 12F);
+            this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
+            this.ClientSize = new System.Drawing.Size(511, 293);
+            this.Controls.Add(this.btnOK);
+            this.Controls.Add(crawlDepthLabel);
+            this.Controls.Add(this.crawlDepthTextBox);
+            this.Controls.Add(escapeLinksLabel);
+            this.Controls.Add(this.escapeLinksTextBox);
+            this.Controls.Add(label1);
+            this.Controls.Add(initSeedsLabel);
+            this.Controls.Add(this.keywordsTextbox);
+            this.Controls.Add(this.initSeedsTextBox);
+            this.Controls.Add(keepCookieLabel);
+            this.Controls.Add(this.keepCookieCheckBox);
+            this.Controls.Add(lockHostLabel);
+            this.Controls.Add(this.lockHostCheckBox);
+            this.Controls.Add(threadsLabel);
+            this.Controls.Add(this.threadsTextBox);
+            this.Controls.Add(timeoutLabel);
+            this.Controls.Add(this.timeoutTextBox);
+            this.Controls.Add(userAgentLabel);
+            this.Controls.Add(this.userAgentTextBox);
+            this.FormBorderStyle = System.Windows.Forms.FormBorderStyle.FixedDialog;
+            this.MaximizeBox = false;
+            this.MinimizeBox = false;
+            this.Name = "FrmSettings";
+            this.Text = "爬虫设置";
+            this.Load += new System.EventHandler(this.FrmSettings_Load);
+            ((System.ComponentModel.ISupportInitialize)(this.settingsBindingSource)).EndInit();
+            this.ResumeLayout(false);
+            this.PerformLayout();
+
+        }
+
+        #endregion
+
+        private System.Windows.Forms.BindingSource settingsBindingSource;
+        private System.Windows.Forms.TextBox crawlDepthTextBox;
+        private System.Windows.Forms.TextBox escapeLinksTextBox;
+        private System.Windows.Forms.TextBox initSeedsTextBox;
+        private System.Windows.Forms.CheckBox keepCookieCheckBox;
+        private System.Windows.Forms.CheckBox lockHostCheckBox;
+        private System.Windows.Forms.TextBox threadsTextBox;
+        private System.Windows.Forms.TextBox timeoutTextBox;
+        private System.Windows.Forms.TextBox userAgentTextBox;
+        private System.Windows.Forms.Button btnOK;
+        private System.Windows.Forms.TextBox keywordsTextbox;
+    }
+}
@@ -0,0 +1,38 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Data;
+using System.Drawing;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Windows.Forms;
+using System.Xml.Serialization;
+using ScrapingSpider.Core.Models;
+
+namespace ScrapingSpider.Forms
+{
+    public partial class FrmSettings : Form
+    {
+        public Settings Settings { get; set; }
+
+        public FrmSettings()
+        {
+            InitializeComponent();
+        }
+
+        private void FrmSettings_Load(object sender, EventArgs e)
+        {
+            Settings = Properties.Settings.Default.SpiderSettings ?? new Settings();
+            settingsBindingSource.DataSource = Settings;
+        }
+
+        private void btnOK_Click(object sender, EventArgs e)
+        {
+            this.settingsBindingSource.EndEdit();
+            Properties.Settings.Default.SpiderSettings = Settings;
+            Properties.Settings.Default.Save();
+            this.DialogResult = DialogResult.OK;
+        }
+    }
+}
@@ -0,0 +1,150 @@
+<?xml version="1.0" encoding="utf-8"?>
+<root>
+  <!-- 
+    Microsoft ResX Schema 
+    
+    Version 2.0
+    
+    The primary goals of this format is to allow a simple XML format 
+    that is mostly human readable. The generation and parsing of the 
+    various data types are done through the TypeConverter classes 
+    associated with the data types.
+    
+    Example:
+    
+    ... ado.net/XML headers & schema ...
+    <resheader name="resmimetype">text/microsoft-resx</resheader>
+    <resheader name="version">2.0</resheader>
+    <resheader name="reader">System.Resources.ResXResourceReader, System.Windows.Forms, ...</resheader>
+    <resheader name="writer">System.Resources.ResXResourceWriter, System.Windows.Forms, ...</resheader>
+    <data name="Name1"><value>this is my long string</value><comment>this is a comment</comment></data>
+    <data name="Color1" type="System.Drawing.Color, System.Drawing">Blue</data>
+    <data name="Bitmap1" mimetype="application/x-microsoft.net.object.binary.base64">
+        <value>[base64 mime encoded serialized .NET Framework object]</value>
+    </data>
+    <data name="Icon1" type="System.Drawing.Icon, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
+        <value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
+        <comment>This is a comment</comment>
+    </data>
+                
+    There are any number of "resheader" rows that contain simple 
+    name/value pairs.
+    
+    Each data row contains a name, and value. The row also contains a 
+    type or mimetype. Type corresponds to a .NET class that support 
+    text/value conversion through the TypeConverter architecture. 
+    Classes that don't support this are serialized and stored with the 
+    mimetype set.
+    
+    The mimetype is used for serialized objects, and tells the 
+    ResXResourceReader how to depersist the object. This is currently not 
+    extensible. For a given mimetype the value must be set accordingly:
+    
+    Note - application/x-microsoft.net.object.binary.base64 is the format 
+    that the ResXResourceWriter will generate, however the reader can 
+    read any of the formats listed below.
+    
+    mimetype: application/x-microsoft.net.object.binary.base64
+    value   : The object must be serialized with 
+            : System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
+            : and then encoded with base64 encoding.
+    
+    mimetype: application/x-microsoft.net.object.soap.base64
+    value   : The object must be serialized with 
+            : System.Runtime.Serialization.Formatters.Soap.SoapFormatter
+            : and then encoded with base64 encoding.
+
+    mimetype: application/x-microsoft.net.object.bytearray.base64
+    value   : The object must be serialized into a byte array 
+            : using a System.ComponentModel.TypeConverter
+            : and then encoded with base64 encoding.
+    -->
+  <xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
+    <xsd:import namespace="http://www.w3.org/XML/1998/namespace" />
+    <xsd:element name="root" msdata:IsDataSet="true">
+      <xsd:complexType>
+        <xsd:choice maxOccurs="unbounded">
+          <xsd:element name="metadata">
+            <xsd:complexType>
+              <xsd:sequence>
+                <xsd:element name="value" type="xsd:string" minOccurs="0" />
+              </xsd:sequence>
+              <xsd:attribute name="name" use="required" type="xsd:string" />
+              <xsd:attribute name="type" type="xsd:string" />
+              <xsd:attribute name="mimetype" type="xsd:string" />
+              <xsd:attribute ref="xml:space" />
+            </xsd:complexType>
+          </xsd:element>
+          <xsd:element name="assembly">
+            <xsd:complexType>
+              <xsd:attribute name="alias" type="xsd:string" />
+              <xsd:attribute name="name" type="xsd:string" />
+            </xsd:complexType>
+          </xsd:element>
+          <xsd:element name="data">
+            <xsd:complexType>
+              <xsd:sequence>
+                <xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
+                <xsd:element name="comment" type="xsd:string" minOccurs="0" msdata:Ordinal="2" />
+              </xsd:sequence>
+              <xsd:attribute name="name" type="xsd:string" use="required" msdata:Ordinal="1" />
+              <xsd:attribute name="type" type="xsd:string" msdata:Ordinal="3" />
+              <xsd:attribute name="mimetype" type="xsd:string" msdata:Ordinal="4" />
+              <xsd:attribute ref="xml:space" />
+            </xsd:complexType>
+          </xsd:element>
+          <xsd:element name="resheader">
+            <xsd:complexType>
+              <xsd:sequence>
+                <xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
+              </xsd:sequence>
+              <xsd:attribute name="name" type="xsd:string" use="required" />
+            </xsd:complexType>
+          </xsd:element>
+        </xsd:choice>
+      </xsd:complexType>
+    </xsd:element>
+  </xsd:schema>
+  <resheader name="resmimetype">
+    <value>text/microsoft-resx</value>
+  </resheader>
+  <resheader name="version">
+    <value>2.0</value>
+  </resheader>
+  <resheader name="reader">
+    <value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+  </resheader>
+  <resheader name="writer">
+    <value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+  </resheader>
+  <metadata name="crawlDepthLabel.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+  <metadata name="escapeLinksLabel.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+  <metadata name="initSeedsLabel.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+  <metadata name="keepCookieLabel.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+  <metadata name="lockHostLabel.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+  <metadata name="threadsLabel.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+  <metadata name="timeoutLabel.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+  <metadata name="userAgentLabel.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+  <metadata name="settingsBindingSource.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
+    <value>17, 17</value>
+  </metadata>
+  <metadata name="label1.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+</root>
@@ -0,0 +1,20 @@
+//============================================================
+//
+//    Copyright (C) 2012 翟士丹@CYOU●监控部●CYINC.MOMain 版权所有
+//    文件名　：Log4netFactory.cs
+//    功能描述：基于Log4net的Logger工厂
+//    创建标识：StanZhai 2012/07/19
+//    文件版本：1.0.0.0
+//
+//============================================================
+
+namespace ScrapingSpider.Logging
+{
+    public class Log4netFactory
+    {
+        public static ScrapingSpider.Core.Logging.ILogger CreateLogger()
+        {
+            return Log4netLogger.Instance;
+        }
+    }
+}
@@ -0,0 +1,87 @@
+//============================================================
+//
+//    Copyright (C) 2012 翟士丹@CYOU●监控部●CYINC.MOMain 版权所有
+//    文件名　：Log4netLogger.cs
+//    功能描述：日志记录类，王永浩提供
+//    创建标识：StanZhai 2012/07/18
+//    文件版本：1.0.0.0
+//
+//============================================================
+
+using System;
+using ScrapingSpider.Core.Logging;
+
+namespace ScrapingSpider.Logging
+{
+    /// <summary>
+    /// Controller日志辅助
+    /// </summary>
+    /// <author>inetfuture@qq.com</author>
+    public class Log4netLogger : ILogger
+    {
+        private Log4netLogger()
+        {
+            //do nothing;
+        }
+
+        private readonly log4net.ILog _log = log4net.LogManager.GetLogger("Controller");
+        private static ILogger _instance;
+
+        // 单例模式，获取唯一示例
+        public static ILogger Instance {
+            get
+            {
+                if (_instance == null)
+                {
+                    // 初始化配置工作由网站或应用程序初始化
+                    _instance = new Log4netLogger();
+                }
+                return _instance;
+            }
+        }
+
+        /// <summary>
+        /// 添加Info级别日志
+        /// </summary>
+        /// <param name="msg">日志信息</param>
+        public void Info(string msg)
+        {
+            _log.InfoFormat(msg);
+        }
+
+        /// <summary>
+        /// 添加WARN级别日志
+        /// </summary>
+        /// <param name="msg">日志信息</param>
+        public void Warn(string msg)
+        {
+            _log.WarnFormat(msg);
+        }
+
+        
+        /// <summary>
+        /// 添加ERROR级别日志
+        /// </summary>
+        /// <param name="msg">日志信息</param>
+        public void Error(string msg)
+        {
+            _log.ErrorFormat(msg);
+        }
+
+        /// <summary>
+        /// 添加ERROR级别日志，记录异常信息
+        /// </summary>
+        /// <param name="ex">异常实例</param>
+        public void Error(Exception ex)
+        {
+            try
+            {
+                _log.Error(string.Format(ex.Message));
+            }
+            catch (Exception)
+            {
+
+            }
+        }
+    }
+}
@@ -0,0 +1,46 @@
+using System;
+
+namespace ScrapingSpider.Models
+{
+    /// <summary>
+    /// 网页结构化数据模型
+    /// </summary>
+    public class WebPage
+    {
+        private DateTime _publishDate = new DateTime(1990, 1, 1);
+
+        public string Id { get; set; }
+
+        public string Url { get; set; }
+
+        /// <summary>
+        /// Url相对于种子地址的深度
+        /// </summary>
+        public int Depth { get; set; }
+
+        /// <summary>
+        /// Html原始内容
+        /// </summary>
+        public string Raw { get; set; }
+
+        /// <summary>
+        /// 解析后的正文内容
+        /// </summary>
+        public string Content { get; set; }
+
+        public string Title { get; set; }
+
+        public DateTime PublishDate
+        {
+            get { return _publishDate; }
+            set { _publishDate = value; }
+        }
+
+        public DateTime InsertDate { get; set; }
+
+        /// <summary>
+        /// 状态：0，尚未下载，1，已下载
+        /// </summary>
+        public short Status { get; set; }
+    }
+}
@@ -0,0 +1,41 @@
+using System.IO;
+using System.Windows.Forms;
+using ScrapingSpider.DataAccess;
+using ScrapingSpider.Extensions;
+using ScrapingSpider.Forms;
+using ScrapingSpider.Models;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using ScrapingSpider.Core;
+using ScrapingSpider.Logging;
+
+namespace ScrapingSpider
+{
+    class Program
+    {
+        static void Main(string[] args)
+        {
+            MD5Helper.GetMD5HashCode("zsd");
+            // 初始化log4net
+            log4net.Config.XmlConfigurator.Configure(new FileInfo(Path.Combine(Application.StartupPath, "Config", "log4net.config")));
+            FrmSettings frmSettings = new FrmSettings();
+            if (frmSettings.ShowDialog() == DialogResult.OK)
+            {
+                Spider spider = new Spider(frmSettings.Settings, Log4netFactory.CreateLogger());
+                //spider.DataReceivedEvent += receivedArgs =>
+                //{
+                //    WebPage webPage = ArticleParse.GetArticleWebPage(receivedArgs.Html);
+                //    webPage.Id = MD5Helper.GetMD5HashCode(receivedArgs.Url);
+                //    webPage.Url = receivedArgs.Url;
+                //    webPage.Depth = receivedArgs.Depth;
+                //    webPage.InsertDate = DateTime.Now;
+                //    webPage.Status = 1;
+                //    WebPageDao.SaveOrUpdateWebPage(webPage);
+                //};
+                spider.Crawl();
+            }
+        }
+    }
+}
@@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// 有关程序集的常规信息通过以下
+// 特性集控制。更改这些特性值可修改
+// 与程序集关联的信息。
+[assembly: AssemblyTitle("ScrapingSpider")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("ScrapingSpider")]
+[assembly: AssemblyCopyright("Copyright ©  2012")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// 将 ComVisible 设置为 false 使此程序集中的类型
+// 对 COM 组件不可见。如果需要从 COM 访问此程序集中的类型，
+// 则将该类型上的 ComVisible 特性设置为 true。
+[assembly: ComVisible(false)]
+
+// 如果此项目向 COM 公开，则下列 GUID 用于类型库的 ID
+[assembly: Guid("04c94658-fa36-43e1-8658-693a9f3a30b4")]
+
+// 程序集的版本信息由下面四个值组成:
+//
+//      主版本
+//      次版本 
+//      内部版本号
+//      修订号
+//
+// 可以指定所有这些值，也可以使用“内部版本号”和“修订号”的默认值，
+// 方法是按如下所示使用“*”:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+    This file is automatically generated by Visual Studio .Net. It is 
+    used to store generic object data source configuration information.  
+    Renaming the file extension or editing the content of this file may   
+    cause the file to be unrecognizable by the program.
+-->
+<GenericObjectDataSource DisplayName="Settings" Version="1.0" xmlns="urn:schemas-microsoft-com:xml-msdatasource">
+   <TypeInfo>ScrapingSpider.Core.Models.Settings, ScrapingSpider.Core, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null</TypeInfo>
+</GenericObjectDataSource>
@@ -0,0 +1,37 @@
+//------------------------------------------------------------------------------
+// <auto-generated>
+//     此代码由工具生成。
+//     运行时版本:4.0.30319.17929
+//
+//     对此文件的更改可能会导致不正确的行为，并且如果
+//     重新生成代码，这些更改将会丢失。
+// </auto-generated>
+//------------------------------------------------------------------------------
+
+namespace ScrapingSpider.Properties {
+    
+    
+    [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()]
+    [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.Editors.SettingsDesigner.SettingsSingleFileGenerator", "11.0.0.0")]
+    internal sealed partial class Settings : global::System.Configuration.ApplicationSettingsBase {
+        
+        private static Settings defaultInstance = ((Settings)(global::System.Configuration.ApplicationSettingsBase.Synchronized(new Settings())));
+        
+        public static Settings Default {
+            get {
+                return defaultInstance;
+            }
+        }
+        
+        [global::System.Configuration.UserScopedSettingAttribute()]
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
+        public global::ScrapingSpider.Core.Models.Settings SpiderSettings {
+            get {
+                return ((global::ScrapingSpider.Core.Models.Settings)(this["SpiderSettings"]));
+            }
+            set {
+                this["SpiderSettings"] = value;
+            }
+        }
+    }
+}
@@ -0,0 +1,9 @@
+<?xml version='1.0' encoding='utf-8'?>
+<SettingsFile xmlns="http://schemas.microsoft.com/VisualStudio/2004/01/settings" CurrentProfile="(Default)" GeneratedClassNamespace="ScrapingSpider.Properties" GeneratedClassName="Settings">
+  <Profiles />
+  <Settings>
+    <Setting Name="SpiderSettings" Type="ScrapingSpider.Core.Models.Settings" Scope="User">
+      <Value Profile="(Default)" />
+    </Setting>
+  </Settings>
+</SettingsFile>
@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">x86</Platform>
+    <ProductVersion>8.0.30703</ProductVersion>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{947A6286-6AAB-4AEE-9AF0-D4474EAF1876}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>ScrapingSpider</RootNamespace>
+    <AssemblyName>ScrapingSpider</AssemblyName>
+    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
+    <TargetFrameworkProfile>Client</TargetFrameworkProfile>
+    <FileAlignment>512</FileAlignment>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
+    <RestorePackages>true</RestorePackages>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x86' ">
+    <PlatformTarget>x86</PlatformTarget>
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x86' ">
+    <PlatformTarget>x86</PlatformTarget>
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup>
+    <StartupObject />
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="HtmlAgilityPack">
+      <HintPath>..\..\packages\HtmlAgilityPack.1.4.6\lib\Net40-client\HtmlAgilityPack.dll</HintPath>
+    </Reference>
+    <Reference Include="log4net">
+      <HintPath>..\..\packages\log4net.2.0.0\lib\net40-client\log4net.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.configuration" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Drawing" />
+    <Reference Include="System.Windows.Forms" />
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Data.DataSetExtensions" />
+    <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="DataAccess\SQLHelper.cs" />
+    <Compile Include="DataAccess\WebPageDao.cs" />
+    <Compile Include="Extensions\ArticleParse.cs" />
+    <Compile Include="Extensions\MD5Helper.cs" />
+    <Compile Include="Forms\FrmSettings.cs">
+      <SubType>Form</SubType>
+    </Compile>
+    <Compile Include="Forms\FrmSettings.Designer.cs">
+      <DependentUpon>FrmSettings.cs</DependentUpon>
+    </Compile>
+    <Compile Include="Logging\Log4netFactory.cs" />
+    <Compile Include="Logging\Log4netLogger.cs" />
+    <Compile Include="Models\WebPage.cs" />
+    <Compile Include="Program.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Properties\Settings.Designer.cs">
+      <AutoGen>True</AutoGen>
+      <DesignTimeSharedInput>True</DesignTimeSharedInput>
+      <DependentUpon>Settings.settings</DependentUpon>
+    </Compile>
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="App.config" />
+    <None Include="Config\log4net.config">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Include="packages.config" />
+    <None Include="Properties\DataSources\ScrapingSpider.Models.Settings.datasource" />
+    <None Include="Properties\Settings.settings">
+      <Generator>SettingsSingleFileGenerator</Generator>
+      <LastGenOutput>Settings.Designer.cs</LastGenOutput>
+    </None>
+  </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="Forms\FrmSettings.resx">
+      <DependentUpon>FrmSettings.cs</DependentUpon>
+    </EmbeddedResource>
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\ScrapingSpider.Core\ScrapingSpider.Core.csproj">
+      <Project>{636e94a7-2f32-47df-b358-7f9e1ea8ba0d}</Project>
+      <Name>ScrapingSpider.Core</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="$(SolutionDir)\.nuget\nuget.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>
@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="HtmlAgilityPack" version="1.4.6" targetFramework="net40-Client" />
+  <package id="log4net" version="2.0.0" targetFramework="net40-Client" />
+</packages>