GithubStar/ToolGood.Words
1
0
Fork 0
mirror of https://github.com/toolgood/ToolGood.Words.git synced 2026-10-24 17:30:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

1.2.0版本, 搜索改进, 速度加快N倍

Browse Source
This commit is contained in:
toolgood
2016-12-24 22:47:24 +08:00
parent 511e2e9d24
commit 4e2b95e35f
11 changed files with 601 additions and 873 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.MD
+39
View File
@@ -52,4 +52,43 @@ ToolGood.Words
Assert.AreEqual("all", all[1].SearchString);
```
字符搜索优化`StringSearch`
TextSearch : 184ms
TrieFilter : 175ms
FastFilter : 106ms
StringSearch : 32ms
----------------------- Find All -----------------------------------
TextSearch : 1,572ms
TrieFilter : 1,360ms
FastFilter : 308ms
StringSearch : 376ms
注:`FastFilter`来源: http://www.cnblogs.com/yeerh/archive/2011/10/20/2219035.html
注: 在 Find All测试中,`FastFilter`只能检测出7个:
[0]: "主席"
[1]: "赵洪祝"
[2]: "中国"
[3]: "铁道部"
[4]: "党"
[5]: "胡锦涛"
[6]: "倒台"
`StringSearch`检测出14个:
[0]: "党"
[1]: "党委"
[2]: "西藏"
[3]: "党"
[4]: "党委"
[5]: "主席"
[6]: "赵洪祝"
[7]: "中国"
[8]: "铁道部"
[9]: "党"
[10]: "胡锦涛"
[11]: "锦涛"
[12]: "倒台"
[13]: "黑社会"
ToolGood.Words.Contrast/Program.cs
+4 -23
View File
@@ -11,43 +11,28 @@ namespace ToolGood.Words.Contrast
{
class Program
{
//static HashFilter hf = new HashFilter();
static TrieSearch tf = new TrieSearch();
//static TrieSearch2 ts1 = new TrieSearch2();
static TrieFilter tf1 = new TrieFilter();
static FastFilter ff = new FastFilter();
static StringSearch word = new StringSearch();
static TextSearch search;
//static StringSearch stringSearch;
//static IllegalWords words_1 = new IllegalWords();
//static IllegalWords words_1_quick;
//static IllegalWords words_2;
//static IllegalWords words_2_quick;
static void Main(string[] args)
{
ReadBadWord();
var text = File.ReadAllText("Talk.txt");
//ts1.HasBadWord(text);
Run("TrieSearch", () => { tf.HasBadWord(text); });
//Run("TrieSearch 1 ", () => { ts1.HasBadWord(text); });
Run("TextSearch ", () => { search.ContainsAny(text); });
Run("TrieFilter", () => { tf1.HasBadWord(text); });
Run("FastFilter", () => { ff.HasBadWord(text); });
Run("StringSearch ", () => { word.ContainsAny(text); });
Console.WriteLine("----------------------- Find All -----------------------------------");
Run("TrieSearch", () => { tf.FindAll(text); });
//Run("TrieSearch 1 ", () => { ts1.FindAll(text); });
Run("TextSearch ", () => { search.FindAll(text); });
Run("TrieFilter", () => { tf1.FindAll(text); });
Run("FastFilter", () => { ff.FindAll(text); });
Run("StringSearch ", () => { word.FindAll(text); });
Console.ReadKey();
@@ -67,16 +52,12 @@ namespace ToolGood.Words.Contrast
static void ReadBadWord()
{
//stringSearch = new StringSearch();
List<string> list = new List<string>();
using (StreamReader sw = new StreamReader(File.OpenRead("BadWord.txt"))) {
string key = sw.ReadLine();
while (key != null) {
if (key != string.Empty) {
//hf.AddKey(key);
tf.AddKey(key);
tf1.AddKey(key);
//ts1.AddKey(key);
ff.AddKey(key);
@@ -87,7 +68,7 @@ namespace ToolGood.Words.Contrast
}
search = new TextSearch();
search.Keywords = list.ToArray();
word.SetKeywords(list);
}
ToolGood.Words/FastSearch.cs
-196
View File
@@ -1,196 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Collections;
using Number = System.UInt16;
using HashSet = ToolGood.Words.HashStringSet;
namespace ToolGood.Words
{
/// <summary>
/// 使用Hash优化算法
/// </summary>
public class FastSearch
{
const int CharCount = char.MaxValue + 1;
private int maxWordLength = 0;
private int minWordLength = int.MaxValue;
private HashSet m_hashSet = new HashSet();
private Number[] m_fastCheck = new Number[CharCount];
private Number[] m_startLength = new Number[CharCount];
private Number[] m_endLength = new Number[CharCount];
public void AddKey(string word)
{
const int maxLen = sizeof(Number) * 8;
if (word.Length > maxLen)
{
throw new Exception("参数最大" + maxLen + "个字符");
}
maxWordLength = Math.Max(maxWordLength, word.Length);
minWordLength = Math.Min(minWordLength, word.Length);
//字符出现的位置(1-16),
for (int i = 0; i < word.Length; i++)
{
m_fastCheck[word[i]] |= (Number)(1 << i);
}
Number mask = (Number)(1 << (word.Length - 1));
//以x开始的字符的长度
m_startLength[word[0]] |= mask;
//以y结束的字符的长度
m_endLength[word[word.Length - 1]] |= mask;
m_hashSet.Add(word);
}
public bool HasBadWord(string text)
{
for (int index = 0; index < text.Length; index++)
{
int count = 0;
int maxIndex = Math.Min(maxWordLength + index, text.Length);
char begin = text[index];
for (int j = index; j < maxIndex; j++)
{
char current = text[j];
Number mask = (Number)(1 << count);
//先判断字符出现的位置是否匹配
if ((m_fastCheck[current] & mask) == 0)
{
if (count > 1)
{
index += (count - 1);
}
break;
}
++count;
//再判断尾字符和首字符的长度是否有匹配.
if ((m_endLength[current] & mask) != 0 && (m_startLength[begin] & mask) != 0)
{
//进行hash比较
if (m_hashSet.Contains(text, index, count))
{
return true;
}
}
}
}
return false;
}
public string FindFirst(string text)
{
for (int index = 0; index < text.Length; index++)
{
int count = 0;
int maxIndex = Math.Min(maxWordLength + index, text.Length);
char begin = text[index];
for (int j = index; j < maxIndex; j++)
{
char current = text[j];
Number mask = (Number)(1 << count);
//先判断字符出现的位置是否匹配
if ((m_fastCheck[current] & mask) == 0)
{
if (count > 1)
{
index += (count - 1);
}
break;
}
++count;
//再判断尾字符和首字符的长度是否有匹配.
if ((m_endLength[current] & mask) != 0 && (m_startLength[begin] & mask) != 0)
{
//进行hash比较
if (m_hashSet.Contains(text, index, count))
{
return text.Substring(index, count);
}
}
}
}
return string.Empty;
}
public List<string> FindAll(string text)
{
List<string> result = new List<string>();
for (int index = 0; index < text.Length; index++)
{
int count = 0;
int maxIndex = Math.Min(maxWordLength + index, text.Length);
char begin = text[index];
for (int j = index; j < maxIndex; j++)
{
char current = text[j];
Number mask = (Number)(1 << count);
if ((m_fastCheck[current] & mask) == 0)
{
if (count > 1)
{
index += (count - 1);
}
break;
}
++count;
if ((m_endLength[current] & mask) != 0 && (m_startLength[begin] & mask) != 0)
{
if (m_hashSet.Contains(text, index, count))
{
result.Add(text.Substring(index, count));
index += (count - 1);
break;
}
}
}
}
return result;
}
public string Replace(string text, char maskChar = '*')
{
char[] chars = null;
for (int index = 0; index < text.Length; index++)
{
int count = 0;
int maxIndex = Math.Min(maxWordLength + index, text.Length);
char begin = text[index];
for (int j = index; j < maxIndex; j++)
{
char current = text[j];
Number mask = (Number)(1 << count);
if ((m_fastCheck[current] & mask) == 0)
{
if (count > 1)
{
index += (count - 1);
}
break;
}
++count;
if ((m_endLength[current] & mask) != 0 && (m_startLength[begin] & mask) != 0)
{
if (m_hashSet.Contains(text, index, count))
{
if (chars == null) chars = text.ToArray();
for (int i = index; i < index + count; i++)
{
chars[i] = maskChar;
}
index += (count - 1);
break;
}
}
}
}
return chars == null ? text : new string(chars);
}
}
}
ToolGood.Words/IllegalWordDetection.cs
+204 -204
View File
@@ -1,233 +1,233 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
//using System;
//using System.Collections;
//using System.Collections.Generic;
//using System.Linq;
//using System.Text;
namespace ToolGood.Words
{
public class IllegalWordDetection
{
/// <summary>
/// 存了所有的长度大于1的敏感词汇
/// </summary>
static HashSet<string> wordsSet = new HashSet<string>();
/// <summary>
/// 存了某一个词在所有敏感词中的位置，（超出8个的截断为第8个位置）
/// </summary>
static byte[] fastCheck = new byte[char.MaxValue];
/// <summary>
/// 存了所有敏感词的长度信息，“Key”值为所有敏感词的第一个词，敏感词的长度会截断为8
/// </summary>
static byte[] fastLength = new byte[char.MaxValue];
/// <summary>
/// 保有所有敏感词汇的第一个词的记录，可用来判断是否一个词是一个或者多个敏感词汇的“第一个词”，且可判断以某一个词作为第一个词的一系列的敏感词的最大的长度
/// </summary>
static byte[] startCache = new byte[char.MaxValue];
static char[] dectectedBuffer = null;
const string SkipList = " \t\r\n~!@#$%^&*()_+-=【】、{}|;':\"，。、《》？αβγδεζηθικλμνξοπρστυφχψωΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩ。，、；：？！…—·ˉ¨‘’“”々～‖∶＂＇｀｜〃〔〕〈〉《》「」『』．〖〗【】（）［］｛｝ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ⒈⒉⒊⒋⒌⒍⒎⒏⒐⒑⒒⒓⒔⒕⒖⒗⒘⒙⒚⒛㈠㈡㈢㈣㈤㈥㈦㈧㈨㈩①②③④⑤⑥⑦⑧⑨⑩⑴⑵⑶⑷⑸⑹⑺⑻⑼⑽⑾⑿⒀⒁⒂⒃⒄⒅⒆⒇≈≡≠＝≤≥＜＞≮≯∷±＋－×÷／∫∮∝∞∧∨∑∏∪∩∈∵∴⊥∥∠⌒⊙≌∽√§№☆★○●◎◇◆□℃‰€■△▲※→←↑↓〓¤°＃＆＠＼︿＿￣―♂♀┌┍┎┐┑┒┓─┄┈├┝┞┟┠┡┢┣│┆┊┬┭┮┯┰┱┲┳┼┽┾┿╀╁╂╃└┕┖┗┘┙┚┛━┅┉┤┥┦┧┨┩┪┫┃┇┋┴┵┶┷┸┹┺┻╋╊╉╈╇╆╅╄";
static BitArray SkipBitArray = new BitArray(char.MaxValue);
/// <summary>
/// 保有所有敏感词汇的最后一个词的记录，仅用来判断是否一个词是一个或者多个敏感词汇的“最后一个词”
/// </summary>
static BitArray endCache = new BitArray(char.MaxValue);
//namespace ToolGood.Words
//{
// public class IllegalWordDetection
// {
// /// <summary>
// /// 存了所有的长度大于1的敏感词汇
// /// </summary>
// static HashSet<string> wordsSet = new HashSet<string>();
// /// <summary>
// /// 存了某一个词在所有敏感词中的位置，（超出8个的截断为第8个位置）
// /// </summary>
// static byte[] fastCheck = new byte[char.MaxValue];
// /// <summary>
// /// 存了所有敏感词的长度信息，“Key”值为所有敏感词的第一个词，敏感词的长度会截断为8
// /// </summary>
// static byte[] fastLength = new byte[char.MaxValue];
// /// <summary>
// /// 保有所有敏感词汇的第一个词的记录，可用来判断是否一个词是一个或者多个敏感词汇的“第一个词”，且可判断以某一个词作为第一个词的一系列的敏感词的最大的长度
// /// </summary>
// static byte[] startCache = new byte[char.MaxValue];
// static char[] dectectedBuffer = null;
// const string SkipList = " \t\r\n~!@#$%^&*()_+-=【】、{}|;':\"，。、《》？αβγδεζηθικλμνξοπρστυφχψωΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩ。，、；：？！…—·ˉ¨‘’“”々～‖∶＂＇｀｜〃〔〕〈〉《》「」『』．〖〗【】（）［］｛｝ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ⒈⒉⒊⒋⒌⒍⒎⒏⒐⒑⒒⒓⒔⒕⒖⒗⒘⒙⒚⒛㈠㈡㈢㈣㈤㈥㈦㈧㈨㈩①②③④⑤⑥⑦⑧⑨⑩⑴⑵⑶⑷⑸⑹⑺⑻⑼⑽⑾⑿⒀⒁⒂⒃⒄⒅⒆⒇≈≡≠＝≤≥＜＞≮≯∷±＋－×÷／∫∮∝∞∧∨∑∏∪∩∈∵∴⊥∥∠⌒⊙≌∽√§№☆★○●◎◇◆□℃‰€■△▲※→←↑↓〓¤°＃＆＠＼︿＿￣―♂♀┌┍┎┐┑┒┓─┄┈├┝┞┟┠┡┢┣│┆┊┬┭┮┯┰┱┲┳┼┽┾┿╀╁╂╃└┕┖┗┘┙┚┛━┅┉┤┥┦┧┨┩┪┫┃┇┋┴┵┶┷┸┹┺┻╋╊╉╈╇╆╅╄";
// static BitArray SkipBitArray = new BitArray(char.MaxValue);
// /// <summary>
// /// 保有所有敏感词汇的最后一个词的记录，仅用来判断是否一个词是一个或者多个敏感词汇的“最后一个词”
// /// </summary>
// static BitArray endCache = new BitArray(char.MaxValue);
unsafe public static void Init(string[] badwords)
{
if (badwords == null || badwords.Length == 0)
return;
// unsafe public static void Init(string[] badwords)
// {
// if (badwords == null || badwords.Length == 0)
// return;
int wordLength = 0;
int maxWordLength = int.MinValue;
// int wordLength = 0;
// int maxWordLength = int.MinValue;
for (int stringIndex = 0, len = badwords.Length; stringIndex < len; ++stringIndex) {
if (string.IsNullOrEmpty(badwords[stringIndex]))
continue;
// for (int stringIndex = 0, len = badwords.Length; stringIndex < len; ++stringIndex) {
// if (string.IsNullOrEmpty(badwords[stringIndex]))
// continue;
string strBadWord = OriginalToLower(badwords[stringIndex]);
///求得单个的敏感词汇的长度
wordLength = strBadWord.Length;
maxWordLength = Math.Max(wordLength, maxWordLength);
// string strBadWord = OriginalToLower(badwords[stringIndex]);
// ///求得单个的敏感词汇的长度
// wordLength = strBadWord.Length;
// maxWordLength = Math.Max(wordLength, maxWordLength);
fixed (char* pWordStart = strBadWord) {
for (int i = 0; i < wordLength; ++i) {
///准确记录8位以内的敏感词汇的某个词在词汇中的“位置”
if (i < 7)
fastCheck[*(pWordStart + i)] |= (byte)(1 << i);
else///8位以外的敏感词汇的词直接限定在第8位
fastCheck[*(pWordStart + i)] |= 0x80;///0x80在内存中即为1000 0000，因为一个byte顶多标示8位，故超出8位的都位或上0x80，截断成第8位
}
// fixed (char* pWordStart = strBadWord) {
// for (int i = 0; i < wordLength; ++i) {
// ///准确记录8位以内的敏感词汇的某个词在词汇中的“位置”
// if (i < 7)
// fastCheck[*(pWordStart + i)] |= (byte)(1 << i);
// else///8位以外的敏感词汇的词直接限定在第8位
// fastCheck[*(pWordStart + i)] |= 0x80;///0x80在内存中即为1000 0000，因为一个byte顶多标示8位，故超出8位的都位或上0x80，截断成第8位
// }
///缓存敏感词汇的长度
int cachedWordslength = Math.Min(8, wordLength);
char firstWord = *pWordStart;
///记录敏感词汇的“大致长度（超出8个字的敏感词汇会被截取成8的长度）”，“key”值为敏感词汇的第一个词
fastLength[firstWord] |= (byte)(1 << (cachedWordslength - 1));
///缓存出当前以badWord第一个字开头的一系列的敏感词汇的最长的长度
if (startCache[firstWord] < cachedWordslength)
startCache[firstWord] = (byte)(cachedWordslength);
// ///缓存敏感词汇的长度
// int cachedWordslength = Math.Min(8, wordLength);
// char firstWord = *pWordStart;
// ///记录敏感词汇的“大致长度（超出8个字的敏感词汇会被截取成8的长度）”，“key”值为敏感词汇的第一个词
// fastLength[firstWord] |= (byte)(1 << (cachedWordslength - 1));
// ///缓存出当前以badWord第一个字开头的一系列的敏感词汇的最长的长度
// if (startCache[firstWord] < cachedWordslength)
// startCache[firstWord] = (byte)(cachedWordslength);
///存好敏感词汇的最后一个词汇的“出现情况”
endCache[*(pWordStart + wordLength - 1)] = true;
///将长度大于1的敏感词汇都压入到字典中
if (!wordsSet.Contains(strBadWord))
wordsSet.Add(strBadWord);
}
}
// ///存好敏感词汇的最后一个词汇的“出现情况”
// endCache[*(pWordStart + wordLength - 1)] = true;
// ///将长度大于1的敏感词汇都压入到字典中
// if (!wordsSet.Contains(strBadWord))
// wordsSet.Add(strBadWord);
// }
// }
/// 初始化好一个用来存检测到的字符串的buffer
dectectedBuffer = new char[maxWordLength];
/// 记录应该跳过的不予检测的词
fixed (char* start = SkipList) {
char* itor = start;
char* end = start + SkipList.Length;
while (itor < end) SkipBitArray[*itor++] = true;
}
}
// /// 初始化好一个用来存检测到的字符串的buffer
// dectectedBuffer = new char[maxWordLength];
// /// 记录应该跳过的不予检测的词
// fixed (char* start = SkipList) {
// char* itor = start;
// char* end = start + SkipList.Length;
// while (itor < end) SkipBitArray[*itor++] = true;
// }
// }
unsafe static string OriginalToLower(string text)
{
fixed (char* newText = text) {
char* itor = newText;
char* end = newText + text.Length;
char c;
// unsafe static string OriginalToLower(string text)
// {
// fixed (char* newText = text) {
// char* itor = newText;
// char* end = newText + text.Length;
// char c;
while (itor < end) {
c = *itor;
// while (itor < end) {
// c = *itor;
if ('A' <= c && c <= 'Z') {
*itor = (char)(c | 0x20);
}
// if ('A' <= c && c <= 'Z') {
// *itor = (char)(c | 0x20);
// }
++itor;
}
}
// ++itor;
// }
// }
return text;
}
// return text;
// }
unsafe static bool EnsuranceLower(string text)
{
fixed (char* newText = text) {
char* itor = newText;
char* end = newText + text.Length;
char c;
// unsafe static bool EnsuranceLower(string text)
// {
// fixed (char* newText = text) {
// char* itor = newText;
// char* end = newText + text.Length;
// char c;
while (itor < end) {
c = *itor;
// while (itor < end) {
// c = *itor;
if ('A' <= c && c <= 'Z') {
return true;
}
// if ('A' <= c && c <= 'Z') {
// return true;
// }
++itor;
}
}
// ++itor;
// }
// }
return false;
}
// return false;
// }
/// <summary>
/// 过滤字符串,默认遇到敏感词汇就以'*'代替
/// </summary>
/// <param name="text"></param>
/// <param name="mask"></param>
/// <returns></returns>
unsafe public static string Filter(string text, string mask = "*")
{
Dictionary<int, int> dic = DetectIllegalWords(text);
///如果没有敏感词汇，则直接返回出去
if (dic.Count == 0)
return text;
// /// <summary>
// /// 过滤字符串,默认遇到敏感词汇就以'*'代替
// /// </summary>
// /// <param name="text"></param>
// /// <param name="mask"></param>
// /// <returns></returns>
// unsafe public static string Filter(string text, string mask = "*")
// {
// Dictionary<int, int> dic = DetectIllegalWords(text);
// ///如果没有敏感词汇，则直接返回出去
// if (dic.Count == 0)
// return text;
fixed (char* newText = text, cMask = mask) {
var itor = newText;
Dictionary<int, int>.Enumerator enumerator = dic.GetEnumerator();
///开始替换敏感词汇
while (enumerator.MoveNext()) {
///偏移到敏感词出现的位置
itor = newText + enumerator.Current.Key;
for (int index = 0; index < enumerator.Current.Value; index++) {
///屏蔽掉敏感词汇
*itor++ = *cMask;
}
}
enumerator.Dispose();
}
// fixed (char* newText = text, cMask = mask) {
// var itor = newText;
// Dictionary<int, int>.Enumerator enumerator = dic.GetEnumerator();
// ///开始替换敏感词汇
// while (enumerator.MoveNext()) {
// ///偏移到敏感词出现的位置
// itor = newText + enumerator.Current.Key;
// for (int index = 0; index < enumerator.Current.Value; index++) {
// ///屏蔽掉敏感词汇
// *itor++ = *cMask;
// }
// }
// enumerator.Dispose();
// }
return text;
}
// return text;
// }
/// <summary>
/// 判断text是否有敏感词汇,如果有返回敏感的词汇的位置,利用指针操作来加快运算速度
/// </summary>
/// <param name="text"></param>
/// <returns></returns>
unsafe public static Dictionary<int, int> DetectIllegalWords(string text)
{
var findResult = new Dictionary<int, int>();
if (string.IsNullOrEmpty(text))
return findResult;
if (EnsuranceLower(text))
text = text.ToLower();
// /// <summary>
// /// 判断text是否有敏感词汇,如果有返回敏感的词汇的位置,利用指针操作来加快运算速度
// /// </summary>
// /// <param name="text"></param>
// /// <returns></returns>
// unsafe public static Dictionary<int, int> DetectIllegalWords(string text)
// {
// var findResult = new Dictionary<int, int>();
// if (string.IsNullOrEmpty(text))
// return findResult;
// if (EnsuranceLower(text))
// text = text.ToLower();
fixed (char* ptext = text, detectedStrStart = dectectedBuffer) {
///缓存字符串的初始位置
char* itor = (fastCheck[*ptext] & 0x01) == 0 ? ptext + 1 : ptext;
///缓存字符串的末尾位置
char* end = ptext + text.Length;
// fixed (char* ptext = text, detectedStrStart = dectectedBuffer) {
// ///缓存字符串的初始位置
// char* itor = (fastCheck[*ptext] & 0x01) == 0 ? ptext + 1 : ptext;
// ///缓存字符串的末尾位置
// char* end = ptext + text.Length;
while (itor < end) {
///如果text的第一个词不是敏感词汇或者当前遍历到了text第一个词的后面的词，则循环检测到text词汇的倒数第二个词，看看这一段子字符串中有没有敏感词汇
if ((fastCheck[*itor] & 0x01) == 0) {
while (itor < end - 1 && (fastCheck[*(++itor)] & 0x01) == 0) ;
}
// while (itor < end) {
// ///如果text的第一个词不是敏感词汇或者当前遍历到了text第一个词的后面的词，则循环检测到text词汇的倒数第二个词，看看这一段子字符串中有没有敏感词汇
// if ((fastCheck[*itor] & 0x01) == 0) {
// while (itor < end - 1 && (fastCheck[*(++itor)] & 0x01) == 0) ;
// }
///如果有只有一个词的敏感词，且当前的字符串的“非第一个词”满足这个敏感词，则先加入已检测到的敏感词列表
if (startCache[*itor] != 0 && (fastLength[*itor] & 0x01) > 0) {
///返回敏感词在text中的位置，以及敏感词的长度，供过滤功能用
findResult.Add((int)(itor - ptext), 1);
}
// ///如果有只有一个词的敏感词，且当前的字符串的“非第一个词”满足这个敏感词，则先加入已检测到的敏感词列表
// if (startCache[*itor] != 0 && (fastLength[*itor] & 0x01) > 0) {
// ///返回敏感词在text中的位置，以及敏感词的长度，供过滤功能用
// findResult.Add((int)(itor - ptext), 1);
// }
char* strItor = detectedStrStart;
*strItor++ = *itor;
int remainLength = (int)(end - itor - 1);
int skipCount = 0;
///此时已经检测到一个敏感词的“首词”了,记录下第一个检测到的敏感词的位置
///从当前的位置检测到字符串末尾
for (int i = 1; i <= remainLength; ++i) {
char* subItor = itor + i;
/// 跳过一些过滤的字符,比如空格特殊符号之类的
if (SkipBitArray[*subItor]) {
++skipCount;
continue;
}
///如果检测到当前的词在所有敏感词中的位置信息中没有处在第i位的，则马上跳出遍历
if ((fastCheck[*subItor] >> Math.Min(i - skipCount, 7)) == 0) {
break;
}
// char* strItor = detectedStrStart;
// *strItor++ = *itor;
// int remainLength = (int)(end - itor - 1);
// int skipCount = 0;
// ///此时已经检测到一个敏感词的“首词”了,记录下第一个检测到的敏感词的位置
// ///从当前的位置检测到字符串末尾
// for (int i = 1; i <= remainLength; ++i) {
// char* subItor = itor + i;
// /// 跳过一些过滤的字符,比如空格特殊符号之类的
// if (SkipBitArray[*subItor]) {
// ++skipCount;
// continue;
// }
// ///如果检测到当前的词在所有敏感词中的位置信息中没有处在第i位的，则马上跳出遍历
// if ((fastCheck[*subItor] >> Math.Min(i - skipCount, 7)) == 0) {
// break;
// }
*strItor++ = *subItor;
///如果有检测到敏感词的最后一个词，并且此时的“检测到的敏感词汇”的长度也符合要求，则才进一步查看检测到的敏感词汇是否是真的敏感
if ((fastLength[*itor] >> Math.Min(i - 1 - skipCount, 7)) > 0 && endCache[*subItor]) {
///如果此子字符串在敏感词字典中存在，则记录。做此判断是避免敏感词中夹杂了其他敏感词的单词，而上面的算法无法剔除，故先用hash数组来剔除
///上述算法是用于减少大部分的比较消耗
if (wordsSet.Contains(new string(dectectedBuffer, 0, (int)(strItor - detectedStrStart)))) {
int curDectectedStartIndex = (int)(itor - ptext);
findResult[curDectectedStartIndex] = i + 1;
itor = subItor;
break;
}
} else if (i - skipCount > startCache[*itor] && startCache[*itor] < 0x80)///如果超过了以该词为首的一系列的敏感词汇的最大的长度，则不继续判断(前提是该词对应的所有敏感词汇没有超过8个词的)
{
break;
}
}
++itor;
}
}
// *strItor++ = *subItor;
// ///如果有检测到敏感词的最后一个词，并且此时的“检测到的敏感词汇”的长度也符合要求，则才进一步查看检测到的敏感词汇是否是真的敏感
// if ((fastLength[*itor] >> Math.Min(i - 1 - skipCount, 7)) > 0 && endCache[*subItor]) {
// ///如果此子字符串在敏感词字典中存在，则记录。做此判断是避免敏感词中夹杂了其他敏感词的单词，而上面的算法无法剔除，故先用hash数组来剔除
// ///上述算法是用于减少大部分的比较消耗
// if (wordsSet.Contains(new string(dectectedBuffer, 0, (int)(strItor - detectedStrStart)))) {
// int curDectectedStartIndex = (int)(itor - ptext);
// findResult[curDectectedStartIndex] = i + 1;
// itor = subItor;
// break;
// }
// } else if (i - skipCount > startCache[*itor] && startCache[*itor] < 0x80)///如果超过了以该词为首的一系列的敏感词汇的最大的长度，则不继续判断(前提是该词对应的所有敏感词汇没有超过8个词的)
// {
// break;
// }
// }
// ++itor;
// }
// }
return findResult;
}
}
}
// return findResult;
// }
// }
//}
ToolGood.Words/Properties/AssemblyInfo.cs
+2 -2
View File
@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
//可以指定所有这些值，也可以使用“生成号”和“修订号”的默认值，
// 方法是按如下所示使用“*”: :
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.1.2.0")]
[assembly: AssemblyFileVersion("1.1.2.0")]
[assembly: AssemblyVersion("1.2.0.0")]
[assembly: AssemblyFileVersion("1.2.0.0")]
ToolGood.Words/StringSearch.cs
+344
View File
@@ -0,0 +1,344 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace ToolGood.Words
{
public class StringSearch
{
class TreeNode
{
#region Constructor & Methods
public TreeNode(TreeNode parent, char c)
{
_char = c; _parent = parent;
_results = new List<string>();
_transitionsAr = new List<TreeNode>();
_transHash = new Dictionary<char, TreeNode>();
}
public void AddResult(string result)
{
if (_results.Contains(result)) return;
_results.Add(result);
}
public void AddTransition(TreeNode node)
{
_transHash.Add(node.Char, node);
_transitionsAr.Add(node);
}
public TreeNode GetTransition(char c)
{
TreeNode tn;
if (_transHash.TryGetValue(c, out tn)) { return tn; }
return null;
}
public bool ContainsTransition(char c)
{
return _transHash.ContainsKey(c);
}
#endregion
#region Properties
private char _char;
private TreeNode _parent;
private TreeNode _failure;
private List<string> _results;
private List<TreeNode> _transitionsAr;
private Dictionary<char, TreeNode> _transHash;
public char Char
{
get { return _char; }
}
public TreeNode Parent
{
get { return _parent; }
}
/// <summary>
/// Failure function - descendant node
/// </summary>
public TreeNode Failure
{
get { return _failure; }
set { _failure = value; }
}
/// <summary>
/// Transition function - list of descendant nodes
/// </summary>
public List<TreeNode> Transitions
{
get { return _transitionsAr; }
}
/// <summary>
/// Returns list of patterns ending by this letter
/// </summary>
public List<string> Results
{
get { return _results; }
}
#endregion
}
class TrieNode
{
public bool End { get; set; }
public HashSet<string> Results { get; set; }
private Dictionary<char, TrieNode> m_values;
private uint minflag = uint.MaxValue;
private uint maxflag = uint.MinValue;
public TrieNode()
{
m_values = new Dictionary<char, TrieNode>();
Results = new HashSet<string>();
}
public bool TryGetValue(char c, out TrieNode node)
{
if (minflag <= (uint)c && maxflag >= (uint)c) {
return m_values.TryGetValue(c, out node);
}
node = null;
return false;
}
public void Add(TreeNode t, TrieNode node)
{
var c = t.Char;
if (m_values.ContainsKey(c) == false) {
if (minflag > c) { minflag = c; }
if (maxflag < c) { maxflag = c; }
m_values.Add(c, node);
foreach (var item in t.Results) {
node.End = true;
node.Results.Add(item);
}
}
}
public TrieNode[] ToArray()
{
TrieNode[] first = new TrieNode[char.MaxValue + 1];
foreach (var item in m_values) {
first[item.Key] = item.Value;
}
return first;
}
}
private TrieNode _root = new TrieNode();
private TrieNode[] _first = new TrieNode[char.MaxValue + 1];
#region SetKeywords
public void SetKeywords(ICollection<string> _keywords)
{
var tn = BuildTreeWithBFS(_keywords);
SimplifyTree(tn);
}
TreeNode BuildTreeWithBFS(ICollection<string> _keywords)
{
var root = new TreeNode(null, ' ');
foreach (string p in _keywords) {
string t = p;
//if (_quick) {
// t = p.ToLower();
//} else {
// t = WordHelper.ToSenseWord(p);
//}
// add pattern to tree
TreeNode nd = root;
foreach (char c in t) {
TreeNode ndNew = null;
foreach (TreeNode trans in nd.Transitions)
if (trans.Char == c) { ndNew = trans; break; }
if (ndNew == null) {
ndNew = new TreeNode(nd, c);
nd.AddTransition(ndNew);
}
nd = ndNew;
}
nd.AddResult(t);
}
List<TreeNode> nodes = new List<TreeNode>();
// Find failure functions
//ArrayList nodes = new ArrayList();
// level 1 nodes - fail to root node
foreach (TreeNode nd in root.Transitions) {
nd.Failure = root;
foreach (TreeNode trans in nd.Transitions) nodes.Add(trans);
}
// other nodes - using BFS
while (nodes.Count != 0) {
List<TreeNode> newNodes = new List<TreeNode>();
//ArrayList newNodes = new ArrayList();
foreach (TreeNode nd in nodes) {
TreeNode r = nd.Parent.Failure;
char c = nd.Char;
while (r != null && !r.ContainsTransition(c)) r = r.Failure;
if (r == null)
nd.Failure = root;
else {
nd.Failure = r.GetTransition(c);
foreach (string result in nd.Failure.Results)
nd.AddResult(result);
}
// add child nodes to BFS list
foreach (TreeNode child in nd.Transitions)
newNodes.Add(child);
}
nodes = newNodes;
}
root.Failure = root;
return root;
}
void SimplifyTree(TreeNode tn)
{
_root = new TrieNode();
Dictionary<TreeNode, TrieNode> dict = new Dictionary<TreeNode, TrieNode>();
List<TreeNode> list = new List<TreeNode>();
foreach (var item in tn.Transitions) list.Add(item);
while (list.Count > 0) {
foreach (var item in list) {
simplifyNode(item, tn, dict);
}
List<TreeNode> newNodes = new List<TreeNode>();
foreach (var item in list) {
foreach (var node in item.Transitions) {
newNodes.Add(node);
}
}
list = newNodes;
}
addNode(tn, _root, dict);
_first = _root.ToArray();
}
void addNode(TreeNode treeNode, TrieNode tridNode, Dictionary<TreeNode, TrieNode> dict)
{
foreach (var item in treeNode.Transitions) {
var node = dict[item];
tridNode.Add(item, node);
addNode(item, node, dict);
}
}
void simplifyNode(TreeNode treeNode, TreeNode root, Dictionary<TreeNode, TrieNode> dict)
{
List<TreeNode> list = new List<TreeNode>();
var tn = treeNode;
while (tn != root) {
list.Add(tn);
tn = tn.Failure;
}
TrieNode node = new TrieNode();
foreach (var item in list) {
if (dict.ContainsKey(item) == false) {
if (item.Results.Count > 0) {
dict[item] = new TrieNode();
} else {
dict[item] = node;
}
}
}
}
#endregion
public string FindFirst(string text)
{
TrieNode ptr = null;
for (int i = 0; i < text.Length; i++) {
TrieNode tn;
if (ptr==null) {
tn = _first[text[i]];
} else {
if (ptr.TryGetValue(text[i], out tn)==false) {
tn = _first[text[i]];
}
}
if (tn!=null) {
if (tn.End) {
foreach (var item in tn.Results) {
return item;
}
}
}
ptr = tn;
}
return null;
}
public List<string> FindAll(string text)
{
TrieNode ptr = null;
List<string> list = new List<string>();
for (int i = 0; i < text.Length; i++) {
TrieNode tn;
if (ptr == null) {
tn = _first[text[i]];
} else {
if (ptr.TryGetValue(text[i], out tn) == false) {
tn = _first[text[i]];
}
}
if (tn != null) {
if (tn.End) {
foreach (var item in tn.Results) {
list.Add(item);
}
}
}
ptr = tn;
}
return list;
}
public bool ContainsAny(string text)
{
TrieNode ptr = null;
for (int i = 0; i < text.Length; i++) {
TrieNode tn;
if (ptr == null) {
tn = _first[text[i]];
} else {
if (ptr.TryGetValue(text[i], out tn) == false) {
tn = _first[text[i]];
}
}
if (tn != null) {
if (tn.End) {
return true;
}
}
ptr = tn;
}
return false;
}
}
}
ToolGood.Words/TextSearch.cs
+5
View File
@@ -33,6 +33,11 @@ namespace ToolGood.Words
public string Keyword { get; private set; }
public static TextSearchResult Empty { get { return new TextSearchResult(); } }
public override string ToString()
{
return Keyword;
}
}
/// <summary>
/// 完整字符搜索
ToolGood.Words/ToolGood.Words.csproj
+3 -5
View File
@@ -20,7 +20,7 @@
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<AllowUnsafeBlocks>false</AllowUnsafeBlocks>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
@@ -29,6 +29,7 @@
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<AllowUnsafeBlocks>false</AllowUnsafeBlocks>
</PropertyGroup>
<PropertyGroup>
<SignAssembly>true</SignAssembly>
@@ -44,19 +45,16 @@
<Reference Include="System.Data" />
</ItemGroup>
<ItemGroup>
<Compile Include="FastSearch.cs" />
<Compile Include="IllegalWordDetection.cs" />
<Compile Include="IllegalWordsSearch.cs" />
<Compile Include="StringSearch.cs" />
<Compile Include="internal\Dict.cs" />
<Compile Include="internal\PinYinDict.cs" />
<Compile Include="internal\PinYinSearch.cs" />
<Compile Include="internal\TreeNode.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="TextSearch.cs" />
<Compile Include="TrieSearch.cs" />
<Compile Include="WordHelper.cs" />
<Compile Include="internal\HashHelpers.cs" />
<Compile Include="internal\HashStringSet.cs" />
</ItemGroup>
<ItemGroup>
<None Include="ToolGood.Words.snk" />
ToolGood.Words/TrieSearch.cs
-163
View File
@@ -1,163 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace ToolGood.Words
{
public class TrieSearch
{
class TrieNode
{
public bool m_end;
public Dictionary<char, TrieNode> m_values;
private uint minflag = uint.MaxValue;
private uint maxflag = uint.MinValue;
public TrieNode()
{
m_values = new Dictionary<char, TrieNode>();
}
public virtual bool TryGetValue(char c, out TrieNode node)
{
if (minflag <= (uint)c && maxflag >= (uint)c) {
return m_values.TryGetValue(c, out node);
}
node = null;
return false;
}
public TrieNode Add(char c)
{
if (minflag > c) { minflag = c; }
if (maxflag < c) { maxflag = c; }
TrieNode subnode;
if (!m_values.TryGetValue(c, out subnode)) {
subnode = new TrieNode();
m_values.Add(c, subnode);
}
return subnode;
}
}
TrieNode _root = new TrieNode();
/// <summary>
/// 添加关键字
/// </summary>
/// <param name="key"></param>
public void AddKey(string key)
{
if (string.IsNullOrEmpty(key)) {
return;
}
TrieNode node = _root;
for (int i = 0; i < key.Length; i++) {
char c = key[i];
node = node.Add(c);
}
node.m_end = true;
}
/// <summary>
/// 检查是否包含非法字符
/// </summary>
/// <param name="text">输入文本</param>
/// <returns>找到的第1个非法字符.没有则返回string.Empty</returns>
public bool HasBadWord(string text)
{
for (int head = 0; head < text.Length; head++) {
int index = head;
TrieNode node = _root;
while (node.TryGetValue(text[index], out node)) {
if (node.m_end) {
return true;
}
if (text.Length == ++index) {
break;
}
}
}
return false;
}
/// <summary>
/// 检查是否包含非法字符
/// </summary>
/// <param name="text">输入文本</param>
/// <returns>找到的第1个非法字符.没有则返回string.Empty</returns>
public string FindFirst(string text)
{
for (int head = 0; head < text.Length; head++) {
int index = head;
TrieNode node = _root;
while (node.TryGetValue(text[index], out node)) {
if (node.m_end) {
return text.Substring(head, index - head + 1);
}
if (text.Length == ++index) {
break;
}
}
}
return string.Empty;
}
/// <summary>
/// 查找所有非法字符
/// </summary>
/// <param name="text"></param>
/// <returns></returns>
public List<string> FindAll(string text)
{
List<string> result = new List<string>();
for (int head = 0; head < text.Length; head++) {
int index = head;
TrieNode node = _root;
while (node.TryGetValue(text[index], out node)) {
if (node.m_end) {
result.Add(text.Substring(head, index - head + 1));
}
if (text.Length == ++index) {
break;
}
}
}
return result;
}
/// <summary>
/// 替换非法字符
/// </summary>
/// <param name="text"></param>
/// <param name="mask">用于代替非法字符</param>
/// <returns>替换后的字符串</returns>
public string Replace(string text, char mask = '*')
{
char[] chars = null;
for (int head = 0; head < text.Length; head++) {
int index = head;
TrieNode node = _root;
while (node.TryGetValue(text[index], out node)) {
if (node.m_end) {
if (chars == null) chars = text.ToArray();
for (int i = head; i <= index; i++) {
chars[i] = mask;
}
head = index;
}
if (text.Length == ++index) {
break;
}
}
}
return chars == null ? text : new string(chars);
}
}
}
ToolGood.Words/internal/HashHelpers.cs
-53
View File
@@ -1,53 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace ToolGood.Words
{
/// <summary>
/// 从.net 编译出来的.
/// </summary>
internal static class HashHelpers
{
// Fields
internal static readonly int[] primes = new int[] {
3, 7, 11, 0x11, 0x17, 0x1d, 0x25, 0x2f, 0x3b, 0x47, 0x59, 0x6b, 0x83, 0xa3, 0xc5, 0xef,
0x125, 0x161, 0x1af, 0x209, 0x277, 0x2f9, 0x397, 0x44f, 0x52f, 0x63d, 0x78b, 0x91d, 0xaf1, 0xd2b, 0xfd1, 0x12fd,
0x16cf, 0x1b65, 0x20e3, 0x2777, 0x2f6f, 0x38ff, 0x446f, 0x521f, 0x628d, 0x7655, 0x8e01, 0xaa6b, 0xcc89, 0xf583, 0x126a7, 0x1619b,
0x1a857, 0x1fd3b, 0x26315, 0x2dd67, 0x3701b, 0x42023, 0x4f361, 0x5f0ed, 0x72125, 0x88e31, 0xa443b, 0xc51eb, 0xec8c1, 0x11bdbf, 0x154a3f, 0x198c4f,
0x1ea867, 0x24ca19, 0x2c25c1, 0x34fa1b, 0x3f928f, 0x4c4987, 0x5b8b6f, 0x6dda89
};
internal static int GetPrime(int min)
{
for (int i = 0; i < primes.Length; i++) {
int num2 = primes[i];
if (num2 >= min) {
return num2;
}
}
for (int j = min | 1; j < 0x7fffffff; j += 2) {
if (IsPrime(j)) {
return j;
}
}
return min;
}
internal static bool IsPrime(int candidate)
{
if ((candidate & 1) == 0) {
return (candidate == 2);
}
int num = (int)Math.Sqrt((double)candidate);
for (int i = 3; i <= num; i += 2) {
if ((candidate % i) == 0) {
return false;
}
}
return true;
}
}
}
ToolGood.Words/internal/HashStringSet.cs
-227
View File
@@ -1,227 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
namespace ToolGood.Words
{
/// <summary>
/// 参考了MONO的HashSet,针对字符串进行优化
/// 增加了检查部分字符串的方法,可避免字符分割带来的GC问题
/// Contains(String item, int offset, int len)
/// </summary>
class HashStringSet
{
[StructLayout(LayoutKind.Sequential)]
internal struct Slot
{
internal int hashCode;
internal String value;
internal int next;
}
// Fields
private int[] m_buckets;
private int m_count;
private int m_freeList;
private int m_lastIndex;
private Slot[] m_slots;
public HashStringSet()
{
this.m_lastIndex = 0;
this.m_count = 0;
this.m_freeList = -1;
}
public bool Add(String value)
{
int freeList;
if (this.m_buckets == null) {
this.Initialize(0);
}
int hashCode = this.InternalGetHashCode(value);
int index = hashCode % this.m_buckets.Length;
for (int i = this.m_buckets[hashCode % this.m_buckets.Length] - 1; i >= 0; i = this.m_slots[i].next) {
if ((this.m_slots[i].hashCode == hashCode) && this.m_slots[i].value.Equals(value)) {
return false;
}
}
if (this.m_freeList >= 0) {
freeList = this.m_freeList;
this.m_freeList = this.m_slots[freeList].next;
} else {
if (this.m_lastIndex == this.m_slots.Length) {
this.IncreaseCapacity();
index = hashCode % this.m_buckets.Length;
}
freeList = this.m_lastIndex;
this.m_lastIndex++;
}
this.m_slots[freeList].hashCode = hashCode;
this.m_slots[freeList].value = value;
this.m_slots[freeList].next = this.m_buckets[index] - 1;
this.m_buckets[index] = freeList + 1;
this.m_count++;
return true;
}
private void IncreaseCapacity()
{
int min = this.m_count * 2;
if (min < 0) {
min = this.m_count;
}
int prime = HashHelpers.GetPrime(min);
Slot[] destinationArray = new Slot[prime];
if (this.m_slots != null) {
Array.Copy(this.m_slots, 0, destinationArray, 0, this.m_lastIndex);
}
int[] numArray = new int[prime];
for (int i = 0; i < this.m_lastIndex; i++) {
int index = destinationArray[i].hashCode % prime;
destinationArray[i].next = numArray[index] - 1;
numArray[index] = i + 1;
}
this.m_slots = destinationArray;
this.m_buckets = numArray;
}
public void Clear()
{
if (this.m_lastIndex > 0) {
Array.Clear(this.m_slots, 0, this.m_lastIndex);
Array.Clear(this.m_buckets, 0, this.m_buckets.Length);
this.m_lastIndex = 0;
this.m_count = 0;
this.m_freeList = -1;
}
}
public bool Contains(String item)
{
if (this.m_buckets != null) {
int hashCode = this.InternalGetHashCode(item);
for (int i = this.m_buckets[hashCode % this.m_buckets.Length] - 1; i >= 0; i = this.m_slots[i].next) {
if ((this.m_slots[i].hashCode == hashCode) && this.m_slots[i].value.Equals(item)) {
return true;
}
}
}
return false;
}
private void Initialize(int capacity)
{
int prime = HashHelpers.GetPrime(capacity);
this.m_buckets = new int[prime];
this.m_slots = new Slot[prime];
}
//重写原来的HashCode
private int InternalGetHashCode(String item)
{
return InternalGetHashCode(item, 0, item.Length);
}
public bool Remove(String item)
{
if (this.m_buckets != null) {
int hashCode = this.InternalGetHashCode(item);
int index = hashCode % this.m_buckets.Length;
int num3 = -1;
for (int i = this.m_buckets[index] - 1; i >= 0; i = this.m_slots[i].next) {
if ((this.m_slots[i].hashCode == hashCode) && this.m_slots[i].value.Equals(item)) {
if (num3 < 0) {
this.m_buckets[index] = this.m_slots[i].next + 1;
} else {
this.m_slots[num3].next = this.m_slots[i].next;
}
this.m_slots[i].hashCode = -1;
this.m_slots[i].value = null;
this.m_slots[i].next = this.m_freeList;
this.m_count--;
if (this.m_count == 0) {
this.m_lastIndex = 0;
this.m_freeList = -1;
} else {
this.m_freeList = i;
}
return true;
}
num3 = i;
}
}
return false;
}
#region 新增方法,避免字符分割
public bool Contains(String item, int offset, int len)
{
if (this.m_buckets != null) {
int hashCode = InternalGetHashCode(item, offset, len);
for (int i = this.m_buckets[hashCode % this.m_buckets.Length] - 1; i >= 0; i = this.m_slots[i].next) {
if ((this.m_slots[i].hashCode == hashCode) && StringEquals(this.m_slots[i].value, item, offset, len)) {
return true;
}
}
}
return false;
}
static unsafe int InternalGetHashCode(String item, int offset, int len)
{
int h = 0;
fixed (char* c = item) {
char* cc = c + offset;
char* end = cc + len - 1;
for (; cc < end; cc += 2) {
h = (h << 5) - h + *cc;
h = (h << 5) - h + cc[1];
}
++end;
if (cc < end)
h = (h << 5) - h + *cc;
}
return h & 0x7fffffff;
}
static unsafe bool StringEquals(string a, string b, int offset, int len)
{
if (len != a.Length)
return false;
fixed (char* s1 = a, s2 = b) {
char* s1_ptr = s1;
char* s2_ptr = s2 + offset;
if ((offset & 0x01) == 0) //偏移位置为偶数
{
while (len > 1) {
if (((int*)s1_ptr)[0] != ((int*)s2_ptr)[0])
return false;
s1_ptr += 2;
s2_ptr += 2;
len -= 2;
}
} else //偏移位置为奇数
{
while (len > 1) {
if ((s1_ptr)[0] != (s2_ptr)[0] ||
(s1_ptr)[1] != (s2_ptr)[1])
return false;
s1_ptr += 2;
s2_ptr += 2;
len -= 2;
}
}
return len == 0 || *s1_ptr == *s2_ptr;
}
}
#endregion
}
}