mirror of
https://github.com/toolgood/ToolGood.Words.git
synced 2026-10-21 23:55:46 +08:00
toolgood
1210 lines
55 KiB
C#
1210 lines
55 KiB
C#
using Microsoft.International.Converters.PinYinConverter;
|
||
using Studyzy.IMEWLConverter.IME;
|
||
using System;
|
||
using System.Collections.Generic;
|
||
using System.IO;
|
||
using System.Linq;
|
||
using System.Net;
|
||
using System.Text;
|
||
using System.Text.RegularExpressions;
|
||
using System.Web;
|
||
using ToolGood.PinYin.Pretreatment;
|
||
using ToolGood.Words;
|
||
using ToolGood.Words.internals;
|
||
|
||
namespace ToolGood.Pinyin.Pretreatment
|
||
{
|
||
partial class Program
|
||
{
|
||
static void Main(string[] args)
|
||
{
|
||
// 感谢 hotoo 大佬 整理的拼音 拼音范围 0x4e00-0x9fa5
|
||
// https://github.com/hotoo/pinyin
|
||
// 感谢 mozillazg 大佬 整理的拼音 并指出 拼音来源
|
||
// 拼音来源
|
||
// https://github.com/mozillazg/pinyin-data
|
||
// https://github.com/mozillazg/phrase-pinyin-data
|
||
|
||
MozillazgPinyinHelper.ToStandardization();
|
||
|
||
var start = "𠀀";// '\ud840' '\udc00' - '\udfff'
|
||
var end = "𫠝";// '\ud86e' '\udc1d'
|
||
var mid = "𬺓";
|
||
mid = "𬀩";
|
||
StringBuilder stringBuilder = new StringBuilder(start);
|
||
stringBuilder[0] = (char)0xd840;
|
||
stringBuilder[1] = (char)0xdfff;
|
||
|
||
// 预处理
|
||
// 第一步 处理搜狗词库
|
||
Console.WriteLine("第一步 处理搜狗词库");
|
||
if (File.Exists("scel_1.txt") == false) {
|
||
var scel_1 = GetWords();
|
||
File.WriteAllText("scel_1.txt", string.Join("\n", scel_1));
|
||
scel_1.Clear();
|
||
}
|
||
|
||
// 第二步 精简词库
|
||
Console.WriteLine("第二步 精简词库");
|
||
if (File.Exists("scel_2.txt") == false) {
|
||
var txt = File.ReadAllText("scel_1.txt");
|
||
var lines = txt.Split('\n');
|
||
Dictionary<string, string> dict = new Dictionary<string, string>();
|
||
foreach (var item in lines) {
|
||
var sp = item.Split(' ');
|
||
dict[sp[0]] = sp[1];
|
||
}
|
||
lines = null;
|
||
txt = null;
|
||
List<string> keys = dict.Select(q => q.Key).OrderBy(q => q.Length).ToList();
|
||
|
||
|
||
#region 删除通用的
|
||
keys.RemoveAll(q => q.Contains("大厦") && q.Length > 7);
|
||
keys.RemoveAll(q => q.Contains("公司") && q.Length > 5);
|
||
keys.RemoveAll(q => q.Contains("大楼") && q.Length > 5);
|
||
|
||
keys.RemoveAll(q => q.Contains("而") && q.Length > 5);
|
||
keys.RemoveAll(q => q.Contains("的") && q.Length > 5);
|
||
keys.RemoveAll(q => q.Contains("和") && q.Length > 7);
|
||
keys.RemoveAll(q => q.Contains("或") && q.Length > 5);
|
||
keys.RemoveAll(q => q.Contains("与") && q.Length > 5);
|
||
keys.RemoveAll(q => q.Contains("用") && q.Length > 5);
|
||
keys.RemoveAll(q => q.Contains("地") && q.Length > 5);
|
||
#endregion
|
||
|
||
#region 删除结尾
|
||
|
||
keys.RemoveAll(q => q.EndsWith("厂") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("街道") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("办") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("乡") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("会社") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("大楼") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("症") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("小区") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("社区") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("胡同") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("管委会") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("村") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("液") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("国") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("场") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("室") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("征") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("胶囊") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("处") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("酶") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("片") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("器") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("病") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("证") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("剂") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("钉") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("素") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("术") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("因子") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("细胞") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("系统") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("炎") && q.Length > 5);
|
||
keys.RemoveAll(q => q.EndsWith("区") && q.Length > 7);
|
||
keys.RemoveAll(q => q.EndsWith("寓") && q.Length > 7);
|
||
keys.RemoveAll(q => q.EndsWith("装置") && q.Length > 7);
|
||
#endregion
|
||
|
||
|
||
keys.RemoveAll(q => q.StartsWith("复方") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("一次性") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("因为") && q.Length > 5);
|
||
|
||
keys.RemoveAll(q => q.StartsWith("中国") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("日本") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("德国") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("法国") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("英国") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("美国") && q.Length > 5);
|
||
|
||
#region 去省市
|
||
keys.RemoveAll(q => q.StartsWith("蒙古") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("北京") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("天津") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("河北") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("山西") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("内蒙古") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("辽宁") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("吉林") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("黑龙江") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("上海") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("江苏") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("浙江") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("安徽") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("福建") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("江西") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("山东") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("河南") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("湖北") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("湖南") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("广东") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("广西") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("海南") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("四川") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("贵州") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("云南") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("重庆") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("西藏") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("陕西") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("甘肃") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("青海") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("宁夏") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("新疆") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("香港") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("澳门") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("台湾") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("石家庄") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("太原") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("呼和浩特") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("沈阳") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("长春") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("哈尔滨") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("南京") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("杭州") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("合肥") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("福州") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("南昌") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("济南") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("郑州") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("武汉") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("长沙") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("广州") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("南宁") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("海口") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("成都") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("贵阳") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("昆明") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("拉萨") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("西安") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("兰州") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("西宁") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("银川") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("乌鲁木齐") && q.Length > 5);
|
||
keys.RemoveAll(q => q.StartsWith("台北") && q.Length > 5);
|
||
#endregion
|
||
|
||
|
||
for (int i = 3; i < 6; i++) {
|
||
var keywords = keys.Where(q => q.Length <= i).ToList();
|
||
WordsSearch wordsSearch = new WordsSearch();
|
||
wordsSearch.SetKeywords(keywords);
|
||
|
||
for (int j = keys.Count - 1; j >= 0; j--) {
|
||
var key = keys[j];
|
||
if (key.Length <= i) { break; }
|
||
|
||
var all = wordsSearch.FindAll(key);
|
||
if (all.Count > 1) {
|
||
// Console.WriteLine(key);
|
||
//进行拼音测试，相同则删除
|
||
var py = ReplaceForPinyin(key, all, dict);
|
||
if (py != null) {
|
||
py = py.ToLower();
|
||
if (dict[key].ToLower().Trim() == py) {
|
||
keys.RemoveAt(j);
|
||
}
|
||
}
|
||
|
||
}
|
||
}
|
||
wordsSearch = null;
|
||
GC.Collect();
|
||
}
|
||
|
||
for (int i = keys.Count - 1; i >= 0; i--) {
|
||
var item = keys[i];
|
||
if (item.Length < 5) break;
|
||
var pyf = WordsHelper.GetPinyinFast(item).ToLower();
|
||
|
||
if (dict[item].ToLower().Replace(",", "") == pyf) {
|
||
keys.RemoveAt(i);
|
||
}
|
||
}
|
||
|
||
var fs = File.Open("scel_2.txt", FileMode.OpenOrCreate);
|
||
StreamWriter sw = new StreamWriter(fs);
|
||
foreach (var item in keys) {
|
||
sw.WriteLine($"{item} {dict[item]}");
|
||
}
|
||
sw.Close();
|
||
fs.Close();
|
||
}
|
||
|
||
// 第三步 获取词的所有拼音
|
||
Console.WriteLine("第三步 获取词的所有拼音");
|
||
if (File.Exists("scel_3.txt") == false) {
|
||
Dictionary<char, HashSet<string>> dict = new Dictionary<char, HashSet<string>>();
|
||
var txt = File.ReadAllText("scel_2.txt");
|
||
var lines = txt.Split('\n');
|
||
foreach (var item in lines) {
|
||
if (string.IsNullOrWhiteSpace(item)) { continue; }
|
||
var sp = item.Trim().Split(' ');
|
||
var pys = sp[1].Split(',');
|
||
for (int i = 0; i < sp[0].Length; i++) {
|
||
var c = sp[0][i];
|
||
HashSet<string> list;
|
||
if (dict.TryGetValue(c, out list) == false) {
|
||
list = new HashSet<string>();
|
||
dict[c] = list;
|
||
}
|
||
list.Add(pys[i]);
|
||
}
|
||
}
|
||
|
||
var fs = File.Open("scel_3.txt", FileMode.OpenOrCreate);
|
||
StreamWriter sw = new StreamWriter(fs);
|
||
foreach (var item in dict) {
|
||
sw.WriteLine($"{item.Key} {string.Join(",", item.Value)}");
|
||
}
|
||
sw.Close();
|
||
fs.Close();
|
||
}
|
||
|
||
// 第四步 获取网上的拼音
|
||
Console.WriteLine("第四步 获取网上的拼音");
|
||
if (File.Exists("pinyin_1.txt") == false) {
|
||
var pinyin_1 = GetPinyin();
|
||
File.WriteAllText("pinyin_1.txt", string.Join("\n", pinyin_1));
|
||
pinyin_1.Clear();
|
||
}
|
||
|
||
// 第五步 分离 单字拼音 和 词组拼音
|
||
Console.WriteLine("第五步 分离 单字拼音 和 词组拼音");
|
||
if (File.Exists("pinyin_2_one.txt") == false) {
|
||
var txt = File.ReadAllText("pinyin_1.txt");
|
||
var lines = txt.Split('\n');
|
||
List<string> ones = new List<string>();
|
||
List<string> mores = new List<string>();
|
||
foreach (var line in lines) {
|
||
var sp = line.Split(',');
|
||
if (GetLength(sp[0]) == 1) {
|
||
ones.Add(line);
|
||
} else {
|
||
mores.Add(line);
|
||
}
|
||
}
|
||
ones = ones.OrderBy(q => q).ToList();
|
||
File.WriteAllText("pinyin_2_one.txt", string.Join("\n", ones));
|
||
File.WriteAllText("pinyin_2_more.txt", string.Join("\n", mores));
|
||
ones.Clear();
|
||
mores.Clear();
|
||
}
|
||
|
||
// 第六步 简单 合并 单字拼音， 防止常用拼音被覆盖
|
||
Console.WriteLine("第六步 简单 合并 单字拼音， 防止常用拼音被覆盖");
|
||
if (File.Exists("pinyin_3_one.txt") == false) {
|
||
var txt = File.ReadAllText("pinyin_2_one.txt");
|
||
var lines = txt.Split('\n').ToList();
|
||
lines = lines.OrderBy(q => q).ToList();
|
||
for (int i = lines.Count - 1; i >= 1; i--) {
|
||
if (lines[i].StartsWith(lines[i - 1])) {
|
||
lines.RemoveAt(i);
|
||
}
|
||
}
|
||
File.WriteAllText("pinyin_3_one.txt", string.Join("\n", lines));
|
||
}
|
||
|
||
// 第七步 检查 拼音数 与 词组长度不一样的
|
||
Console.WriteLine("第七步 检查 拼音数 与 词组长度不一样的");
|
||
if (File.Exists("pinyin_4_ok.txt") == false) {
|
||
var txt = File.ReadAllText("pinyin_2_more.txt");
|
||
var lines = txt.Split('\n');
|
||
List<string> oks = new List<string>();
|
||
List<string> errors = new List<string>();
|
||
foreach (var line in lines) {
|
||
var sp = line.Split(',');
|
||
if (GetLength(sp[0]) == sp.Length - 1) {
|
||
oks.Add(line);
|
||
} else {
|
||
errors.Add(line);
|
||
}
|
||
}
|
||
File.WriteAllText("pinyin_4_ok.txt", string.Join("\n", oks));
|
||
File.WriteAllText("pinyin_4_error.txt", string.Join("\n", errors));
|
||
}
|
||
|
||
// 第八步 单字拼音
|
||
Console.WriteLine("第八步 检查 单字拼音 是否出错");
|
||
if (File.Exists("pinyin_5_one.txt") == false) {
|
||
var txt = File.ReadAllText("dict/dict-pinyin.txt");
|
||
var lines = txt.Split('\n').ToList();
|
||
Dictionary<char, List<string>> pysDict = new Dictionary<char, List<string>>();
|
||
|
||
for (int i = lines.Count - 1; i >= 0; i--) {
|
||
var line = lines[i];
|
||
line = Regex.Replace(line, "//.*[\r\n]?", "");
|
||
if (string.IsNullOrWhiteSpace(line)) { continue; }
|
||
var sps = line.Split(" \"': ,\r[]?".ToArray(), StringSplitOptions.RemoveEmptyEntries).ToList();
|
||
var key = sps[0][0];
|
||
sps.RemoveAt(0);
|
||
pysDict[key] = sps;
|
||
}
|
||
|
||
txt = File.ReadAllText("pinyin_3_one.txt");
|
||
lines = txt.Split('\n').ToList();
|
||
for (int i = lines.Count - 1; i >= 0; i--) {
|
||
var line = lines[i];
|
||
line = Regex.Replace(line, "//.*[\r\n]?", "");
|
||
if (string.IsNullOrWhiteSpace(line)) { continue; }
|
||
var sps = line.Split(" \"': ,\r[]?".ToArray(), StringSplitOptions.RemoveEmptyEntries).ToList();
|
||
var key = sps[0][0];
|
||
if (pysDict.ContainsKey(key) == false) {
|
||
sps.RemoveAt(0);
|
||
pysDict[key] = sps;
|
||
}
|
||
}
|
||
Dictionary<char, List<string>> nopysDict = new Dictionary<char, List<string>>();
|
||
|
||
for (int i = 0x3400; i <= 0x9FFF; i++) {
|
||
var c = (char)i;
|
||
if (pysDict.ContainsKey(c) == false) {
|
||
var pys = WordsHelper.GetAllPinyin(c);
|
||
pysDict[c] = pys;
|
||
nopysDict[c] = pys;
|
||
}
|
||
}
|
||
|
||
List<string> ls = new List<string>();
|
||
foreach (var item in pysDict) {
|
||
ls.Add($"{item.Key} {string.Join(",", item.Value)}");
|
||
}
|
||
|
||
List<string> ls2 = new List<string>();
|
||
foreach (var item in nopysDict) {
|
||
ls2.Add($"{item.Key} {string.Join(",", item.Value)}");
|
||
}
|
||
|
||
ls = ls.OrderBy(q => q).ToList();
|
||
for (int i = ls.Count - 1; i >= 0; i--) {
|
||
var l = ls[i];
|
||
if (l[0] > 0x9FFF) {
|
||
ls.RemoveAt(i);
|
||
}
|
||
}
|
||
|
||
|
||
var str = string.Join("\n", ls);
|
||
File.WriteAllText("pinyin_5_one.txt", string.Join("\n", ls));
|
||
File.WriteAllText("pinyin_5_oneNoPy.txt", string.Join("\n", ls2));
|
||
}
|
||
Console.WriteLine("第八步补 获取拼音集 ");
|
||
if (File.Exists("pinyin_5_1_one.txt") == false) {
|
||
var pyText = File.ReadAllText("pinyin_5_one.txt");
|
||
var pyLines = pyText.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
|
||
Dictionary<string, List<string>> dict = new Dictionary<string, List<string>>();
|
||
|
||
foreach (var line in pyLines) {
|
||
var sp = line.Split("\t,:| '\"=>[]， 123456789?".ToArray(), StringSplitOptions.RemoveEmptyEntries);
|
||
List<string> pys = new List<string>();
|
||
|
||
for (int i = 1; i < sp.Length; i++) {
|
||
pys.Add(sp[i]);
|
||
}
|
||
dict[sp[0]] = pys;
|
||
}
|
||
for (int i = 0x3400; i <= 0x9fff; i++) {
|
||
var c = ((char)i);
|
||
var cStr = c.ToString();
|
||
if (cStr == "色") {
|
||
|
||
}
|
||
if (ChineseChar.IsValidChar(c)) {
|
||
List<string> pys2 = new List<string>();
|
||
ChineseChar chineseChar = new ChineseChar(c);
|
||
foreach (string value in chineseChar.Pinyins) {
|
||
if (string.IsNullOrEmpty(value)) { continue; }
|
||
if (value.EndsWith("5")==false && value.EndsWith("0")==false) {
|
||
pys2.Add(AddTone(value));
|
||
} else {
|
||
|
||
}
|
||
}
|
||
List<string> pys;
|
||
if (dict.TryGetValue(cStr, out pys) == false) {
|
||
pys = new List<string>();
|
||
}
|
||
pys2.RemoveAll(q => pys.Contains(q));
|
||
if (pys2.Count > 0) {
|
||
pys.AddRange(pys2);
|
||
dict[cStr] = pys;
|
||
}
|
||
}
|
||
}
|
||
List<string> ls2 = new List<string>();
|
||
foreach (var item in dict) {
|
||
ls2.Add($"{item.Key} {string.Join(",", item.Value)}");
|
||
}
|
||
File.WriteAllText("pinyin_5_1_one.txt", string.Join("\n", ls2));
|
||
|
||
Dictionary<string, List<string>> pysDict = new Dictionary<string, List<string>>();
|
||
var txt = File.ReadAllText("pinyin_3_one.txt");
|
||
var lines = txt.Split('\n').ToList();
|
||
for (int i = lines.Count - 1; i >= 0; i--) {
|
||
var line = lines[i];
|
||
line = Regex.Replace(line, "//.*[\r\n]?", "");
|
||
if (string.IsNullOrWhiteSpace(line)) { continue; }
|
||
var sps = line.Split("\t,:| '\"=>[]， 123456789?".ToArray(), StringSplitOptions.RemoveEmptyEntries).ToList();
|
||
var key = sps[0];
|
||
sps.RemoveAt(0);
|
||
pysDict[key] = sps;
|
||
}
|
||
txt = File.ReadAllText("dict/dict-pinyin.txt");
|
||
lines = txt.Split('\n').ToList();
|
||
for (int i = lines.Count - 1; i >= 0; i--) {
|
||
var line = lines[i];
|
||
line = Regex.Replace(line, "//.*[\r\n]?", "");
|
||
if (string.IsNullOrWhiteSpace(line)) { continue; }
|
||
var sps = line.Split("\t,:| '\"=>[]， 123456789?".ToArray(), StringSplitOptions.RemoveEmptyEntries).ToList();
|
||
var key = sps[0];
|
||
sps.RemoveAt(0);
|
||
pysDict[key] = sps;
|
||
}
|
||
txt = File.ReadAllText("dict/mozillazg_pinyin.txt");
|
||
lines = txt.Split('\n').ToList();
|
||
for (int i = lines.Count - 1; i >= 0; i--) {
|
||
var line = lines[i];
|
||
line = Regex.Replace(line, "//.*[\r\n]?", "");
|
||
if (string.IsNullOrWhiteSpace(line)) { continue; }
|
||
var sps = line.Split("\t,:| '\"=>[]， 123456789?".ToArray(), StringSplitOptions.RemoveEmptyEntries).ToList();
|
||
var key = sps[0];
|
||
sps.RemoveAt(0);
|
||
pysDict[key] = sps;
|
||
}
|
||
|
||
List<string> ls3 = new List<string>();
|
||
foreach (var item in pysDict) {
|
||
if (item.Key.Length == 1) {
|
||
continue;
|
||
}
|
||
if (item.Key[0] >= 0xd840 && item.Key[0] <= 0xd86e) {
|
||
if (item.Key[1] >= 0xdc00 && item.Key[1] <= 0xdfff) {
|
||
ls3.Add($"{item.Key} {string.Join(",", item.Value)}");
|
||
}
|
||
}
|
||
}
|
||
|
||
File.WriteAllText("pinyin_5_2_one.txt", string.Join("\n", ls3));
|
||
}
|
||
|
||
|
||
|
||
// 第九步 获取拼音集
|
||
Console.WriteLine("第九步 获取拼音集 ");
|
||
if (File.Exists("pinyin_6_pys.txt") == false) {
|
||
var txt = File.ReadAllText("pinyin_5_one.txt");
|
||
var lines = txt.Split('\n').ToList();
|
||
|
||
Dictionary<string, string> dict = new Dictionary<string, string>();
|
||
|
||
for (int i = lines.Count - 1; i >= 0; i--) {
|
||
var line = lines[i];
|
||
if (string.IsNullOrWhiteSpace(line)) { continue; }
|
||
var sps = line.Split(" \"': ,\r".ToArray(), StringSplitOptions.RemoveEmptyEntries).ToList();
|
||
for (int j = 1; j < sps.Count; j++) {
|
||
dict[sps[j]] = RemoveTone(sps[j]);
|
||
}
|
||
}
|
||
List<string> ls = new List<string>();
|
||
List<string> ls2 = new List<string>();
|
||
List<string> ls3 = new List<string>();
|
||
List<string> ls4 = new List<string>();
|
||
List<string> ls5 = new List<string>();
|
||
foreach (var item in dict) {
|
||
ls.Add($"{item.Value} { item.Key}");
|
||
ls2.Add("\"" + item.Key.ToUpper()[0] + item.Key.ToLower().Substring(1) + "\"");
|
||
ls3.Add("\"" + item.Value.ToUpper()[0] + item.Value.ToLower().Substring(1) + "\"");
|
||
ls4.Add("\"" + item.Value.ToUpper()[0] + item.Value.ToLower().Substring(1) + "\",\"" + item.Key.ToUpper()[0] + item.Key.ToLower().Substring(1) + "\"");
|
||
ls5.Add(item.Value);
|
||
ls5.Add(item.Key);
|
||
}
|
||
|
||
ls = ls.OrderBy(q => q).ToList();
|
||
ls2 = ls2.OrderBy(q => q).ToList();
|
||
ls3 = ls3.Distinct().OrderBy(q => q).ToList();
|
||
ls4 = ls4.Distinct().OrderBy(q => q).ToList();
|
||
ls5 = ls5.Distinct().OrderBy(q => q).ToList();
|
||
File.WriteAllText("pinyin_6_pys.txt", string.Join("\n", ls));
|
||
File.WriteAllText("pinyin_6_pys_one.txt", string.Join(",", ls2));
|
||
File.WriteAllText("pinyin_6_pys_one2.txt", string.Join(",", ls3));
|
||
File.WriteAllText("pinyin_6_pys_all.txt", string.Join(",", ls4));
|
||
File.WriteAllText("pinyin_6_pys_all2.txt", string.Join(",", ls5));
|
||
}
|
||
|
||
Console.WriteLine("第十步 检查拼音集的拼音是否错误 ");
|
||
//if (File.Exists("pinyin_7_pys.txt") == false) {
|
||
// var txt = File.ReadAllText("pinyin_4_ok.txt");
|
||
// var lines = txt.Split('\n').ToList();
|
||
|
||
// List<string> ls = new List<string>();
|
||
// foreach (var line in lines) {
|
||
// GetBuildPinyin(line, ls);
|
||
// }
|
||
|
||
// File.WriteAllText("pinyin_7_pys.txt", string.Join("\n", ls));
|
||
//}
|
||
|
||
Console.WriteLine("第十一步 检查拼音集的拼音是否错误 ");
|
||
if (File.Exists("pinyin_8_mores_error.txt") == false) {
|
||
var txt = File.ReadAllText("pinyin_4_ok.txt");
|
||
var lines = txt.Split('\n').ToList();
|
||
|
||
List<string> errors = new List<string>();
|
||
for (int i = 0; i < lines.Count - 1; i++) {
|
||
var l1 = lines[i].Split(',');
|
||
var l2 = lines[i + 1].Split(',');
|
||
if (l1[0] == l2[0] && lines[i] != lines[i + 1]) {
|
||
errors.Add(lines[i]);
|
||
errors.Add(lines[i + 1]);
|
||
}
|
||
}
|
||
lines.RemoveAll(q => errors.Contains(q));
|
||
|
||
File.WriteAllText("pinyin_8_mores_error.txt", string.Join("\n", errors));
|
||
File.WriteAllText("pinyin_8_mores_ok.txt", string.Join("\n", lines));
|
||
}
|
||
|
||
|
||
Console.WriteLine("第十二步 使用搜狗拼音 ");
|
||
if (File.Exists("pinyin_9_mores_error.txt") == false) {
|
||
var dictTxt = File.ReadAllText("scel_1.txt");
|
||
Dictionary<string, string> dict = new Dictionary<string, string>();
|
||
var dictLines = dictTxt.Split('\n').ToList();
|
||
foreach (var line in dictLines) {
|
||
if (string.IsNullOrEmpty(line)) {
|
||
continue;
|
||
}
|
||
var sp = line.Split(' ');
|
||
dict[sp[0]] = sp[1];
|
||
}
|
||
|
||
var txt = File.ReadAllText("pinyin_8_mores_ok.txt");
|
||
var lines = txt.Split('\n').ToList();
|
||
List<string> errors = new List<string>();
|
||
List<string> oks = new List<string>();
|
||
List<string> unfind = new List<string>();
|
||
foreach (var line in lines) {
|
||
var sp = line.Split(',').ToList();
|
||
if (dict.TryGetValue(sp[0], out string py)) {
|
||
sp.RemoveAt(0);
|
||
var py2 = RemoveTone(string.Join(",", sp));
|
||
if (py.ToLower() == py2.ToLower() || py.ToLower() == py2.ToLower().Replace("v", "u")) {
|
||
oks.Add(line);
|
||
} else {
|
||
|
||
//var py3 = WordsHelper.GetAllPinyin(sp[0], true);
|
||
errors.Add(line + " | " + py);
|
||
}
|
||
} else {
|
||
unfind.Add(line);
|
||
}
|
||
}
|
||
File.WriteAllText("pinyin_9_mores_error.txt", string.Join("\n", errors));
|
||
File.WriteAllText("pinyin_9_mores_ok.txt", string.Join("\n", oks));
|
||
File.WriteAllText("pinyin_9_mores_unfind.txt", string.Join("\n", unfind));
|
||
}
|
||
|
||
Console.WriteLine("第十三步 修复错误拼音 ");
|
||
if (File.Exists("pinyin_10_mores_ok.txt") == false) {
|
||
var txt = File.ReadAllText("pinyin_9_mores_error.txt");
|
||
var lines = txt.Split('\n').ToList();
|
||
|
||
List<string> ls = new List<string>();
|
||
List<string> errors = new List<string>();
|
||
foreach (var line in lines) {
|
||
var lsp = line.Split(" |".ToArray(), StringSplitOptions.RemoveEmptyEntries);
|
||
var tarPy = lsp[1].Split(",");
|
||
var sp = lsp[0].Split(',');
|
||
|
||
for (int i = 1; i < sp.Length; i++) {
|
||
var py = RemoveTone(sp[i]).ToLower();//.Replace("v", "u");
|
||
var py2 = RemoveTone(sp[i]).ToLower().Replace("v", "u");
|
||
var tpy2 = tarPy[i - 1];
|
||
if (py != tpy2 && py2 != tpy2) {
|
||
var c = sp[0][i - 1];
|
||
var pys = WordsHelper.GetAllPinyin(c, true);
|
||
var count = 0; //读音为1种，则成功
|
||
var trypy = "";// 读音
|
||
foreach (var item in pys) {
|
||
var itemp = RemoveTone(item.ToLower());
|
||
if (itemp == tarPy[i - 1] || itemp.Replace("v", "u") == tarPy[i - 1]) {
|
||
//sp[i] = item.ToLower();
|
||
trypy = item.ToLower();
|
||
count++;
|
||
}
|
||
}
|
||
if (count == 1) {
|
||
sp[i] = trypy;
|
||
}
|
||
}
|
||
}
|
||
var isok = true;
|
||
for (int i = 1; i < sp.Length; i++) {
|
||
var py = RemoveTone(sp[i]).ToLower();//.Replace("v", "u");
|
||
if (py != tarPy[i - 1] && py.Replace("v", "u") != tarPy[i - 1]) {
|
||
isok = false;
|
||
break;
|
||
}
|
||
}
|
||
if (isok) {
|
||
ls.Add(string.Join(",", sp));
|
||
} else {
|
||
errors.Add(line);
|
||
}
|
||
}
|
||
File.WriteAllText("pinyin_10_mores_ok.txt", string.Join("\n", ls));
|
||
File.WriteAllText("pinyin_10_mores_error.txt", string.Join("\n", errors));
|
||
|
||
}
|
||
|
||
Console.WriteLine("第十四步 添加搜狗拼音词组 ");
|
||
if (File.Exists("pinyin_11_mores_ok.txt") == false) {
|
||
var txt = File.ReadAllText("pinyin_9_mores_ok.txt");
|
||
var lines = txt.Split('\n').ToList();
|
||
txt = File.ReadAllText("pinyin_10_mores_ok.txt");
|
||
var lines2 = txt.Split('\n').ToList();
|
||
lines.AddRange(lines2);
|
||
List<string> oldKeys = new List<string>();
|
||
foreach (var item in lines) {
|
||
var sp = item.Split(',');
|
||
oldKeys.Add(sp[0]);
|
||
}
|
||
|
||
|
||
Dictionary<string, int> remove = new Dictionary<string, int>();
|
||
|
||
txt = File.ReadAllText("scel_2.txt");
|
||
lines = txt.Split('\n').ToList();
|
||
Dictionary<string, List<string>> dict = new Dictionary<string, List<string>>();
|
||
foreach (var item in lines) {
|
||
if (string.IsNullOrWhiteSpace(item)) { continue; }
|
||
var sp = item.Split(" ,|".ToArray(), StringSplitOptions.RemoveEmptyEntries).ToList();
|
||
var key = sp[0];
|
||
if (oldKeys.Contains(key)) { continue; }
|
||
sp.RemoveAt(0);
|
||
|
||
var isok = true;
|
||
for (int i = 0; i < key.Length; i++) {
|
||
int count;
|
||
if (remove.TryGetValue(key[i] + sp[i], out count)) {
|
||
if (i == 0) {
|
||
if (count > 2) {
|
||
isok = false;
|
||
break;
|
||
}
|
||
} else {
|
||
isok = false;
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
if (isok == false) { continue; }
|
||
|
||
|
||
List<string> pys = new List<string>();
|
||
for (int i = 0; i < key.Length; i++) {
|
||
var ps = WordsHelper.GetAllPinyin(key[i], true);
|
||
var count = 0; //读音为1种，则成功
|
||
var count2 = 0; //读音为1种，则成功
|
||
var trypy = "";// 读音
|
||
foreach (var p in ps) {
|
||
var itemp = RemoveTone(p.ToLower());
|
||
if (itemp == sp[i] || itemp.Replace("v", "u") == sp[i]) {
|
||
if (count == 0) { trypy = p.ToLower(); }
|
||
count++;
|
||
count2++;
|
||
if (count > 1 && i == 0 && p.ToLower() == itemp) { count--; }
|
||
if (count2 > 1) {
|
||
remove[key[i] + sp[i]] = count2;
|
||
}
|
||
}
|
||
}
|
||
if (count == 1) {
|
||
pys.Add(trypy);
|
||
}
|
||
}
|
||
if (sp.Count == pys.Count) {
|
||
dict[key] = pys;
|
||
}
|
||
}
|
||
|
||
var ls = new List<string>();
|
||
foreach (var key in dict) {
|
||
ls.Add($"{key.Key},{string.Join(",", key.Value)}");
|
||
}
|
||
File.WriteAllText("pinyin_11_mores_ok.txt", string.Join("\n", ls));
|
||
}
|
||
|
||
|
||
|
||
Console.WriteLine("第N步 合并拼音组 ");
|
||
if (File.Exists("pinyin_n_mores_ok.txt") == false) {
|
||
var txt = File.ReadAllText("pinyin_9_mores_ok.txt");
|
||
var lines = txt.Split('\n').ToList();
|
||
txt = File.ReadAllText("pinyin_10_mores_ok.txt");
|
||
var lines2 = txt.Split('\n').ToList();
|
||
lines.AddRange(lines2);
|
||
txt = File.ReadAllText("pinyin_11_mores_ok.txt");
|
||
lines2 = txt.Split('\n').ToList();
|
||
lines.AddRange(lines2);
|
||
|
||
lines = lines.OrderBy(q => q).ToList();
|
||
File.WriteAllText("pinyin_n_mores_ok.txt", string.Join("\n", lines));
|
||
}
|
||
|
||
Console.WriteLine("第N+1步 缩小拼音组 ");
|
||
if (File.Exists("pinyin_n1_mores_ok.txt") == false) {
|
||
var txt = File.ReadAllText("pinyin_n_mores_ok.txt");
|
||
var lines = txt.Split('\n').ToList();
|
||
|
||
Dictionary<string, List<string>> dict = new Dictionary<string, List<string>>();
|
||
foreach (var line in lines) {
|
||
var sp = line.Split(',').ToList();
|
||
var key = sp[0];
|
||
sp.RemoveAt(0);
|
||
dict[key] = sp;
|
||
}
|
||
|
||
var keys = dict.Select(q => q.Key).OrderBy(q => q.Length).ToList();
|
||
for (int i = 3; i < 5; i++) {
|
||
var keywords = keys.Where(q => q.Length <= i).ToList();
|
||
WordsSearch wordsSearch = new WordsSearch();
|
||
wordsSearch.SetKeywords(keywords);
|
||
for (int j = keys.Count - 1; j >= 0; j--) {
|
||
var k = keys[j];
|
||
if (k.Length <= i) { break; }
|
||
var all = wordsSearch.FindAll(k);
|
||
if (all.Count > 1 && all.Any(q => q.Start == 0) && all.Sum(q => q.Keyword.Length) >= k.Length) {
|
||
var py = string.Join(",", dict[k]);
|
||
var rootNode = GetTextNode(k, all);
|
||
var keyspss = rootNode.GetFullTextLine();
|
||
foreach (var item in keyspss) {
|
||
List<string> py2 = new List<string>();
|
||
var sp = item.Split(",".ToCharArray(), StringSplitOptions.RemoveEmptyEntries);
|
||
foreach (var s in sp) {
|
||
py2.AddRange(dict[s]);
|
||
}
|
||
var py3 = string.Join(",", py2);
|
||
if (py3 == py) {
|
||
keys.RemoveAt(j);
|
||
break;
|
||
}
|
||
}
|
||
|
||
}
|
||
}
|
||
}
|
||
|
||
var ls = new List<string>();
|
||
foreach (var key in keys) {
|
||
ls.Add($"{key},{string.Join(",", dict[key])}");
|
||
}
|
||
File.WriteAllText("pinyin_n1_mores_ok.txt", string.Join("\n", ls));
|
||
}
|
||
|
||
Console.WriteLine("第N+2步 修复拼音组 ");
|
||
if (File.Exists("pinyin_n2_mores_ok.txt") == false) {
|
||
var txt = File.ReadAllText("pinyin_n1_mores_ok.txt");
|
||
var lines = txt.Split('\n').ToList();
|
||
|
||
Dictionary<string, List<string>> dict = new Dictionary<string, List<string>>();
|
||
foreach (var line in lines) {
|
||
var sp = line.Split(',').ToList();
|
||
var key = sp[0];
|
||
if (string.IsNullOrEmpty(key)) { continue; }
|
||
sp.RemoveAt(0);
|
||
|
||
for (int i = 0; i < sp.Count; i++) {
|
||
var py = sp[i];
|
||
var pyIndex = GetToneIndex(py);
|
||
py = RemoveTone(py);
|
||
py = AddTone(py + pyIndex.ToString());
|
||
sp[i] = py;
|
||
}
|
||
|
||
dict[key] = sp;
|
||
}
|
||
var ls = new List<string>();
|
||
foreach (var key in dict) {
|
||
ls.Add($"{key.Key},{string.Join(",", key.Value)}");
|
||
}
|
||
File.WriteAllText("pinyin_n2_mores_ok.txt", string.Join("\n", ls));
|
||
}
|
||
|
||
Console.WriteLine("第N+3步 修复拼音组 ");
|
||
if (File.Exists("pinyin_n3_mores_ok.txt") == false) {
|
||
var txt = File.ReadAllText("pinyin_n2_mores_ok.txt");
|
||
var lines = txt.Split('\n').ToList();
|
||
|
||
Dictionary<string, List<string>> dict = new Dictionary<string, List<string>>();
|
||
var pdict = PinyinDict.PyShow;
|
||
for (int j = 0; j < pdict.Length; j++) {
|
||
pdict[j] = pdict[j].ToLower();
|
||
}
|
||
|
||
foreach (var line in lines) {
|
||
var sp = line.Split(',').ToList();
|
||
var key = sp[0];
|
||
sp.RemoveAt(0);
|
||
|
||
for (int i = 0; i < sp.Count; i++) {
|
||
var py = sp[i];
|
||
if (pdict.Contains(py) == false) {
|
||
var pys = WordsHelper.GetAllPinyin(key[i], true);
|
||
for (int j = 0; j < pys.Count; j++) {
|
||
pys[j] = pys[j].ToLower();
|
||
}
|
||
var count = 0; //读音为1种，则成功
|
||
var trypy = "";// 读音
|
||
var tarPy = RemoveTone(py);
|
||
foreach (var item in pys) {
|
||
var itemp = RemoveTone(item.ToLower());
|
||
if (itemp == tarPy || itemp.Replace("v", "u") == tarPy) {
|
||
trypy = item.ToLower();
|
||
count++;
|
||
if (item.ToLower() == itemp) {
|
||
if (count > 1) {
|
||
count--;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if (count == 1) {
|
||
py = trypy;
|
||
}
|
||
}
|
||
sp[i] = py;
|
||
}
|
||
dict[key] = sp;
|
||
}
|
||
|
||
|
||
var ls = new List<string>();
|
||
foreach (var key in dict) {
|
||
ls.Add($"{key.Key},{string.Join(",", key.Value)}");
|
||
}
|
||
File.WriteAllText("pinyin_n3_mores_ok.txt", string.Join("\n", ls));
|
||
}
|
||
|
||
|
||
|
||
// 百度查词组
|
||
//https://hanyu.baidu.com/s?wd=%E6%8C%87%E7%BB%84%E8%AF%8D&from=poem
|
||
}
|
||
|
||
static TextNode GetTextNode(string txt, List<WordsSearchResult> results)
|
||
{
|
||
TextNode[] nodes = new TextNode[txt.Length + 1];
|
||
for (int i = 0; i < nodes.Length; i++) { nodes[i] = new TextNode(); }
|
||
nodes[nodes.Length - 1].IsEnd = true;
|
||
|
||
foreach (var item in results) {
|
||
TextLine line = new TextLine() {
|
||
Start = item.Start,
|
||
End = item.End,
|
||
Keyword = item.Keyword,
|
||
};
|
||
nodes[item.Start].Children.Add(line);
|
||
line.Next = nodes[item.End + 1];
|
||
}
|
||
return nodes[0];
|
||
}
|
||
|
||
|
||
static void GetBuildPinyin(string c, List<string> ls)
|
||
{
|
||
var url = "https://www.baidu.com/baidu?wd={0}&ie=utf-8";
|
||
var ch = HttpUtility.UrlEncode(c).ToUpper();
|
||
var u = string.Format(url, ch);
|
||
|
||
WebClientEx web = new WebClientEx();
|
||
web.ResetHeaders();
|
||
web.Encoding = Encoding.UTF8;
|
||
var html = web.DownloadString(u);
|
||
html = Regex.Replace(html, "<!.*\n", "");
|
||
html = Regex.Replace(html, @"<head[^>]*?>[\s\S]*?</head>", "");
|
||
html = Regex.Replace(html, @"<script[^>]*?>[\s\S]*?</script>", "");
|
||
html = Regex.Replace(html, @"<style[^>]*?>[\s\S]*?</style>", "");
|
||
|
||
HtmlToText convert = new HtmlToText();
|
||
var t = convert.Convert(html);
|
||
|
||
var ms = Regex.Matches(t, "(拼音|读音|发音)(:|：)?[a-zāáǎàōóǒòēéěèīíǐìūúǔùǖǘǚǜü].*");
|
||
foreach (Match item in ms) {
|
||
ls.Add($"{c} {item.Value}");
|
||
}
|
||
ms = Regex.Matches(t, "(同|读作|异体字:).+");
|
||
foreach (Match item in ms) {
|
||
ls.Add($"{c} {item.Value}");
|
||
}
|
||
}
|
||
|
||
static int GetToneIndex(string text)
|
||
{
|
||
var tone = @"aāáǎàa|oōóǒòo|eēéěèe|iīíǐìi|uūúǔùu|vǖǘǚǜü"
|
||
.Replace("a", " ").Replace("o", " ").Replace("i", " ")
|
||
.Replace("u", " ").Replace("v", " ")
|
||
.Split('|');
|
||
foreach (var c in text) {
|
||
foreach (var to in tone) {
|
||
var index = to.IndexOf(c);
|
||
if (index > 0) {
|
||
return index;
|
||
}
|
||
}
|
||
}
|
||
return 5;
|
||
}
|
||
|
||
static string AddTone(string pinyin)
|
||
{
|
||
pinyin = pinyin.ToLower();
|
||
if (pinyin.Contains("j") || pinyin.Contains("q") || pinyin.Contains("x")) {
|
||
pinyin = pinyin.Replace("v", "u");
|
||
}
|
||
if (pinyin.Contains("iou")) {
|
||
pinyin = pinyin.Replace("iou", "iu");
|
||
}
|
||
if (pinyin.Contains("uei")) {
|
||
pinyin = pinyin.Replace("uei", "ui");
|
||
}
|
||
if (pinyin.Contains("uen")) {
|
||
pinyin = pinyin.Replace("uen", "un");
|
||
}
|
||
|
||
if (pinyin.EndsWith("1")) {
|
||
if (pinyin.Contains("a")) {
|
||
pinyin = pinyin.Replace("a", "ā");
|
||
} else if (pinyin.Contains("o")) {
|
||
pinyin = pinyin.Replace("o", "ō");
|
||
} else if (pinyin.Contains("e")) {
|
||
pinyin = pinyin.Replace("e", "ē");
|
||
|
||
} else if (pinyin.Contains("iu")) {
|
||
pinyin = pinyin.Replace("u", "ū");
|
||
} else if (pinyin.Contains("ui")) {
|
||
pinyin = pinyin.Replace("i", "ī");
|
||
|
||
} else if (pinyin.Contains("u")) {
|
||
pinyin = pinyin.Replace("u", "ū");
|
||
} else if (pinyin.Contains("i")) {
|
||
pinyin = pinyin.Replace("i", "ī");
|
||
} else if (pinyin.Contains("v")) {
|
||
pinyin = pinyin.Replace("v", "ǖ");
|
||
} else {
|
||
throw new Exception("");
|
||
}
|
||
} else if (pinyin.EndsWith("2")) {
|
||
if (pinyin.Contains("a")) {
|
||
pinyin = pinyin.Replace("a", "á");
|
||
} else if (pinyin.Contains("o")) {
|
||
pinyin = pinyin.Replace("o", "ó");
|
||
} else if (pinyin.Contains("e")) {
|
||
pinyin = pinyin.Replace("e", "é");
|
||
|
||
} else if (pinyin.Contains("iu")) {
|
||
pinyin = pinyin.Replace("u", "ú");
|
||
} else if (pinyin.Contains("ui")) {
|
||
pinyin = pinyin.Replace("i", "í");
|
||
|
||
|
||
} else if (pinyin.Contains("u")) {
|
||
pinyin = pinyin.Replace("u", "ú");
|
||
} else if (pinyin.Contains("i")) {
|
||
pinyin = pinyin.Replace("i", "í");
|
||
} else if (pinyin.Contains("v")) {
|
||
pinyin = pinyin.Replace("v", "ǘ");
|
||
} else {
|
||
throw new Exception("");
|
||
}
|
||
} else if (pinyin.EndsWith("3")) {
|
||
if (pinyin.Contains("a")) {
|
||
pinyin = pinyin.Replace("a", "ǎ");
|
||
} else if (pinyin.Contains("o")) {
|
||
pinyin = pinyin.Replace("o", "ǒ");
|
||
} else if (pinyin.Contains("e")) {
|
||
pinyin = pinyin.Replace("e", "ě");
|
||
|
||
} else if (pinyin.Contains("iu")) {
|
||
pinyin = pinyin.Replace("u", "ǔ");
|
||
} else if (pinyin.Contains("ui")) {
|
||
pinyin = pinyin.Replace("i", "ǐ");
|
||
|
||
} else if (pinyin.Contains("u")) {
|
||
pinyin = pinyin.Replace("u", "ǔ");
|
||
} else if (pinyin.Contains("i")) {
|
||
pinyin = pinyin.Replace("i", "ǐ");
|
||
} else if (pinyin.Contains("v")) {
|
||
pinyin = pinyin.Replace("v", "ǚ");
|
||
} else {
|
||
throw new Exception("");
|
||
}
|
||
} else if (pinyin.EndsWith("4")) {
|
||
if (pinyin.Contains("a")) {
|
||
pinyin = pinyin.Replace("a", "à");
|
||
} else if (pinyin.Contains("o")) {
|
||
pinyin = pinyin.Replace("o", "ò");
|
||
} else if (pinyin.Contains("e")) {
|
||
pinyin = pinyin.Replace("e", "è");
|
||
|
||
} else if (pinyin.Contains("iu")) {
|
||
pinyin = pinyin.Replace("u", "ù");
|
||
} else if (pinyin.Contains("ui")) {
|
||
pinyin = pinyin.Replace("i", "ì");
|
||
|
||
} else if (pinyin.Contains("u")) {
|
||
pinyin = pinyin.Replace("u", "ù");
|
||
} else if (pinyin.Contains("i")) {
|
||
pinyin = pinyin.Replace("i", "ì");
|
||
} else if (pinyin.Contains("v")) {
|
||
pinyin = pinyin.Replace("v", "ǜ");
|
||
} else {
|
||
throw new Exception("");
|
||
}
|
||
} else if (pinyin.EndsWith("0") || pinyin.EndsWith("5")) {
|
||
if (pinyin.Contains("a")) {
|
||
} else if (pinyin.Contains("o")) {
|
||
} else if (pinyin.Contains("e")) {
|
||
} else if (pinyin.Contains("i")) {
|
||
} else if (pinyin.Contains("u")) {
|
||
} else if (pinyin.Contains("v")) {
|
||
pinyin = pinyin.Replace("v", "ü");
|
||
} else {
|
||
throw new Exception("");
|
||
}
|
||
} else if (pinyin.EndsWith("6") || pinyin.EndsWith("7") || pinyin.EndsWith("8") || pinyin.EndsWith("9")) {
|
||
throw new Exception("");
|
||
}
|
||
pinyin = Regex.Replace(pinyin, @"\d", "");
|
||
|
||
return pinyin;
|
||
}
|
||
|
||
static string RemoveTone(string pinyin)
|
||
{
|
||
string s = "āáǎàōóǒòēéěèīíǐìūúǔùǖǘǚǜüńň";
|
||
string t = "aaaaooooeeeeiiiiuuuuvvvvvnnm";
|
||
for (int i = 0; i < s.Length; i++) {
|
||
pinyin = pinyin.Replace(s[i].ToString(), t[i].ToString());
|
||
}
|
||
return pinyin;
|
||
}
|
||
|
||
static string ReplaceForPinyin(string key, List<WordsSearchResult> all, Dictionary<string, string> dict)
|
||
{
|
||
LineNode[] lineNodes = new LineNode[key.Length + 1];
|
||
for (int i = 0; i < lineNodes.Length; i++) { lineNodes[i] = new LineNode(); }
|
||
lineNodes[lineNodes.Length - 1].IsEnd = true;
|
||
|
||
foreach (var item in all) {
|
||
LineText lineText = new LineText() {
|
||
Start = item.Start,
|
||
End = item.End,
|
||
Words = item.Keyword,
|
||
Pinyin = dict[item.Keyword],
|
||
NextNode = lineNodes[item.End + 1]
|
||
};
|
||
lineNodes[item.Start].Children.Add(lineText);
|
||
}
|
||
return GetPinyin(lineNodes[0], "");
|
||
}
|
||
static string GetPinyin(LineNode node, string pinyin)
|
||
{
|
||
if (node.IsEnd) {
|
||
return string.Join(",", pinyin.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries));
|
||
}
|
||
foreach (var item in node.Children) {
|
||
var py = pinyin + "," + item.Pinyin;
|
||
var s = GetPinyin(item.NextNode, py);
|
||
if (s != null) {
|
||
return s;
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
static List<string> GetWords()
|
||
{
|
||
var files = Directory.GetFiles("Scel");
|
||
HashSet<string> list = new HashSet<string>();
|
||
|
||
foreach (var file in files) {
|
||
GetWords(file, list);
|
||
}
|
||
return list.OrderBy(q => q).ToList();
|
||
}
|
||
|
||
static void GetWords(string file, HashSet<string> list)
|
||
{
|
||
SougouPinyinScel scel = new SougouPinyinScel();
|
||
var t = scel.Import(file);
|
||
|
||
foreach (var item in t) {
|
||
var w = item.Word.Trim();
|
||
var py = string.Join(",", item.Pinyin);
|
||
|
||
list.Add($"{w} {py}");
|
||
}
|
||
}
|
||
|
||
static List<string> GetPinyin()
|
||
{
|
||
var files = Directory.GetFiles("dict", "*.txt");
|
||
HashSet<string> list = new HashSet<string>();
|
||
|
||
foreach (var file in files) {
|
||
GetPinyin(file, list);
|
||
}
|
||
return list.OrderBy(q => q).ToList();
|
||
}
|
||
|
||
static void GetPinyin(string file, HashSet<string> list)
|
||
{
|
||
var text = File.ReadAllText(file);
|
||
var line = text.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
|
||
foreach (var l in line) {
|
||
var t = Regex.Replace(l, "//.*[\r\n]?", "");
|
||
if (string.IsNullOrWhiteSpace(t)) {
|
||
continue;
|
||
}
|
||
var sp = t.Split("\t,:| '\"=>[]， 123456789?".ToArray(), StringSplitOptions.RemoveEmptyEntries);
|
||
list.Add(string.Join(",", sp));
|
||
}
|
||
}
|
||
|
||
|
||
private static int GetLength(string text)
|
||
{
|
||
var stringInfo = new System.Globalization.StringInfo(text);
|
||
return stringInfo.LengthInTextElements;
|
||
}
|
||
|
||
public static String Substring(String text, int start, int end)
|
||
{
|
||
var stringInfo = new System.Globalization.StringInfo(text);
|
||
return stringInfo.SubstringByTextElements(start, end);
|
||
}
|
||
public class LineNode
|
||
{
|
||
public bool IsEnd { get; set; }
|
||
public List<LineText> Children = new List<LineText>();
|
||
}
|
||
|
||
public class LineText
|
||
{
|
||
public int Start;
|
||
public int End;
|
||
public string Words;
|
||
public string Pinyin;
|
||
public LineNode NextNode;
|
||
}
|
||
|
||
}
|
||
}
|