using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text.RegularExpressions; using ToolGood.Bedrock; namespace ToolGood.PinYin.Build { class Program { static void Main(string[] args) { // 生成单字拼音 var pyShow = new List() { "" }; var upyShow = new List(); var pyText = File.ReadAllText("dict\\_py.txt"); var pyLines = pyText.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); Dictionary> dict = new Dictionary>(); foreach (var line in pyLines) { var sp = line.Split("\t,:| '\"=>[], 123456789?".ToArray(), StringSplitOptions.RemoveEmptyEntries); for (int i = 1; i < sp.Length; i++) { var py = sp[i]; //pyName.Add(py.ToUpper()[0] + py.Substring(1)); upyShow.Add(py.ToLower()); } } upyShow = upyShow.Distinct().OrderBy(q => q).ToList(); foreach (var item in upyShow) { var py = RemoveTone(item); pyShow.Add(py.ToUpper()[0] + py.Substring(1)); pyShow.Add(item.ToUpper()[0] + item.Substring(1)); } foreach (var line in pyLines) { var sp = line.Split("\t,:| '\"=>-[], 123456789?".ToArray(), StringSplitOptions.RemoveEmptyEntries); if (sp.Length > 1) { var key = sp[0]; List indexs = new List(); for (int i = 1; i < sp.Length; i++) { var py = sp[i]; var idx = upyShow.IndexOf(py) * 2 + 1; if (idx == -1) { throw new Exception(""); } indexs.Add(idx); } dict[key] = indexs; } } List pyData = new List(); for (int i = 0x3400; i <= 0x9fd5; i++) { var c = ((char)i).ToString(); if (dict.TryGetValue(c, out List indexs)) { List idxs = new List(); foreach (var index in indexs) { idxs.Add(index.ToString("X")); } if (idxs[0] == "FFFFFFFF") { throw new Exception(""); } pyData.Add(string.Join(",", idxs)); } else { pyData.Add("0"); } } var outText = string.Join(",", pyShow); outText += "\n" + string.Join("\n", pyData); File.WriteAllText("pyIndex.txt", outText); Compression("pyIndex.txt"); File.WriteAllText("_pyShow.js.txt", Newtonsoft.Json.JsonConvert.SerializeObject(pyShow)); List pyIndex2 = new List() { 0 }; List pyData2 = new List(); for (int i = 0; i < pyData.Count; i++) { var idxs = pyData[i]; if (idxs != "0") { foreach (var idx in idxs.Split(',')) { pyData2.Add(ushort.Parse(idx, System.Globalization.NumberStyles.HexNumber)); } } pyIndex2.Add((ushort)pyData2.Count); } File.WriteAllText("_pyIndex.js.txt", Newtonsoft.Json.JsonConvert.SerializeObject(pyIndex2)); File.WriteAllText("_pyData.js.txt", Newtonsoft.Json.JsonConvert.SerializeObject(pyData2)); // 获取 姓名拼音 Dictionary> pyName = new Dictionary>(); pyText = File.ReadAllText("dict\\_pyName.txt"); pyLines = pyText.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); foreach (var line in pyLines) { var sp = line.Split("\t,:| '\"=>-[], ?".ToArray(), StringSplitOptions.RemoveEmptyEntries); if (sp.Length > 1) { var key = sp[0]; if (key == "单") { } List indexs = new List(); for (int i = 1; i < sp.Length; i++) { var py = sp[i]; py = AddTone(py); var idx = upyShow.IndexOf(py) * 2 + 1; if (idx == -1) { throw new Exception(""); } indexs.Add(idx); } pyName[key] = indexs; } } List ls = new List(); foreach (var item in pyName) { List idx = new List(); List idxs = new List(); foreach (var index in item.Value) { idxs.Add(index.ToString("X")); } ls.Add($"{item.Key},{string.Join(",", idxs)}"); } File.WriteAllText("pyName.txt", string.Join("\n", ls)); Compression("pyName.txt"); File.WriteAllText("_pyName.js.txt", Newtonsoft.Json.JsonConvert.SerializeObject(pyName)); //生成多字拼音 Dictionary> pyWords = new Dictionary>(); pyText = File.ReadAllText("dict\\_word.txt"); pyLines = pyText.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); foreach (var line in pyLines) { var sp = line.Split(", ".ToCharArray(), StringSplitOptions.RemoveEmptyEntries).ToList(); var key = sp[0]; if (key.Length == 1) { continue; } sp.RemoveAt(0); pyWords[key] = sp; } // 搜狗拼音也有错误的 pyText = File.ReadAllText("dict\\_wordRevise.txt"); pyLines = pyText.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); foreach (var line in pyLines) { var sp = line.Split(", []=|:\t".ToCharArray(), StringSplitOptions.RemoveEmptyEntries).ToList(); var key = sp[0]; if (key.Length == 1) { continue; } sp.RemoveAt(0); pyWords[key] = sp; } //Words.StringSearch stringSearch = new Words.StringSearch(); //stringSearch.SetKeywords(pyWords.Keys.ToList()); Dictionary> tempClearWords = new Dictionary>(); List tempClearKeys = new List(); foreach (var item in pyWords) { var py = Words.WordsHelper.GetPinYinFast(item.Key, true).ToLower(); if (py == string.Join("", item.Value)) { tempClearWords[item.Key] = item.Value; tempClearKeys.Add(item.Key); } } var pyWords2 = new Dictionary>(); foreach (var item in pyWords) { pyWords2[item.Key] = item.Value; } foreach (var item in tempClearWords) { pyWords2.Remove(item.Key); } List AddKeys = new List(); var keys = pyWords2.Select(q => q.Key).ToList(); Words.WordsSearch wordsSearch = new Words.WordsSearch(); wordsSearch.SetKeywords(keys); foreach (var item in tempClearKeys) { if (wordsSearch.ContainsAny(item)) { AddKeys.Add(item); } } HashSet starts = new HashSet(); HashSet ends = new HashSet(); foreach (var item in tempClearKeys) { for (int i = 1; i < item.Length; i++) { var start = item.Substring(0, item.Length - i); var end = item.Substring(i); ends.Add(start); starts.Add(end); } } List AddKeys2 = new List(); List keys2 = new List(); foreach (var item in pyWords2) { var py = Words.WordsHelper.GetPinYinFast(item.Key, true).ToLower(); if (RemoveTone(py) != RemoveTone(string.Join("", item.Value))) { for (int i = 1; i < item.Key.Length; i++) { var start = item.Key.Substring(0, item.Key.Length - i); if (keys2.Contains(start)) { continue; } var end = item.Key.Substring(i); if (starts.Contains(start) && ends.Contains(end)) { keys2.Add(start); } } } } keys2 = keys2.Distinct().ToList(); wordsSearch = new Words.WordsSearch(); wordsSearch.SetKeywords(keys2); foreach (var item in tempClearKeys) { if (item.Length >= 4) { continue; } //排除诗句 歇后语 var all = wordsSearch.FindAll(item); if (all.Any(q => q.End + 1 == item.Length)) { AddKeys2.Add(item); } } AddKeys.AddRange(AddKeys2); AddKeys.AddRange(keys); AddKeys = AddKeys.Distinct().ToList(); ls = new List(); foreach (var item in AddKeys) { var str = item; List pys = pyWords[str]; foreach (var py in pys) { var idx = upyShow.IndexOf(py) * 2 + 1; if (idx == -1) { throw new Exception(""); } str += "," + idx.ToString("X"); } ls.Add(str); } ls = ls.OrderBy(q => q).ToList(); File.WriteAllText("pyWords.txt", string.Join("\n", ls)); Compression("pyWords.txt"); //File.WriteAllText("pyWords.js.txt", string.Join("|", ls)); File.WriteAllText("_pyWordsKey.js.txt", Newtonsoft.Json.JsonConvert.SerializeObject(AddKeys)); pyIndex2 = new List() { 0 }; pyData2 = new List(); foreach (var item in AddKeys) { var str = item; List pys = pyWords[str]; foreach (var py in pys) { var idx = upyShow.IndexOf(py) * 2 + 1; if (idx == -1) { throw new Exception(""); } pyData2.Add(idx); } pyIndex2.Add(pyData2.Count); } File.WriteAllText("_pyWordsIndex.js.txt", Newtonsoft.Json.JsonConvert.SerializeObject(pyIndex2)); File.WriteAllText("_pyWordsData.js.txt", Newtonsoft.Json.JsonConvert.SerializeObject(pyData2)); } static string RemoveTone(string pinyin) { string s = "āáǎàōóǒòēéěèīíǐìūúǔùǖǘǚǜüńň"; string t = "aaaaooooeeeeiiiiuuuuvvvvvnnm"; for (int i = 0; i < s.Length; i++) { pinyin = pinyin.Replace(s[i].ToString(), t[i].ToString()); } return pinyin; } static string AddTone(string pinyin) { pinyin = pinyin.ToLower(); if (pinyin.Contains("j") || pinyin.Contains("q") || pinyin.Contains("x")) { pinyin = pinyin.Replace("v", "u"); } if (pinyin.Contains("iou")) { pinyin = pinyin.Replace("iou", "iu"); } if (pinyin.Contains("uei")) { pinyin = pinyin.Replace("uei", "ui"); } if (pinyin.Contains("uen")) { pinyin = pinyin.Replace("uen", "un"); } if (pinyin.EndsWith("1")) { if (pinyin.Contains("a")) { pinyin = pinyin.Replace("a", "ā"); } else if (pinyin.Contains("o")) { pinyin = pinyin.Replace("o", "ō"); } else if (pinyin.Contains("e")) { pinyin = pinyin.Replace("e", "ē"); } else if (pinyin.Contains("iu")) { pinyin = pinyin.Replace("u", "ū"); } else if (pinyin.Contains("ui")) { pinyin = pinyin.Replace("i", "ī"); } else if (pinyin.Contains("u")) { pinyin = pinyin.Replace("u", "ū"); } else if (pinyin.Contains("i")) { pinyin = pinyin.Replace("i", "ī"); } else if (pinyin.Contains("v")) { pinyin = pinyin.Replace("v", "ǖ"); } else { throw new Exception(""); } } else if (pinyin.EndsWith("2")) { if (pinyin.Contains("a")) { pinyin = pinyin.Replace("a", "á"); } else if (pinyin.Contains("o")) { pinyin = pinyin.Replace("o", "ó"); } else if (pinyin.Contains("e")) { pinyin = pinyin.Replace("e", "é"); } else if (pinyin.Contains("iu")) { pinyin = pinyin.Replace("u", "ú"); } else if (pinyin.Contains("ui")) { pinyin = pinyin.Replace("i", "í"); } else if (pinyin.Contains("u")) { pinyin = pinyin.Replace("u", "ú"); } else if (pinyin.Contains("i")) { pinyin = pinyin.Replace("i", "í"); } else if (pinyin.Contains("v")) { pinyin = pinyin.Replace("v", "ǘ"); } else { throw new Exception(""); } } else if (pinyin.EndsWith("3")) { if (pinyin.Contains("a")) { pinyin = pinyin.Replace("a", "ǎ"); } else if (pinyin.Contains("o")) { pinyin = pinyin.Replace("o", "ǒ"); } else if (pinyin.Contains("e")) { pinyin = pinyin.Replace("e", "ě"); } else if (pinyin.Contains("iu")) { pinyin = pinyin.Replace("u", "ǔ"); } else if (pinyin.Contains("ui")) { pinyin = pinyin.Replace("i", "ǐ"); } else if (pinyin.Contains("u")) { pinyin = pinyin.Replace("u", "ǔ"); } else if (pinyin.Contains("i")) { pinyin = pinyin.Replace("i", "ǐ"); } else if (pinyin.Contains("v")) { pinyin = pinyin.Replace("v", "ǚ"); } else { throw new Exception(""); } } else if (pinyin.EndsWith("4")) { if (pinyin.Contains("a")) { pinyin = pinyin.Replace("a", "à"); } else if (pinyin.Contains("o")) { pinyin = pinyin.Replace("o", "ò"); } else if (pinyin.Contains("e")) { pinyin = pinyin.Replace("e", "è"); } else if (pinyin.Contains("iu")) { pinyin = pinyin.Replace("u", "ù"); } else if (pinyin.Contains("ui")) { pinyin = pinyin.Replace("i", "ì"); } else if (pinyin.Contains("u")) { pinyin = pinyin.Replace("u", "ù"); } else if (pinyin.Contains("i")) { pinyin = pinyin.Replace("i", "ì"); } else if (pinyin.Contains("v")) { pinyin = pinyin.Replace("v", "ǜ"); } else { throw new Exception(""); } } else if (pinyin.EndsWith("0") || pinyin.EndsWith("5")) { if (pinyin.Contains("a")) { } else if (pinyin.Contains("o")) { } else if (pinyin.Contains("e")) { } else if (pinyin.Contains("i")) { } else if (pinyin.Contains("u")) { } else if (pinyin.Contains("v")) { pinyin = pinyin.Replace("v", "ü"); } else { throw new Exception(""); } } else if (pinyin.EndsWith("6") || pinyin.EndsWith("7") || pinyin.EndsWith("8") || pinyin.EndsWith("9")) { throw new Exception(""); } pinyin = Regex.Replace(pinyin, @"\d", ""); return pinyin; } private static void Compression(string file) { var bytes = File.ReadAllBytes(file); var bs = CompressionUtil.GzipCompress(bytes); Directory.CreateDirectory("dict"); File.WriteAllBytes("dict\\" + file + ".z", bs); var bs2 = CompressionUtil.BrCompress(bytes); File.WriteAllBytes("dict\\" + file + ".br", bs2); } static int GetToneIndex(string text) { var tone = @"aāáǎàa|oōóǒòo|eēéěèe|iīíǐìi|uūúǔùu|vǖǘǚǜü" .Replace("a", " ").Replace("o", " ").Replace("i", " ") .Replace("u", " ").Replace("v", " ") .Split('|'); foreach (var c in text) { foreach (var to in tone) { var index = to.IndexOf(c); if (index > 0) { return index; } } } return 5; } } }