From 14fb5d00e8cceef70fa42180cee3258b700710f8 Mon Sep 17 00:00:00 2001 From: Tabing010102 Date: Sat, 9 Mar 2024 23:10:28 +0800 Subject: [PATCH 1/5] add dict support --- SakuraTranslator/SakuraTranslatorEndpoint.cs | 127 +++++++++++++++++-- 1 file changed, 116 insertions(+), 11 deletions(-) diff --git a/SakuraTranslator/SakuraTranslatorEndpoint.cs b/SakuraTranslator/SakuraTranslatorEndpoint.cs index 8d47da9..c55129c 100644 --- a/SakuraTranslator/SakuraTranslatorEndpoint.cs +++ b/SakuraTranslator/SakuraTranslatorEndpoint.cs @@ -1,6 +1,10 @@ -using System; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System; using System.Collections; +using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net; using System.Text; using XUnity.AutoTranslator.Plugin.Core.Endpoints; @@ -20,11 +24,49 @@ public class SakuraTranslatorEndpoint : ITranslateEndpoint // params private string _endpoint; private string _apiType; + private bool _useDict; + private string _dictMode; + private Dictionary _dict; + + // local var + private string _fullDictStr; public void Initialize(IInitializationContext context) { _endpoint = context.GetOrCreateSetting("Sakura", "Endpoint", "http://127.0.0.1:8080/completion"); _apiType = context.GetOrCreateSetting("Sakura", "ApiType", string.Empty); + if (!bool.TryParse(context.GetOrCreateSetting("Sakura", "UseDict", string.Empty), out _useDict)) + { + _useDict = false; + } + _dictMode = context.GetOrCreateSetting("Sakura", "DictMode", "Full"); + var dictStr = context.GetOrCreateSetting("Sakura", "Dict", string.Empty); + if (!string.IsNullOrEmpty(dictStr)) + { + try + { + _dict = new Dictionary(); + JObject dictJObj = JsonConvert.DeserializeObject(dictStr) as JObject; + foreach (var item in dictJObj) + { + _dict.Add(item.Key, item.Value.ToString()); + } + if (_dict.Count == 0) + { + _useDict = false; + _fullDictStr = string.Empty; + } + else + { + _fullDictStr = string.Join("\n", _dict.Select(x => $"{x.Key}->{x.Value}").ToArray()); + } + } + catch + { + _useDict = false; + _fullDictStr = string.Empty; + } + } } public IEnumerator Translate(ITranslationContext context) @@ -118,9 +160,67 @@ private string MakeRequestJson(string line) } else if (_apiType == "OpenAI") { - json = $"{{" + - $"\"model\": \"sukinishiro\"," + - $"\"messages\": [" + + json = MakeOpenAIPrompt(line); + } + else + { + json = $"{{\"frequency_penalty\": 0.2, \"n_predict\": 1000, \"prompt\": \"将下面的日文文本翻译成中文：{line}\", \"repeat_penalty\": 1, \"temperature\": 0.1, \"top_k\": 40, \"top_p\": 0.3}}"; + } + + return json; + } + + private string MakeOpenAIPrompt(string line) + { + string messagesStr = string.Empty; + if (_useDict) + { + var messages = new List + { + new PromptMessage + { + role = "system", + content = "你是一个轻小说翻译模型，可以流畅通顺地以日本轻小说的风格将日文翻译成简体中文，并联系上下文正确使用人称代词，注意不要擅自添加原文中没有的代词，也不要擅自增加或减少换行。" + } + }; + string dictStr; + if (_dictMode == "Full") + { + dictStr = _fullDictStr; + } + else + { + var usedDict = _dict.Where(x => line.Contains(x.Key)); + if (usedDict.Count() > 0) + { + dictStr = string.Join("\n", usedDict.Select(x => $"{x.Key}->{x.Value}").ToArray()); + } + else + { + dictStr = string.Empty; + } + } + if (string.IsNullOrEmpty(dictStr)) + { + messages.Add(new PromptMessage + { + role = "user", + content = $"将下面的日文文本翻译成中文：{line}" + }); + } + else + { + messages.Add(new PromptMessage + { + role = "user", + content = $"根据以下术语表：\n{dictStr}\n将下面的日文文本根据上述术语表的对应关系和注释翻译成中文：{line}" + }); + } + messagesStr = JsonConvert.SerializeObject(messages, Formatting.None); + } + else + { + messagesStr = "[" + $"{{" + $"\"role\": \"system\"," + $"\"content\": \"你是一个轻小说翻译模型，可以流畅通顺地以日本轻小说的风格将日文翻译成简体中文，并联系上下文正确使用人称代词，不擅自添加原文中没有的代词。\"" + @@ -129,7 +229,13 @@ private string MakeRequestJson(string line) $"\"role\": \"user\"," + $"\"content\": \"将下面的日文文本翻译成中文：{line}\"" + $"}}" + - $"]," + + $"]"; + } + return $"{{" + + $"\"model\": \"sukinishiro\"," + + $"\"messages\": " + + messagesStr + + $"," + $"\"temperature\": 0.1," + $"\"top_p\": 0.3," + $"\"max_tokens\": 1000," + @@ -139,13 +245,12 @@ private string MakeRequestJson(string line) $"\"um_beams\": 1," + $"\"repetition_penalty\": 1.0" + $"}}"; - } - else - { - json = $"{{\"frequency_penalty\": 0.2, \"n_predict\": 1000, \"prompt\": \"将下面的日文文本翻译成中文：{line}\", \"repeat_penalty\": 1, \"temperature\": 0.1, \"top_k\": 40, \"top_p\": 0.3}}"; - } + } - return json; + class PromptMessage + { + public string role { get; set; } + public string content { get; set; } } } } From 8dba2dce4345cd77834f0c949a2265c8fcbd34f7 Mon Sep 17 00:00:00 2001 From: Tabing010102 Date: Sat, 9 Mar 2024 23:31:41 +0800 Subject: [PATCH 2/5] add assembly version --- SakuraTranslator/SakuraTranslatorEndpoint.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/SakuraTranslator/SakuraTranslatorEndpoint.cs b/SakuraTranslator/SakuraTranslatorEndpoint.cs index c55129c..2f57f6b 100644 --- a/SakuraTranslator/SakuraTranslatorEndpoint.cs +++ b/SakuraTranslator/SakuraTranslatorEndpoint.cs @@ -6,9 +6,13 @@ using System.IO; using System.Linq; using System.Net; +using System.Reflection; using System.Text; using XUnity.AutoTranslator.Plugin.Core.Endpoints; +[assembly: AssemblyVersion("0.2.3")] +[assembly: AssemblyFileVersion("0.2.3")] + namespace SakuraTranslator { public class SakuraTranslatorEndpoint : ITranslateEndpoint From 6a7b88e28e7dfbfec01d76d7beb5511e813f2364 Mon Sep 17 00:00:00 2001 From: Tabing010102 Date: Sun, 10 Mar 2024 00:07:39 +0800 Subject: [PATCH 3/5] replace JsonSerializer --- SakuraTranslator/SakuraTranslatorEndpoint.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/SakuraTranslator/SakuraTranslatorEndpoint.cs b/SakuraTranslator/SakuraTranslatorEndpoint.cs index 2f57f6b..bf73615 100644 --- a/SakuraTranslator/SakuraTranslatorEndpoint.cs +++ b/SakuraTranslator/SakuraTranslatorEndpoint.cs @@ -220,7 +220,7 @@ private string MakeOpenAIPrompt(string line) content = $"根据以下术语表：\n{dictStr}\n将下面的日文文本根据上述术语表的对应关系和注释翻译成中文：{line}" }); } - messagesStr = JsonConvert.SerializeObject(messages, Formatting.None); + messagesStr = SerializePromptMessages(messages); } else { @@ -251,6 +251,15 @@ private string MakeOpenAIPrompt(string line) $"}}"; } + private string SerializePromptMessages(List messages) + { + string result = "["; + result += string.Join(",", messages.Select(x => $"{{\"role\":\"{x.role}\",\"content\":\"{x.content}\"}}").ToArray()); + result += "]"; + result = result.Replace("\n", "\\n"); + return result; + } + class PromptMessage { public string role { get; set; } From 533f04f34257031a1a2c87cce63ebf58e3e860e3 Mon Sep 17 00:00:00 2001 From: Tabing010102 Date: Sun, 10 Mar 2024 09:17:10 +0800 Subject: [PATCH 4/5] add EscapeJsonString --- SakuraTranslator/SakuraTranslatorEndpoint.cs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/SakuraTranslator/SakuraTranslatorEndpoint.cs b/SakuraTranslator/SakuraTranslatorEndpoint.cs index bf73615..c8f1769 100644 --- a/SakuraTranslator/SakuraTranslatorEndpoint.cs +++ b/SakuraTranslator/SakuraTranslatorEndpoint.cs @@ -254,12 +254,26 @@ private string MakeOpenAIPrompt(string line) private string SerializePromptMessages(List messages) { string result = "["; - result += string.Join(",", messages.Select(x => $"{{\"role\":\"{x.role}\",\"content\":\"{x.content}\"}}").ToArray()); + result += string.Join(",", messages.Select(x => $"{{\"role\":\"{x.role}\"," + + $"\"content\":\"{EscapeJsonString(x.content)}\"}}").ToArray()); result += "]"; - result = result.Replace("\n", "\\n"); return result; } + private string EscapeJsonString(string str) + { + return str + .Replace("\\", "\\\\") + .Replace("/", "\\/") + .Replace("\b", "\\b") + .Replace("\f", "\\f") + .Replace("\n", "\\n") + .Replace("\r", "\\r") + .Replace("\t", "\\t") + .Replace("\v", "\\v") + .Replace("\"", "\\\""); + } + class PromptMessage { public string role { get; set; } From 225431fc7935bee9ad98d20065c45550efd873cc Mon Sep 17 00:00:00 2001 From: Tabing010102 Date: Sun, 10 Mar 2024 09:17:43 +0800 Subject: [PATCH 5/5] change naming --- SakuraTranslator/SakuraTranslatorEndpoint.cs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/SakuraTranslator/SakuraTranslatorEndpoint.cs b/SakuraTranslator/SakuraTranslatorEndpoint.cs index c8f1769..435d119 100644 --- a/SakuraTranslator/SakuraTranslatorEndpoint.cs +++ b/SakuraTranslator/SakuraTranslatorEndpoint.cs @@ -183,8 +183,8 @@ private string MakeOpenAIPrompt(string line) { new PromptMessage { - role = "system", - content = "你是一个轻小说翻译模型，可以流畅通顺地以日本轻小说的风格将日文翻译成简体中文，并联系上下文正确使用人称代词，注意不要擅自添加原文中没有的代词，也不要擅自增加或减少换行。" + Role = "system", + Content = "你是一个轻小说翻译模型，可以流畅通顺地以日本轻小说的风格将日文翻译成简体中文，并联系上下文正确使用人称代词，注意不要擅自添加原文中没有的代词，也不要擅自增加或减少换行。" } }; string dictStr; @@ -208,16 +208,16 @@ private string MakeOpenAIPrompt(string line) { messages.Add(new PromptMessage { - role = "user", - content = $"将下面的日文文本翻译成中文：{line}" + Role = "user", + Content = $"将下面的日文文本翻译成中文：{line}" }); } else { messages.Add(new PromptMessage { - role = "user", - content = $"根据以下术语表：\n{dictStr}\n将下面的日文文本根据上述术语表的对应关系和注释翻译成中文：{line}" + Role = "user", + Content = $"根据以下术语表：\n{dictStr}\n将下面的日文文本根据上述术语表的对应关系和注释翻译成中文：{line}" }); } messagesStr = SerializePromptMessages(messages); @@ -254,8 +254,8 @@ private string MakeOpenAIPrompt(string line) private string SerializePromptMessages(List messages) { string result = "["; - result += string.Join(",", messages.Select(x => $"{{\"role\":\"{x.role}\"," + - $"\"content\":\"{EscapeJsonString(x.content)}\"}}").ToArray()); + result += string.Join(",", messages.Select(x => $"{{\"role\":\"{x.Role}\"," + + $"\"content\":\"{EscapeJsonString(x.Content)}\"}}").ToArray()); result += "]"; return result; } @@ -276,8 +276,8 @@ private string EscapeJsonString(string str) class PromptMessage { - public string role { get; set; } - public string content { get; set; } + public string Role { get; set; } + public string Content { get; set; } } } }