Merge pull request #14 from bb-io/develop

Develop
bb-io · Aug 27, 2024 · 8575747 · 8575747
2 parents a7bad86 + 8f18c7b
commit 8575747
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 19 deletions.
diff --git a/Apps.AzueOpenAI/Actions/XliffActions.cs b/Apps.AzueOpenAI/Actions/XliffActions.cs
@@ -61,11 +61,12 @@ public async Task<TranslateXliffResponse> TranslateXliff(
             bucketSize ?? 1500,
             glossary.Glossary, promptRequest);
 
+        var updatedResults = Utils.Xliff.Extensions.CheckTagIssues(xliffDocument.TranslationUnits, translatedTexts);
         var stream = await _fileManagementClient.DownloadAsync(input.File);
-        var updatedFile = Blackbird.Xliff.Utils.Utils.XliffExtensions.UpdateOriginalFile(stream, translatedTexts);
+        var updatedFile = Blackbird.Xliff.Utils.Utils.XliffExtensions.UpdateOriginalFile(stream, updatedResults);
         string contentType = input.File.ContentType ?? "application/xml";
         var fileReference = await _fileManagementClient.UploadAsync(updatedFile, contentType, input.File.Name);
-        return new TranslateXliffResponse { File = fileReference, Usage = usage };
+        return new TranslateXliffResponse { File = fileReference, Usage = usage, Changes = updatedResults.Count };
     }
 
     [Action("Get Quality Scores for XLIFF file",
@@ -338,7 +339,9 @@ private string GetSystemPrompt(bool translator)
         {
             string json = JsonConvert.SerializeObject(batch.Select(x => "{ID:" + x.Id + "}" + x.Source));
 
-            var userPrompt = GetUserPrompt(prompt, xliff, json);
+            var userPrompt = GetUserPrompt(prompt +
+                "Reply with the processed text preserving the same format structure as provided, your output will need to be deserialized programmatically afterwards.",
+                xliff, json);
 
             if (glossary != null)
             {
@@ -359,30 +362,31 @@ private string GetSystemPrompt(bool translator)
 
             usageDto += promptUsage;
             var translatedText = response.Trim();
-
+            string filteredText = "";
             try
             {
-                var result = JsonConvert.DeserializeObject<string[]>(Regex.Match(translatedText,"\\[[\\s\\S]+\\]").Value);
-
-                if (result.Length != batch.Count())
+                 filteredText = Regex.Match(translatedText, "\\[[\\s\\S]+(\\])").Value;
+                if (String.IsNullOrEmpty(filteredText))
                 {
-                    throw new InvalidOperationException(
-                        "OpenAI returned inappropriate response. " +
-                        "The number of translated texts does not match the number of source texts. " +
-                        "Probably there is a duplication or a missing text in translation unit. " +
-                        "Try change model or bucket size (to lower values) or add retries to this action.");
+                    var index = translatedText.LastIndexOf("\",") == -1 ? translatedText.LastIndexOf("\"\n,") : translatedText.LastIndexOf("\",");
+                    index = index == -1 ? translatedText.LastIndexOf("\n\",") == -1? translatedText.LastIndexOf("\\n\",") : translatedText.LastIndexOf("\n\",") : index;
+                    filteredText = translatedText.Remove(index) + "\"]"; 
                 }
+                filteredText = Regex.Match(filteredText, "\\[[\\s\\S]+(\\])").Value;
+                var result = JsonConvert.DeserializeObject<string[]>(filteredText);
 
                 results.AddRange(result);
             }
             catch (Exception e)
-            {
-                throw new Exception(
+                {
+                    throw new Exception(
                     $"Failed to parse the translated text. Exception message: {e.Message}; Exception type: {e.GetType()}");
             }
+
         }
+
+        return (results.Where(z => Regex.Match(z ,"\\{ID:(.*?)\\}(.+)$").Groups[1].Value != "").ToDictionary(x => Regex.Match(x, "\\{ID:(.*?)\\}(.+)$").Groups[1].Value, y => Regex.Match(y, "\\{ID:(.*?)\\}(.+)$").Groups[2].Value), usageDto);
 
-        return (results.ToDictionary(x => Regex.Match(x, "\\{ID:(.*?)\\}(.+)$").Groups[1].Value, y => Regex.Match(y, "\\{ID:(.*?)\\}(.+)$").Groups[2].Value), usageDto);
     }
 
     string GetUserPrompt(string prompt, ParsedXliff xliffDocument, string json)

diff --git a/Apps.AzueOpenAI/Apps.AzureOpenAI.csproj b/Apps.AzueOpenAI/Apps.AzureOpenAI.csproj
@@ -6,7 +6,7 @@
     <Nullable>enable</Nullable>
 	<Product>Azure OpenAI</Product>
 	<Description>Azure OpenAI Service offers industry-leading coding and language AI models that you can fine-tune to your specific needs for a variety of use cases.</Description>
-	<Version>1.1.2</Version>
+	<Version>1.1.3</Version>
 	<AssemblyName>Apps.AzureOpenAI</AssemblyName>
   </PropertyGroup>
 
@@ -27,4 +27,4 @@
 	<ItemGroup>
 	  <Folder Include="Utils\Xliff\" />
 	</ItemGroup>
-</Project>
+</Project>
diff --git a/Apps.AzueOpenAI/Models/Responses/Xliff/TranslateXliffResponse.cs b/Apps.AzueOpenAI/Models/Responses/Xliff/TranslateXliffResponse.cs
@@ -7,4 +7,6 @@ public class TranslateXliffResponse
 {
     public FileReference File { get; set; }
     public UsageDto Usage { get; set; }
+
+    public int Changes { get; set; }
 }
diff --git a/Apps.AzueOpenAI/Utils/Xliff/Extensions.cs b/Apps.AzueOpenAI/Utils/Xliff/Extensions.cs
@@ -59,12 +59,13 @@ private static List<Tag> GetTags(string src)
         foreach ( var tag in tags.Select(x => x.Value)) 
         {
             count++;
+            var type = Regex.Match(tag, "<(.*?) ").Groups[1].Value;
             parsedTags.Add(new Tag 
             {
                 Position = count,
                 Id = Regex.Match(tag.ToLower(), "id=\"(.*?)\"").Groups[1].Value,
-                Value = tag,
-                Type = Regex.Match(tag, "<(.*?) ").Groups[1].Value
+                Value = type == "g" || type == "x" ? "" : tag,
+                Type = type
             });
 
         }