Dev (#2)

+ Added `KwsCatalog.Merge` + Added `KwsKeyword.Merge` + Added `KwsCatalog.UnvalidatedKeywords` + Added `KwsKeyword.Document` + Added `I8NToolAppConfig.TextFileExtensions` + Added `I8NToolConfig.TextFileExtensions` + Added extractor API `--textFileExt` argument + Added `wan24I8NTool:exclude` + Added `gettextd` to the extractor default configuration + Added `KwsCatalog.Properties` + Added `KwsKeyword.Properties`
nd1012 · Mar 17, 2024 · 8366b4d · 8366b4d
1 parent 5a5cc8f
commit 8366b4d
Show file tree

Hide file tree

Showing 19 changed files with 408 additions and 55 deletions.
diff --git a/README.md b/README.md
@@ -67,7 +67,7 @@ expressions, until the final quoted keyword was extracted from a line.
 These default translation method names can be parsed from source code:
 
 - `_`
-- `gettext(n)("..."`
+- `gettext(n|d)("..."`
 - `Translate(Plural)("..."`
 - `GetTerm("..."`
 
@@ -89,16 +89,26 @@ argument after a numeric argument:
 
 - `ExitCode(N, "..."`
 
+**NOTE**: (Multiline) concatenated string value definitions (like 
+`@"Part \ a" + $"Part b {variable}"`) or interpolations can't be parsed. The 
+matched string literal must be JSON style escaped.
+
 To force including any string (from a constant definition, for example), 
 simply add a comment `wan24I8NTool:include` at the end of the line - example:
 
 ```cs
 public const string NAME = "Any PO included keyword";// wan24I8NTool:include
 ```
 
-**NOTE**: (Multiline) concatenated string value definitions (like 
-`@"Part \ a" + $"Part b {variable}"`) or interpolations can't be parsed. The 
-matched string literal must be JSON style escaped.
+To force excluding any line, simply add a comment `wan24I8NTool:exclude` to 
+that line - example:
+
+```cs
+string text = gettext("Text file keyword");// wan24I8NTool:exclude
+```
+
+This will avoid including the text file ID `Text file keyword` (see below) 
+as an additional term to translate into the catalog.
 
 ### Custom parser configuration
 
@@ -137,6 +147,10 @@ Example parser JSON configuration:
 		".ext",
 		...
 	],
+	"TextFileExtensions": [// (optional) Text file extensions to include when walking through a folder tree (may be overridden by --textFileExt)
+		".ext.txt",
+		...
+	],
 	"MergeOutput": true,// (optional) Merge the extracted keywords to the existing output PO file
 	"FailOnError": true,// (optional) To fail thewhole process on any error
 	"Merge": false// (optional) Set to true to merge your custom configuration with the default configuration
@@ -165,19 +179,37 @@ configuration using a `AppConfig` structure
 - `CLI`: [`wan24-CLI`](https://github.com/nd1012/wan24-CLI) configuration 
 using a `CliAppConfig` structure
 
+### About including text files
+
+The extractor API does also incude text files - per default filtered by:
+
+- `*.translate.txt` (for simple text)
+- `*.translate.md` (for MarkDown text)
+
+When doing so, you can address the translation using the filename without 
+extensions as keyword - example for `Info text.translate.txt`:
+
+```cs
+string text = _("Info text");// wan24I8NTool:exclude
+```
+
+The `wan24I8NTool:exclude` does tell the extractor to skip adding `Info text` 
+as keyword to translate, because what you want to translate, is the text file 
+content, but not the keyword used to load the translation in that case.
+
 ### Build, extract, display and use an i8n file
 
 i8n files contain optional compressed translation terms. They can be created 
 from PO/MO and/or JSON dictionary (keyword as key, translation as an array of 
-strings as value) input files like this:
+strings as value) or wan24-I8NKws input files like this:
 
 ```bash
 wan24I8NTool i8n -compress --poInput /path/to/input.po --output /path/to/output.i8n
 ```
 
 An i8n file can be embedded into an app, for example.
 
-To convert all `*.json|po|mo` files in the current folder to `*.i8n` files:
+To convert all `*.json|po|mo|kws` files in the current folder to `*.i8n` files:
 
 ```bash
 wan24I8NTool i8n buildmany -compress -verbose
@@ -201,10 +233,10 @@ To extract some i8n file to a PO file:
 wan24I8NTool i8n extract --input /path/to/input.i8n --poOutput /path/to/output.po
 ```
 
-To extract some i8n file to a wan24-I8NLws file:
+To extract some i8n file to a wan24-I8NKws file:
 
 ```bash
-wan24I8NTool i8n extract --input /path/to/input.i8n --kwsOutput /path/to/output.po
+wan24I8NTool i8n extract --input /path/to/input.i8n --kwsOutput /path/to/output.kws
 ```
 
 **NOTE**: For more options and usage instructions please use the CLI API help 

diff --git a/config/c.json b/config/c.json
@@ -15,7 +15,7 @@
 			"ReplaceOnly": true
 		},
 		{
-			"Pattern": "^.*(_+|gettextn?)\\s*\\(\\s*(%{rxStringLiteral}).*$",
+			"Pattern": "^.*(_+|gettext[nd]?)\\s*\\(\\s*(%{rxStringLiteral}).*$",
 			"Options": "NonBacktracking",
 			"Replacement": "$4"
 		},

diff --git a/config/cpp.json b/config/cpp.json
@@ -15,7 +15,7 @@
 			"ReplaceOnly": true
 		},
 		{
-			"Pattern": "^.*(_+|gettextn?)\\s*\\(\\s*(%{rxStringLiteral}).*$",
+			"Pattern": "^.*(_+|gettext[nd]?)\\s*\\(\\s*(%{rxStringLiteral}).*$",
 			"Options": "NonBacktracking",
 			"Replacement": "$4"
 		},

diff --git a/config/dotnet.json b/config/dotnet.json
@@ -15,7 +15,7 @@
 			"ReplaceOnly": true
 		},
 		{
-			"Pattern": "^.*(_+|gettextn?|Translate(Plural)?|GetTerm|Std(In|OutErr)|Description|DisplayText)\\s*\\(\\s*(%{rxStringLiteral}).*$",
+			"Pattern": "^.*(_+|gettext[nd]?|Translate(Plural)?|GetTerm|Std(In|OutErr)|Description|DisplayText)\\s*\\(\\s*(%{rxStringLiteral}).*$",
 			"Options": "NonBacktracking",
 			"Replacement": "$4"
 		},

diff --git a/config/php.json b/config/php.json
@@ -15,7 +15,7 @@
 			"ReplaceOnly": true
 		},
 		{
-			"Pattern": "^.*(_+|gettextn?)\\s*\\(\\s*(%{rxStringLiteral}).*$",
+			"Pattern": "^.*(_+|gettext[nd]?)\\s*\\(\\s*(%{rxStringLiteral}).*$",
 			"Options": "NonBacktracking",
 			"Replacement": "$4"
 		},

diff --git a/src/wan24-I8N-Compressed/README.md b/src/wan24-I8N-Compressed/README.md
@@ -67,7 +67,7 @@ expressions, until the final quoted keyword was extracted from a line.
 These default translation method names can be parsed from source code:
 
 - `_`
-- `gettext(n)("..."`
+- `gettext(n|d)("..."`
 - `Translate(Plural)("..."`
 - `GetTerm("..."`
 
@@ -89,16 +89,26 @@ argument after a numeric argument:
 
 - `ExitCode(N, "..."`
 
+**NOTE**: (Multiline) concatenated string value definitions (like 
+`@"Part \ a" + $"Part b {variable}"`) or interpolations can't be parsed. The 
+matched string literal must be JSON style escaped.
+
 To force including any string (from a constant definition, for example), 
 simply add a comment `wan24I8NTool:include` at the end of the line - example:
 
 ```cs
 public const string NAME = "Any PO included keyword";// wan24I8NTool:include
 ```
 
-**NOTE**: (Multiline) concatenated string value definitions (like 
-`@"Part \ a" + $"Part b {variable}"`) or interpolations can't be parsed. The 
-matched string literal must be JSON style escaped.
+To force excluding any line, simply add a comment `wan24I8NTool:exclude` to 
+that line - example:
+
+```cs
+string text = gettext("Text file keyword");// wan24I8NTool:exclude
+```
+
+This will avoid including the text file ID `Text file keyword` (see below) 
+as an additional term to translate into the catalog.
 
 ### Custom parser configuration
 
@@ -137,6 +147,10 @@ Example parser JSON configuration:
 		".ext",
 		...
 	],
+	"TextFileExtensions": [// (optional) Text file extensions to include when walking through a folder tree (may be overridden by --textFileExt)
+		".ext.txt",
+		...
+	],
 	"MergeOutput": true,// (optional) Merge the extracted keywords to the existing output PO file
 	"FailOnError": true,// (optional) To fail thewhole process on any error
 	"Merge": false// (optional) Set to true to merge your custom configuration with the default configuration
@@ -165,19 +179,37 @@ configuration using a `AppConfig` structure
 - `CLI`: [`wan24-CLI`](https://github.com/nd1012/wan24-CLI) configuration 
 using a `CliAppConfig` structure
 
+### About including text files
+
+The extractor API does also incude text files - per default filtered by:
+
+- `*.translate.txt` (for simple text)
+- `*.translate.md` (for MarkDown text)
+
+When doing so, you can address the translation using the filename without 
+extensions as keyword - example for `Info text.translate.txt`:
+
+```cs
+string text = _("Info text");// wan24I8NTool:exclude
+```
+
+The `wan24I8NTool:exclude` does tell the extractor to skip adding `Info text` 
+as keyword to translate, because what you want to translate, is the text file 
+content, but not the keyword used to load the translation in that case.
+
 ### Build, extract, display and use an i8n file
 
 i8n files contain optional compressed translation terms. They can be created 
 from PO/MO and/or JSON dictionary (keyword as key, translation as an array of 
-strings as value) input files like this:
+strings as value) or wan24-I8NKws input files like this:
 
 ```bash
 wan24I8NTool i8n -compress --poInput /path/to/input.po --output /path/to/output.i8n
 ```
 
 An i8n file can be embedded into an app, for example.
 
-To convert all `*.json|po|mo` files in the current folder to `*.i8n` files:
+To convert all `*.json|po|mo|kws` files in the current folder to `*.i8n` files:
 
 ```bash
 wan24I8NTool i8n buildmany -compress -verbose
@@ -201,10 +233,10 @@ To extract some i8n file to a PO file:
 wan24I8NTool i8n extract --input /path/to/input.i8n --poOutput /path/to/output.po
 ```
 
-To extract some i8n file to a wan24-I8NLws file:
+To extract some i8n file to a wan24-I8NKws file:
 
 ```bash
-wan24I8NTool i8n extract --input /path/to/input.i8n --kwsOutput /path/to/output.po
+wan24I8NTool i8n extract --input /path/to/input.i8n --kwsOutput /path/to/output.kws
 ```
 
 **NOTE**: For more options and usage instructions please use the CLI API help 

diff --git a/src/wan24-I8N-Compressed/wan24-I8N-Compressed.csproj b/src/wan24-I8N-Compressed/wan24-I8N-Compressed.csproj
@@ -36,7 +36,7 @@
 
   <ItemGroup>
     <PackageReference Include="wan24-Compression" Version="2.5.0" Condition="'$(Configuration)' != 'Trunk'" />
-    <PackageReference Include="wan24-Core" Version="2.12.0" Condition="'$(Configuration)' != 'Trunk'" />
+    <PackageReference Include="wan24-Core" Version="2.13.0" Condition="'$(Configuration)' != 'Trunk'" />
   </ItemGroup>
 
   <ItemGroup>
@@ -50,6 +50,7 @@
     <None Update="LICENSE">
       <Pack>True</Pack>
       <PackagePath>\</PackagePath>
+      <CopyToOutputDirectory>Never</CopyToOutputDirectory>
     </None>
     <None Update="README.md">
       <Pack>True</Pack>

diff --git a/src/wan24-I8N/wan24-I8N.csproj b/src/wan24-I8N/wan24-I8N.csproj
@@ -35,7 +35,7 @@
   </Target>
 
   <ItemGroup>
-    <PackageReference Include="wan24-Core" Version="2.12.0" Condition="'$(Configuration)' != 'Trunk'" />
+    <PackageReference Include="wan24-Core" Version="2.13.0" Condition="'$(Configuration)' != 'Trunk'" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/wan24-I8NKws/KwsCatalog.cs b/src/wan24-I8NKws/KwsCatalog.cs
@@ -28,6 +28,19 @@ public KwsCatalog(in Dictionary<string, string[]> dict) : this()
                               });
         }
 
+        /// <summary>
+        /// Constructor
+        /// </summary>
+        /// <param name="dict">Dictionary</param>
+        public KwsCatalog(in Dictionary<string, string> dict) : this()
+        {
+            Keywords.AddRange(from kvp in dict
+                              select new KwsKeyword(kvp.Key)
+                              {
+                                  Translations = [kvp.Value]
+                              });
+        }
+
         /// <summary>
         /// Get a keyword
         /// </summary>
@@ -66,6 +79,17 @@ public KwsCatalog(in Dictionary<string, string[]> dict) : this()
         /// </summary>
         public bool RightToLeft { get; set; }
 
+        /// <summary>
+        /// Properties
+        /// </summary>
+        public Dictionary<string, string> Properties { get; init; } = [];
+
+        /// <summary>
+        /// Keywords
+        /// </summary>
+        [NoValidation]
+        public HashSet<KwsKeyword> Keywords { get; init; } = [];
+
         /// <summary>
         /// Complete keywords (non-obsolete)
         /// </summary>
@@ -115,10 +139,10 @@ public KwsCatalog(in Dictionary<string, string[]> dict) : this()
         public IEnumerable<KwsKeyword> NonObsoleteKeywords => Keywords.Where(k => !k.Obsolete);
 
         /// <summary>
-        /// Keywords
+        /// Unvalidated keywords
         /// </summary>
         [NoValidation]
-        public HashSet<KwsKeyword> Keywords { get; init; } = [];
+        public IEnumerable<KwsKeyword> UnvalidatedKeywords => Keywords.Where(k => !k.Validate(throwOnError: false));
 
         /// <summary>
         /// Try adding a new keyword
@@ -192,6 +216,36 @@ public bool Validate(in bool throwOnError = true, in bool requireCompleteTransla
             return true;
         }
 
+        /// <summary>
+        /// Merge with another catalog (existing keywords will be revisioned for merging)
+        /// </summary>
+        /// <param name="other">Other catalog to merge into this catalog</param>
+        /// <param name="ignoreLocale">Ignore merging with another locale?</param>
+        /// <returns>Merged existing keywords</returns>
+        public KwsKeyword[] Merge(KwsCatalog other, bool ignoreLocale = false)
+        {
+            if (!ignoreLocale && other.Locale != Locale) throw new ArgumentException("Locale mismatch", nameof(other));
+            List<KwsKeyword> res = [];
+            Modified = DateTime.UtcNow;
+            // Merge the catalog meta data
+            if (string.IsNullOrWhiteSpace(Project) && !string.IsNullOrWhiteSpace(other.Project)) Project = other.Project;
+            if (string.IsNullOrWhiteSpace(Translator) && !string.IsNullOrWhiteSpace(other.Translator)) Translator = other.Translator;
+            RightToLeft = other.RightToLeft;
+            foreach (var kvp in other.Properties) Properties[kvp.Key] = kvp.Value;
+            // Merge keywords
+            foreach(KwsKeyword otherKeyword in other.NonObsoleteKeywords)
+            {
+                if (this[otherKeyword.ID] is not KwsKeyword existing)
+                {
+                    Keywords.Add(otherKeyword);
+                    continue;
+                }
+                existing.Merge(otherKeyword);
+                res.Add(existing);
+            }
+            return [.. res];
+        }
+
         /// <summary>
         /// Convert to a dictionary (only non-obsolete keywords)
         /// </summary>