Merge remote-tracking branch 'origin/master' into chore/fv/12-11

HISTORY.md

@@ -1,5 +1,33 @@
 # Keyman Version History
 
+## 18.0.69 alpha 2024-07-05
+
+* fix(core): allow to successfully build on Ubuntu 24.04 (#11926)
+* chore(windows): correct output file for 64-bit build of keyman32 in build.sh (#11930)
+* chore(android,ios): Add Crowdin localization for Polytonic Greek (#11877)
+
+## 18.0.68 alpha 2024-07-04
+
+* refactor(windows): merge keyman64 build into keyman32 (#11906)
+* refactor(windows): remove wm_keyman_keydown and wm_keyman_keyup (#11920)
+
+## 18.0.67 alpha 2024-07-03
+
+* refactor(common/models): move TS priority-queue implementation to web-utils (#11867)
+
+## 18.0.66 alpha 2024-07-02
+
+* fix(developer): handle second parameter of index correctly in kmcmplib compiler (#11815)
+
+## 18.0.65 alpha 2024-07-01
+
+* fix(developer): prevent non-BMP characters in key part of rule (#11806)
+* chore(linux): remove unused building with pbuilder (#11862)
+
+## 18.0.64 alpha 2024-06-28
+
+* fix(web): use fat-finger data with simple keypresses (#11854)
+
 ## 18.0.63 alpha 2024-06-26
 
 * feat(linux): implement Linux side of SimulateAltGr option :checkered_flag: (#11852)

VERSION.md

@@ -1 +1 @@
-18.0.64
+18.0.70

android/KMEA/app/src/main/java/com/keyman/engine/DisplayLanguages.java

@@ -62,6 +62,7 @@ public static final DisplayLanguageType[] getDisplayLanguages(Context context) {
       new DisplayLanguageType("nl-NL", "Nederlands (Dutch)"),
       new DisplayLanguageType("ann", "Obolo"),
       new DisplayLanguageType("pl-PL", "Polski (Polish)"),
+      new DisplayLanguageType("el", "Polytonic Greek"),
       new DisplayLanguageType("pt-PT", "Português do Portugal"),
       new DisplayLanguageType("ff-ZA", "Pulaar-Fulfulde"), // or Fulah
       new DisplayLanguageType("ru-RU", "Pyccĸий (Russian)"),

common/models/templates/src/index.ts

@@ -2,7 +2,6 @@ export {
   SENTINEL_CODE_UNIT, applyTransform, buildMergedTransform, isHighSurrogate, isLowSurrogate, isSentinel,
   transformToSuggestion, defaultApplyCasing
 } from "./common.js";
-export { default as PriorityQueue, Comparator } from "./priority-queue.js";
 export { default as QuoteBehavior } from "./quote-behavior.js";
 export { Tokenization, tokenize, getLastPreCaretToken, wordbreak } from "./tokenization.js";
 export { default as TrieModel, TrieModelOptions } from "./trie-model.js";

common/models/templates/src/trie-model.ts

@@ -26,12 +26,11 @@
 // Should probably make a 'lm-utils' submodule.
 
 // Allows the kmwstring bindings to resolve.
-import { extendString } from "@keymanapp/web-utils";
+import { extendString, PriorityQueue } from "@keymanapp/web-utils";
 import { default as defaultWordBreaker } from "@keymanapp/models-wordbreakers";
 
 import { applyTransform, isHighSurrogate, isSentinel, SENTINEL_CODE_UNIT, transformToSuggestion } from "./common.js";
 import { getLastPreCaretToken } from "./tokenization.js";
-import PriorityQueue from "./priority-queue.js";
 
 extendString();
 

common/web/lm-worker/src/main/correction/distance-modeler.ts

@@ -1,4 +1,5 @@
-import { Comparator, isHighSurrogate, SENTINEL_CODE_UNIT, PriorityQueue } from '@keymanapp/models-templates';
+import { isHighSurrogate, SENTINEL_CODE_UNIT } from '@keymanapp/models-templates';
+import { QueueComparator as Comparator, PriorityQueue } from '@keymanapp/web-utils';
 
 import { ClassicalDistanceCalculation, EditToken } from './classical-calculation.js';
 import { ExecutionTimer, STANDARD_TIME_BETWEEN_DEFERS } from './execution-timer.js';

common/web/lm-worker/src/test/mocha/cases/edit-distance/distance-modeler.js

@@ -2,6 +2,7 @@ import { assert } from 'chai';
 import * as models from '#./models/index.js';
 import * as correction from '#./correction/index.js';
 
+import { PriorityQueue } from '@keymanapp/web-utils';
 import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
 
 function buildTestTimer() {
@@ -142,7 +143,7 @@ describe('Correction Distance Modeler', function() {
       assert.equal(edges.length, expectedChildCount);
 
       // One final bit, which is a bit of integration - we know the top two nodes that should result.
-      let queue = new models.PriorityQueue(correction.QUEUE_NODE_COMPARATOR, edges);
+      let queue = new PriorityQueue(correction.QUEUE_NODE_COMPARATOR, edges);
 
       let firstEdge = queue.dequeue();
       assert.equal(firstEdge.priorInput[0].sample.insert, 't');
@@ -185,13 +186,13 @@ describe('Correction Distance Modeler', function() {
       ];
 
       let layer1Edges = rootNode.buildSubstitutionEdges(synthDistribution1);
-      let layer1Queue = new models.PriorityQueue(correction.QUEUE_NODE_COMPARATOR, layer1Edges);
+      let layer1Queue = new PriorityQueue(correction.QUEUE_NODE_COMPARATOR, layer1Edges);
 
       let tEdge = layer1Queue.dequeue();
       assertEdgeChars(tEdge, 't', 't');
 
       let layer2Edges = tEdge.buildSubstitutionEdges(synthDistribution2);
-      let layer2Queue = new models.PriorityQueue(correction.QUEUE_NODE_COMPARATOR, layer2Edges);
+      let layer2Queue = new PriorityQueue(correction.QUEUE_NODE_COMPARATOR, layer2Edges);
 
       let eEdge = layer2Queue.dequeue();
       assertEdgeChars(eEdge, 'e', 'e');
@@ -208,7 +209,7 @@ describe('Correction Distance Modeler', function() {
       let layer3eEdges  = eEdge.buildSubstitutionEdges(synthDistribution3);
       let layer3hEdges  = hEdge.buildSubstitutionEdges(synthDistribution3);
       let layer3ehEdges = ehEdge.buildSubstitutionEdges(synthDistribution3);
-      let layer3Queue = new models.PriorityQueue(correction.QUEUE_NODE_COMPARATOR, layer3eEdges.concat(layer3hEdges).concat(layer3ehEdges));
+      let layer3Queue = new PriorityQueue(correction.QUEUE_NODE_COMPARATOR, layer3eEdges.concat(layer3hEdges).concat(layer3ehEdges));
 
       // Find the first result with an actual word directly represented.
       let bestEdge;

common/web/utils/src/index.ts

@@ -23,6 +23,8 @@ export { default as TimeoutPromise, timedPromise } from "./timeoutPromise.js";
 
 export { Uni_IsSurrogate1, Uni_IsSurrogate2 } from "./surrogates.js";
 
+export { default as PriorityQueue, QueueComparator } from "./priority-queue.js"
+
 // // Uncomment the following line and run the bundled output to verify successful
 // // esbuild bundling of this submodule:
 // console.log(Version.CURRENT.toString());

common/models/templates/src/priority-queue.ts → common/web/utils/src/priority-queue.ts

@@ -11,11 +11,11 @@
  * - value > 0 if `b` should come before `a`
  * - 0 if they should be treated equally.
  */
-export type Comparator<Type> = (a: Type, b: Type) => number;
+export type QueueComparator<Type> = (a: Type, b: Type) => number;
 
 export default class PriorityQueue<Type> {
 
-  private comparator: Comparator<Type>;
+  private comparator: QueueComparator<Type>;
   private heap: Type[];
 
   /**
@@ -29,8 +29,8 @@ export default class PriorityQueue<Type> {
    * the first parameter should precede the second parameter.
    * @param initialEntries
    */
-  constructor(comparator: Comparator<Type>, initialEntries?: Type[]);
-  constructor(arg1: Comparator<Type> | PriorityQueue<Type>, initialEntries?: Type[]) {
+  constructor(comparator: QueueComparator<Type>, initialEntries?: Type[]);
+  constructor(arg1: QueueComparator<Type> | PriorityQueue<Type>, initialEntries?: Type[]) {
     if(typeof arg1 != 'function') {
       this.comparator = arg1.comparator;
       // Shallow-copies are fine.

common/models/templates/test/test-priority-queue.js → common/web/utils/src/test/priorityQueue.js

@@ -3,7 +3,7 @@
  */
 
 import { assert } from 'chai';
-import { PriorityQueue } from '@keymanapp/models-templates';
+import { PriorityQueue } from '@keymanapp/web-utils';
 
 describe('Priority queue', function() {
   it('can act as a min-heap', function () {

common/windows/build.sh

@@ -18,7 +18,7 @@ builder_describe "Keyman common Windows modules" \
 
 builder_parse "$@"
 
-if [[ $BUILDER_OS != windows ]]; then
+if [[ $BUILDER_OS != win ]]; then
   builder_echo grey "Platform is not Windows; skipping common/windows"
   exit 0
 fi

core/tests/unit/ldml/test_kmx_plus.cpp

@@ -1,3 +1,4 @@
+#include <cstdint>
 #include <test_assert.h>
 #include "kmx/kmx_plus.h"
 #include "kmx/kmx_xstring.h"

crowdin.yml

@@ -34,6 +34,7 @@ files:
     languages_mapping:
       # Prevent invalid region pap-rPAP. Leaving "in" for Indonesian
       android_code:
+        el-polyton: b+el # TODO: figure out polyton variant
         es-419: b+es+419
         pap: pap
         shu-latn-n: b+shu+latn
@@ -44,6 +45,7 @@ files:
     languages_mapping:
       # Prevent invalid region pap-rPAP
       android_code:
+        el-polyton: b+el # TODO: figure out polyton variant
         es-419: b+es+419
         pap: pap
         shu-latn-n: b+shu+latn
@@ -79,27 +81,31 @@ files:
     languages_mapping:
       osx_code:
         pt-PT: pt-PT.lproj
+        el-polyton: el.lproj
 
   - source: /ios/engine/KMEI/KeymanEngine/en.lproj/Localizable.strings
     dest: /ios/engine/Localizable.strings
     translation: /ios/engine/KMEI/KeymanEngine/%osx_code%/%original_file_name%
     languages_mapping:
       osx_code:
         pt-PT: pt-PT.lproj
+        el-polyton: el.lproj
 
   - source: /ios/engine/KMEI/KeymanEngine/en.lproj/Localizable.stringsdict
     dest: /ios/engine/Localizable.stringsdict
     translation: /ios/engine/KMEI/KeymanEngine/%osx_code%/%original_file_name%
     languages_mapping:
       osx_code:
         pt-PT: pt-PT.lproj
+        el-polyton: el.lproj
 
   - source: /ios/keyman/Keyman/Keyman/en.lproj/Localizable.strings
     dest: /ios/app/Localizable.strings
     translation: /ios/keyman/Keyman/Keyman/%osx_code%/%original_file_name%
     languages_mapping:
       osx_code:
         pt-PT: pt-PT.lproj
+        el-polyton: el.lproj
 
   # Linux files
 
@@ -136,7 +142,7 @@ files:
       osx_code:
         pt-PT: pt-PT.lproj
 
-  - source: /mac/Keyman4MacIM/Keyman4MacIM/KMKeyboardHelpWindow/en.lproj/KMKeyboardHelpWindowController.strings  
+  - source: /mac/Keyman4MacIM/Keyman4MacIM/KMKeyboardHelpWindow/en.lproj/KMKeyboardHelpWindowController.strings
     dest: /mac/app/KMKeyboardHelpWindowController.strings
     translation: /mac/Keyman4MacIM/Keyman4MacIM/KMKeyboardHelpWindow/%osx_code%/%original_file_name%
     languages_mapping:

developer/src/common/include/kmn_compiler_errors.h

@@ -179,6 +179,7 @@
 #define CERR_ExtendedStringTooLong                         0x00004076
 #define CERR_VirtualKeyExpansionTooLong                    0x00004077
 #define CERR_CharacterRangeTooLong                         0x00004078
+#define CERR_NonBMPCharactersNotSupportedInKeySection      0x00004079
 
 #define CWARN_TooManyWarnings                              0x00002080
 #define CWARN_OldVersion                                   0x00002081

developer/src/kmc-kmn/src/compiler/kmn-compiler-messages.ts

@@ -571,6 +571,8 @@ export class KmnCompilerMessages {
   static ERROR_CharacterRangeTooLong                          = SevError | 0x078;
   static Error_CharacterRangeTooLong                          = () => m(this.ERROR_CharacterRangeTooLong, `Character range is too large and cannot be expanded`);
 
+  static ERROR_NonBMPCharactersNotSupportedInKeySection       = SevError | 0x079;
+  static Error_NonBMPCharactersNotSupportedInKeySection       = () => m(this.ERROR_NonBMPCharactersNotSupportedInKeySection, `Characters with codepoints over U+FFFF are not supported in the key part of the rule`);
 
   static WARN_TooManyWarnings                                 = SevWarn | 0x080;
   static Warn_TooManyWarnings                                 = () => m(this.WARN_TooManyWarnings, `Too many warnings or errors`);

developer/src/kmcmplib/src/CompMsg.cpp

@@ -111,6 +111,7 @@ std::map<KMX_DWORD, const KMX_CHAR*> CompilerErrorMap = {
     { CERR_ExtendedStringTooLong                         , "Extended string is too long" },
     { CERR_VirtualKeyExpansionTooLong                    , "Virtual key expansion is too large" },
     { CERR_CharacterRangeTooLong                         , "Character range is too large and cannot be expanded" },
+    { CERR_NonBMPCharactersNotSupportedInKeySection      , "Characters with codepoints over U+FFFF are not supported in the key part of the rule" },
 
     { CHINT_UnreachableRule                              , "This rule will never be matched as another rule takes precedence"},
     { CHINT_NonUnicodeFile                               , "Keyman Developer has detected that the file has ANSI encoding. Consider converting this file to UTF-8"},

developer/src/kmcmplib/src/Compiler.cpp

@@ -97,6 +97,8 @@
 #include <codecvt>
 #include <locale>
 #include <string>
+#include <iostream>
+#include <sstream>
 
 #include "UnreachableRules.h"
 #include "CheckForDuplicates.h"
@@ -134,6 +136,7 @@ namespace kmcmp{
 
 int xatoi(PKMX_WCHAR *p);
 int atoiW(PKMX_WCHAR p);
+bool isIntegerWstring(PKMX_WCHAR p);
 void safe_wcsncpy(PKMX_WCHAR out, PKMX_WCHAR in, int cbMax);
 int GetDeadKey(PFILE_KEYBOARD fk, PKMX_WCHAR p);
 
@@ -1211,7 +1214,7 @@ int GetCompileTargetsFromTargetsStore(const KMX_WCHAR* store) {
 KMX_BOOL IsValidKeyboardVersion(KMX_WCHAR *dpString) {   // I4140
   /**
     version format: /^\d+(\.\d+)*$/
-    e.g. 9.0.3, 1.0, 1.2.3.4, 6.2.1.4.6.4, 11.22.3 are all ok; 
+    e.g. 9.0.3, 1.0, 1.2.3.4, 6.2.1.4.6.4, 11.22.3 are all ok;
     empty string is not permitted; whitespace is not permitted
   */
 
@@ -1472,7 +1475,14 @@ KMX_DWORD ProcessKeyLineImpl(PFILE_KEYBOARD fk, PKMX_WCHAR str, KMX_BOOL IsUnico
 
     str = p + 1;
     if ((msg = GetXString(fk, str, u">", pklKey, GLOBAL_BUFSIZE - 1, (int)(str - pp), &p, TRUE, IsUnicode)) != CERR_None) return msg;
+
     if (pklKey[0] == 0) return CERR_ZeroLengthString;
+
+    if(Uni_IsSurrogate1(pklKey[0])) {
+      // #11643: non-BMP characters do not makes sense for key codes
+      return CERR_NonBMPCharactersNotSupportedInKeySection;
+    }
+
     if (xstrlen(pklKey) > 1) AddWarning(CWARN_KeyBadLength);
   } else {
     if ((msg = GetXString(fk, str, u">", pklIn, GLOBAL_BUFSIZE - 1, (int)(str - pp), &p, TRUE, IsUnicode)) != CERR_None) return msg;
@@ -1654,9 +1664,10 @@ KMX_DWORD ExpandKp(PFILE_KEYBOARD fk, PFILE_KEY kpp, KMX_DWORD storeIndex)
       default:
         return CERR_CodeInvalidInKeyStore;
       }
-    }
-    else
-    {
+    } else if(Uni_IsSurrogate1(*pn)) {
+      // #11643: non-BMP characters do not makes sense for key codes
+      return CERR_NonBMPCharactersNotSupportedInKeySection;
+    } else {
       k->Key = *pn;				// set the key to store offset.
       k->ShiftFlags = 0;
     }
@@ -2014,7 +2025,7 @@ KMX_DWORD GetXStringImpl(PKMX_WCHAR tstr, PFILE_KEYBOARD fk, PKMX_WCHAR str, KMX
           kmcmp::CheckStoreUsage(fk, i, TRUE, FALSE, FALSE);
 
           r = u16tok(NULL, p_sep_com, &context);  // I3481
-          if (!r) return CERR_InvalidIndex;
+          if (!r || !*r || !isIntegerWstring(r) || atoiW(r) < 1) return CERR_InvalidIndex;
         }
         tstr[mx++] = UC_SENTINEL;
         tstr[mx++] = CODE_INDEX;
@@ -3362,6 +3373,26 @@ int atoiW(PKMX_WCHAR p)
   return i;
 }
 
+/**
+ * Checks if a wide-character C-string represents an integer.
+ * It does not strip whitespace, and depends on the action of atoi()
+ * to determine if the C-string is an integer.
+ *
+ * @param p a pointer to a wide-character C-string
+ *
+ * @return true if p represents an integer, false otherwise
+*/
+bool isIntegerWstring(PKMX_WCHAR p) {
+  if (!p || !*p)
+    return false;
+  PKMX_STR q = wstrtostr(p);
+  std::ostringstream os;
+  os << atoi(q);
+  int cmp = strcmp(q, os.str().c_str());
+  delete[] q;
+  return cmp == 0 ? true : false;
+}
+
 KMX_DWORD kmcmp::CheckUTF16(int n)
 {
   const int res[] = {