test(developer): check correct use of u16chr when second parameter could be null

developer/src/kmcmplib/tests/gtest-compiler-test.cpp

@@ -15,9 +15,11 @@ KMX_DWORD ProcessBeginLine(PFILE_KEYBOARD fk, PKMX_WCHAR p);
 KMX_DWORD ValidateMatchNomatchOutput(PKMX_WCHAR p);
 KMX_BOOL IsValidKeyboardVersion(KMX_WCHAR *dpString);
 PKMX_WCHAR GetDelimitedString(PKMX_WCHAR *p, KMX_WCHAR const * Delimiters, KMX_WORD Flags);
+int LineTokenType(PKMX_WCHAR *str);
 KMX_DWORD GetXStringImpl(PKMX_WCHAR tstr, PFILE_KEYBOARD fk, PKMX_WCHAR str, KMX_WCHAR const * token,
   PKMX_WCHAR output, int max, int offset, PKMX_WCHAR *newp, int isUnicode
 );
+KMX_DWORD ProcessEthnologueStore(PKMX_WCHAR p);
 KMX_DWORD GetRHS(PFILE_KEYBOARD fk, PKMX_WCHAR p, PKMX_WCHAR buf, int bufsize, int offset, int IsUnicode);
 bool isIntegerWstring(PKMX_WCHAR p);
 bool hasPreamble(std::u16string result);
@@ -29,6 +31,7 @@ namespace kmcmp {
     extern int nErrors;
     extern int ErrChr;
     extern int BeginLine[4];
+    extern int CompileTarget;
     KMX_BOOL AddCompileWarning(char* buf);
 }
 
@@ -59,6 +62,7 @@ class CompilerTest : public testing::Test {
             kmcmp::BeginLine[BEGIN_UNICODE] = -1;
             kmcmp::BeginLine[BEGIN_NEWCONTEXT] = -1;
             kmcmp::BeginLine[BEGIN_POSTKEYSTROKE] = -1;
+            kmcmp::CompileTarget = CKF_KEYMAN;
         }
 
         void initFileKeyboard(FILE_KEYBOARD &fk) {
@@ -556,7 +560,106 @@ TEST_F(CompilerTest, GetDelimitedString_test) {
 }
 
 // LinePrefixType GetLinePrefixType(PKMX_WCHAR *p)
-// int LineTokenType(PKMX_WCHAR *str)
+
+TEST_F(CompilerTest, LineTokenType_test) {
+    KMX_WCHAR str[LINESIZE];
+    PKMX_WCHAR p = nullptr;
+
+    // T_BLANK, lptOther, empty string
+    u16cpy(str, u"");
+    p = str;
+    EXPECT_EQ(T_BLANK, LineTokenType(&p));
+
+    // T_BLANK, lptOther, one space
+    u16cpy(str, u" ");
+    p = str;
+    EXPECT_EQ(T_BLANK, LineTokenType(&p));
+
+    // T_BLANK, mismatched prefix, CKF_KEYMAN, lptKeymanWebOnly
+    u16cpy(str, u"$keymanweb:");
+    p = str;
+    kmcmp::CompileTarget = CKF_KEYMAN;
+    EXPECT_EQ(T_BLANK, LineTokenType(&p));
+
+    // T_BLANK, mismatched prefix, CKF_KEYMANWEB, lptKeymanOnly
+    u16cpy(str, u"$keymanonly:");
+    p = str;
+    kmcmp::CompileTarget = CKF_KEYMANWEB;
+    EXPECT_EQ(T_BLANK, LineTokenType(&p));
+
+    // T_BLANK, nothing after prefix
+    u16cpy(str, u"$keyman:");
+    p = str;
+    kmcmp::CompileTarget = CKF_KEYMAN;
+    EXPECT_EQ(T_BLANK, LineTokenType(&p));
+
+    // T_STORE (=T_W_START)
+    u16cpy(str, u"store(b)");
+    p = str;
+    EXPECT_EQ(T_STORE, LineTokenType(&p));
+    EXPECT_EQ(u16len(u"store"), p - str);
+    EXPECT_TRUE(!u16cmp(p, u"(b)"));
+
+    // T_BITMAPS (=T_W_END)
+    u16cpy(str, u"bitmaps \"b\"");
+    p = str;
+    EXPECT_EQ(T_BITMAPS, LineTokenType(&p));
+    EXPECT_EQ(u16len(u"bitmaps "), p - str);
+    EXPECT_TRUE(!u16cmp(p, u"\"b\""));
+
+    // T_STORE, preceeded by one space
+    u16cpy(str, u" store(b)");
+    p = str;
+    EXPECT_EQ(T_STORE, LineTokenType(&p));
+    EXPECT_EQ(u16len(u" store"), p - str);
+    EXPECT_TRUE(!u16cmp(p, u"(b)"));
+
+    // T_STORE, preceeded by two spaces
+    u16cpy(str, u"  store(b)");
+    p = str;
+    EXPECT_EQ(T_STORE, LineTokenType(&p));
+    EXPECT_EQ(u16len(u"  store"), p - str);
+    EXPECT_TRUE(!u16cmp(p, u"(b)"));
+
+    // T_STORE, followed by one space
+    u16cpy(str, u"store (b)");
+    p = str;
+    EXPECT_EQ(T_STORE, LineTokenType(&p));
+    EXPECT_EQ(u16len(u"store "), p - str);
+    EXPECT_TRUE(!u16cmp(p, u"(b)"));
+
+    // T_STORE, followed by two spaces
+    u16cpy(str, u"store  (b)");
+    p = str;
+    EXPECT_EQ(T_STORE, LineTokenType(&p));
+    EXPECT_EQ(u16len(u"store  "), p - str);
+    EXPECT_TRUE(!u16cmp(p, u"(b)"));
+
+    // T_COMMENT
+    u16cpy(str, u"c ");
+    p = str;
+    EXPECT_EQ(T_COMMENT, LineTokenType(&p));
+    EXPECT_EQ(0, p - str);
+
+    // comment without following space ... potential bug, but ReadLine() currently ensures following space
+    u16cpy(str, u"c");
+    p = str;
+    EXPECT_EQ(T_UNKNOWN, LineTokenType(&p));
+    EXPECT_EQ(0, p - str);
+
+    // T_KEYTOKEY
+    u16cpy(str, u"abc");
+    p = str;
+    EXPECT_EQ(T_KEYTOKEY, LineTokenType(&p));
+    EXPECT_EQ(0, p - str);
+
+     // T_UNKNOWN
+    u16cpy(str, u"z");
+    p = str;
+    EXPECT_EQ(T_UNKNOWN, LineTokenType(&p));
+    EXPECT_EQ(0, p - str);
+}
+
 // KMX_BOOL StrValidChrs(PKMX_WCHAR q, KMX_WCHAR const * chrs)
 // KMX_DWORD GetXString(PFILE_KEYBOARD fk, PKMX_WCHAR str, KMX_WCHAR const * token,
 //  PKMX_WCHAR output, int max, int offset, PKMX_WCHAR *newp, int /*isVKey*/, int isUnicode
@@ -1157,7 +1260,24 @@ TEST_F(CompilerTest, GetXStringImpl_type_o_test) {
 // KMX_DWORD process_save(PFILE_KEYBOARD fk, PKMX_WCHAR q, PKMX_WCHAR tstr, int *mx)
 // int xatoi(PKMX_WCHAR *p)
 // int GetGroupNum(PFILE_KEYBOARD fk, PKMX_WCHAR p)
-// KMX_DWORD ProcessEthnologueStore(PKMX_WCHAR p)
+
+TEST_F(CompilerTest, ProcessEthnologueStore_test) {
+    EXPECT_EQ(CERR_None, ProcessEthnologueStore((PKMX_WCHAR)u"abc"));
+    EXPECT_EQ(CWARN_PunctuationInEthnologueCode, ProcessEthnologueStore((PKMX_WCHAR)u";abc"));
+    EXPECT_EQ(CWARN_PunctuationInEthnologueCode, ProcessEthnologueStore((PKMX_WCHAR)u",abc"));
+    EXPECT_EQ(CERR_None, ProcessEthnologueStore((PKMX_WCHAR)u" abc"));
+    EXPECT_EQ(CERR_InvalidEthnologueCode, ProcessEthnologueStore((PKMX_WCHAR)u"abc "));
+    EXPECT_EQ(CERR_InvalidEthnologueCode, ProcessEthnologueStore((PKMX_WCHAR)u"abcd"));
+    EXPECT_EQ(CERR_InvalidEthnologueCode, ProcessEthnologueStore((PKMX_WCHAR)u"ab"));
+    EXPECT_EQ(CERR_InvalidEthnologueCode, ProcessEthnologueStore((PKMX_WCHAR)u"a"));
+    EXPECT_EQ(CERR_InvalidEthnologueCode, ProcessEthnologueStore((PKMX_WCHAR)u"a2b"));
+    EXPECT_EQ(CERR_None, ProcessEthnologueStore((PKMX_WCHAR)u"")); // needs correcting ... see #11955
+    EXPECT_EQ(CERR_None, ProcessEthnologueStore((PKMX_WCHAR)u"abc def"));
+    EXPECT_EQ(CERR_None, ProcessEthnologueStore((PKMX_WCHAR)u"abc  def"));
+    EXPECT_EQ(CWARN_PunctuationInEthnologueCode, ProcessEthnologueStore((PKMX_WCHAR)u"abc,def"));
+    EXPECT_EQ(CWARN_PunctuationInEthnologueCode, ProcessEthnologueStore((PKMX_WCHAR)u"abc;def"));
+}
+
 // KMX_DWORD ProcessHotKey(PKMX_WCHAR p, KMX_DWORD *hk)
 // void SetChecksum(PKMX_BYTE buf, PKMX_DWORD CheckSum, KMX_DWORD sz)
 // KMX_BOOL kmcmp::CheckStoreUsage(PFILE_KEYBOARD fk, int storeIndex, KMX_BOOL fIsStore, KMX_BOOL fIsOption, KMX_BOOL fIsCall)

developer/src/kmcmplib/tests/gtest-kmx_u16-test.cpp

@@ -1,60 +1,168 @@
 #include <gtest/gtest.h>
 #include "../src/kmx_u16.h"
+#include "../src/compfile.h"
+#include "../../../../common/include/km_types.h"
 
-/*class kmx_u16_Test : public testing::Test {
-    protected:
-    	kmx_u16_Test() {}
-	    ~kmx_u16_Test() override {}
-	    void SetUp() override {}
-	    void TearDown() override {}
-};*/
+TEST(kmx_u16_Test, u16chr) {
+  KMX_WCHAR str[LINESIZE];
+
+  u16cpy(str, u"abc");
+	EXPECT_EQ(1, u16chr(str, 'b') - str); // in string
+	u16cpy(str, u"abc");
+  EXPECT_EQ(NULL, u16chr(str, 'd')); // not in string
+	u16cpy(str, u"abc");
+  EXPECT_EQ(3, u16chr(str, '\0') - str); // locate null terminator
+}
+
+TEST(kmx_u16_Test, u16chr_compare_to_strchr) {
+  // Compare behaviour of strchr:
+  char str[LINESIZE];
+
+  strcpy(str, "abc");
+	EXPECT_EQ(1, strchr(str, 'b') - str); // in string
+	strcpy(str, "abc");
+  EXPECT_EQ(NULL, strchr(str, 'd')); // not in string
+	strcpy(str, "abc");
+  EXPECT_EQ(3, strchr(str, '\0') - str); // locate null terminator
+}
 
 TEST(kmx_u16_Test, u16tok_char_delim) {
   // For char delimiter: KMX_WCHAR * u16tok(KMX_WCHAR *p, const KMX_WCHAR ch,  KMX_WCHAR **ctx) ;
 
+	KMX_WCHAR str[LINESIZE];
   KMX_WCHAR *ctx = nullptr;
-  EXPECT_EQ(nullptr, u16tok(nullptr, ' ', &ctx));
 
-  KMX_WCHAR buffer[128] = u"test a space  and two";
-  EXPECT_TRUE(!u16cmp(u"test", u16tok(buffer, ' ', &ctx)));
+  // sequence of tokens
+  u16cpy(str, u"test a space  and two");
+  ctx = nullptr;
+  EXPECT_TRUE(!u16cmp(u"test", u16tok(str, ' ', &ctx)));
   EXPECT_TRUE(!u16cmp(u"a", u16tok(nullptr, ' ', &ctx)));
   EXPECT_TRUE(!u16cmp(u"space", u16tok(nullptr, ' ', &ctx)));
   EXPECT_TRUE(!u16cmp(u"and", u16tok(nullptr, ' ', &ctx)));
   EXPECT_TRUE(!u16cmp(u"two", u16tok(nullptr, ' ', &ctx)));
+  EXPECT_EQ(nullptr, ctx);
 
-  KMX_WCHAR buffer_space[128] = u" ";
-  EXPECT_EQ(nullptr, u16tok(buffer_space, ' ', &ctx));
+  // only a delimiter
+  u16cpy(str, u" ");
+  ctx = nullptr;
+  EXPECT_EQ(nullptr, u16tok(str, ' ', &ctx));
   EXPECT_EQ(nullptr, u16tok(nullptr, ' ', &ctx));
+
+  // delimiters at end
+  u16cpy(str, u"a b   ");
+  ctx = nullptr;
+  EXPECT_TRUE(!u16cmp(u"a", u16tok(str, ' ', &ctx)));
+  EXPECT_TRUE(!u16cmp(u"b", u16tok(nullptr, ' ', &ctx)));
+  EXPECT_EQ(nullptr, u16tok(nullptr, ' ', &ctx));
+
+  // no string, no context
+  ctx = nullptr;
+  EXPECT_EQ(nullptr, u16tok(nullptr, ' ', &ctx));
+
+	// delimited string
+	u16cpy(str, u"abc|def");
+  ctx = nullptr;
+	EXPECT_EQ(str, u16tok(str, '|', &ctx));
+	EXPECT_TRUE(!u16cmp(u"abc", str));
+	EXPECT_TRUE(!u16cmp(u"def", ctx));
 }
 
 TEST(kmx_u16_Test, u16tok_str_delim) {
   // For string delimiter: KMX_WCHAR * u16tok(KMX_WCHAR* p, const KMX_WCHAR* ch, KMX_WCHAR** ctx) ;
 
+	KMX_WCHAR str[LINESIZE];
   KMX_WCHAR *ctx = nullptr;
-  EXPECT_EQ(nullptr, u16tok(nullptr, u" ", &ctx));
 
-  KMX_WCHAR buffer[128] = u"test a space  and two";
-  EXPECT_TRUE(!u16cmp(u"test", u16tok(buffer, u" ", &ctx)));
+  // sequence of tokens
+  u16cpy(str, u"test a space  and two");
+  ctx = nullptr;
+  EXPECT_TRUE(!u16cmp(u"test", u16tok(str, u" ", &ctx)));
   EXPECT_TRUE(!u16cmp(u"a", u16tok(nullptr, u" ", &ctx)));
   EXPECT_TRUE(!u16cmp(u"space", u16tok(nullptr, u" ", &ctx)));
   EXPECT_TRUE(!u16cmp(u"and", u16tok(nullptr, u" ", &ctx)));
   EXPECT_TRUE(!u16cmp(u"two", u16tok(nullptr, u" ", &ctx)));
+  EXPECT_EQ(nullptr, ctx);
 
-  KMX_WCHAR buffer_space[128] = u" ";
-  EXPECT_EQ(nullptr, u16tok(buffer_space, u" ", &ctx));
+  // only a delimiter
+  u16cpy(str, u" ");
+  ctx = nullptr;
+  EXPECT_EQ(nullptr, u16tok(str, u" ", &ctx));
   EXPECT_EQ(nullptr, u16tok(nullptr, u" ", &ctx));
+
+  // delimiters at end
+  u16cpy(str, u"a b   ");
+  ctx = nullptr;
+  EXPECT_TRUE(!u16cmp(u"a", u16tok(str, u" ", &ctx)));
+  EXPECT_TRUE(!u16cmp(u"b", u16tok(nullptr, u" ", &ctx)));
+  EXPECT_EQ(nullptr, u16tok(nullptr, u" ", &ctx));
+
+  // no string, no context
+  ctx = nullptr;
+	EXPECT_EQ(nullptr, u16tok(nullptr, u"", &ctx));
+
+	// delimited string
+	u16cpy(str, u"abc|def");
+  ctx = nullptr;
+	EXPECT_EQ(str, u16tok(str, u"|", &ctx));
+	EXPECT_TRUE(!u16cmp(u"abc", str));
+	EXPECT_TRUE(!u16cmp(u"def", ctx));
+
+  // multiple delimiters
+  u16cpy(str, u"abc<def>ghi");
+  ctx = nullptr;
+  const KMX_WCHAR *delim = u"<>";
+	EXPECT_TRUE(!u16cmp(u"abc", u16tok(str, delim, &ctx)));
+  EXPECT_TRUE(!u16cmp(u"def", u16tok(nullptr, delim, &ctx)));
+  EXPECT_TRUE(!u16cmp(u"ghi", u16tok(nullptr, delim, &ctx)));
+  EXPECT_EQ(nullptr, ctx);
+
+  // check sensitivity of ordering with multiple delimiters
+  u16cpy(str, u"abc<def>ghi");
+  ctx = nullptr;
+  const KMX_WCHAR *reverse_delim = u"><";
+	EXPECT_TRUE(!u16cmp(u"abc", u16tok(str, reverse_delim, &ctx)));
+  EXPECT_TRUE(!u16cmp(u"def", u16tok(nullptr, reverse_delim, &ctx)));
+  EXPECT_TRUE(!u16cmp(u"ghi", u16tok(nullptr, reverse_delim, &ctx)));
+  EXPECT_EQ(nullptr, ctx);
 }
 
 TEST(kmx_u16_Test, u16tok_str_compare_to_strtok) {
   // Compare behaviour of strtok:
-  char sbuffer[128] = "test a space  and two";
-  EXPECT_TRUE(!strcmp("test", strtok(sbuffer, " ")));
+	char str[LINESIZE];
+
+  // sequence of tokens
+  strcpy(str, "test a space  and two");
+  EXPECT_TRUE(!strcmp("test", strtok(str, " ")));
   EXPECT_TRUE(!strcmp("a", strtok(nullptr, " ")));
   EXPECT_TRUE(!strcmp("space", strtok(nullptr, " ")));
   EXPECT_TRUE(!strcmp("and", strtok(nullptr, " ")));
   EXPECT_TRUE(!strcmp("two", strtok(nullptr, " ")));
+  EXPECT_EQ(nullptr, strtok(nullptr, " "));
+
+  // only a delimiter
+  strcpy(str, " ");
+  EXPECT_EQ(nullptr, strtok(str, " "));
+  EXPECT_EQ(nullptr, strtok(nullptr, " "));
 
-  char sbuffer_space[128] = " ";
-  EXPECT_EQ(nullptr, strtok(sbuffer_space, " "));
+  // delimiters at end
+  strcpy(str, "a b   ");
+  EXPECT_TRUE(!strcmp("a", strtok(str, " ")));
+  EXPECT_TRUE(!strcmp("b", strtok(nullptr, " ")));
   EXPECT_EQ(nullptr, strtok(nullptr, " "));
+
+  // multiple delimiters
+  strcpy(str, "abc<def>ghi");
+  const char *delim = "<>";
+	EXPECT_TRUE(!strcmp("abc", strtok(str, delim)));
+  EXPECT_TRUE(!strcmp("def", strtok(nullptr, delim)));
+  EXPECT_TRUE(!strcmp("ghi", strtok(nullptr, delim)));
+  EXPECT_EQ(nullptr, strtok(nullptr, delim));
+
+  // check sensitivity of ordering with multiple delimiters
+  strcpy(str, "abc<def>ghi");
+  const char *reverse_delim = "><";
+	EXPECT_TRUE(!strcmp("abc", strtok(str, reverse_delim)));
+  EXPECT_TRUE(!strcmp("def", strtok(nullptr, reverse_delim)));
+  EXPECT_TRUE(!strcmp("ghi", strtok(nullptr, reverse_delim)));
+  EXPECT_EQ(nullptr, strtok(nullptr, reverse_delim));
 }

developer/src/kmcmplib/tests/meson.build

@@ -188,4 +188,4 @@ gtestkmx_u16test = executable('gtest-kmx_u16-test', 'gtest-kmx_u16-test.cpp',
     dependencies: [ icuuc_dep, gtest_dep, gmock_dep ],
   )
 
-test('gtest-kmx_u16-test', gtestkmx_u16test)
+test('gtest-kmx_u16-test', gtestkmx_u16test)