Refactor.

src/main/c/parts/yaep.c

@@ -2217,6 +2217,7 @@ _VLO_expand_memory (vlo_t * vlo, size_t additional_length)
 //include "yaep.h"
 //include "xmq.h"
 //include "parts/always.h"
+//include "parts/text.h"
 
 /* Terminals are stored a in term set using bits in a bit array.
    The array consists of long ints, typedefed as term_set_el_t.
@@ -2391,7 +2392,7 @@ struct YaepSymb
         {
             /* The following refers for all rules with the nonterminal
                symbol is in the left hand side of the rules. */
-            YaepRule*rules;
+            YaepRule *rules;
             /* Each nonterm is given a unique integer starting from 0. */
             int nonterm_id;
             /* The following value is nonzero if nonterminal may derivate
@@ -2978,6 +2979,7 @@ static void symb_empty(YaepParseState *ps, YaepVocabulary *symbs);
 static void symb_finish_adding_terms(YaepParseState *ps);
 static void symb_print(FILE* f, YaepSymb *symb, bool code_p);
 static void yaep_error(YaepParseState *ps, int code, const char*format, ...);
+static int default_read_token(YaepParseRun *ps, void **attr);
 
 // Global variables /////////////////////////////////////////////////////
 
@@ -7337,6 +7339,9 @@ int yaepParse(YaepParseRun *pr, YaepGrammar *g)
    *ambiguous_p = false;
     pl_init(ps);
     tok_init_p = parse_init_p = false;
+
+    if (!ps->run.read_token) ps->run.read_token = default_read_token;
+
     if ((code = setjmp(error_longjump_buff)) != 0)
     {
         pl_fin(ps);
@@ -7767,5 +7772,23 @@ static void print_state_set(YaepParseState *ps,
         }
     }
 }
+
+static int default_read_token(YaepParseRun *ps, void **attr)
+{
+    *attr = NULL;
+    if (ps->buffer_i >= ps->buffer_stop) return -1;
+
+    int uc = 0;
+    size_t len = 0;
+    bool ok = decode_utf8(ps->buffer_i, ps->buffer_stop, &uc, &len);
+    if (!ok)
+    {
+        fprintf(stderr, "xmq: broken utf8\n");
+        exit(1);
+    }
+    ps->buffer_i += len;
+
+    return uc;
+}
 /****************** YAEP parser single source file end **********************/
 #endif // YAEP_MODULE

src/main/c/parts/yaep.h

@@ -87,9 +87,9 @@ typedef struct YaepTreeNode YaepTreeNode;
 struct YaepParseRun
 {
     void *user_data; // Points to user supplied data, useful inside the callbacked read_token and syntax_error.
-    int *buffer_start; // Points to first token.
-    int *buffer_stop; // Points to address after last token.
-    int *buffer_i; // Points to the next token to read.
+    const char *buffer_start; // Points to first token.
+    const char *buffer_stop; // Points to address after last token.
+    const char *buffer_i; // Points to the next token to read.
     // If read_token is NULL then use the built in read_token that uses buffer start,stop and i.
     int (*read_token)(YaepParseRun *ps, void **attr);
     // If syntax_error is NULL then use the built in stderr error message printer.

src/main/c/xmq.c

@@ -4209,28 +4209,6 @@ YaepParseRun *xmq_get_yaep_parse_run(XMQDoc *doc)
     return (YaepParseRun*)doc->yaep_parse_run_;
 }
 
-static char *input_i_;
-static char *input_start_;
-static char *input_stop_;
-
-static int read_yaep_token(YaepParseRun *ps, void **attr)
-{
-  *attr = NULL;
-  if (input_i_ >= input_stop_) return -1;
-
-  int uc = 0;
-  size_t len = 0;
-  bool ok = decode_utf8(input_i_, input_stop_, &uc, &len);
-  if (!ok)
-  {
-      fprintf(stderr, "xmq: broken utf8\n");
-      exit(1);
-  }
-  input_i_ += len;
-
-  return uc;
-}
-
 void handle_yaep_syntax_error(int err_tok_num,
                               void *err_tok_attr,
                               int start_ignored_tok_num,
@@ -4239,6 +4217,7 @@ void handle_yaep_syntax_error(int err_tok_num,
                               void *start_recovered_tok_attr)
 {
     printf("ixml: syntax error\n");
+    /*
     int start = err_tok_num - 10;
     if (start < 0) start = 0;
     int stop = err_tok_num + 10;
@@ -4250,6 +4229,7 @@ void handle_yaep_syntax_error(int err_tok_num,
     printf("\n");
     for (int i = start; i < err_tok_num; ++i) printf (" ");
     printf("^\n");
+    */
 }
 
 const char *node_yaep_type_to_string(YaepTreeNodeType t)
@@ -4407,20 +4387,18 @@ bool xmqParseBufferWithIXML(XMQDoc *doc, const char *start, const char *stop, XM
     if (!doc || !start || !ixml_grammar) return false;
     if (!stop) stop = start+strlen(start);
 
-    input_start_ = input_i_ = strndup(start, stop-start);
-    if (!input_start_) return false;
-
-    input_stop_ = input_start_ + strlen(input_start_);
-
     yaep_set_one_parse_flag(xmq_get_yaep_grammar(ixml_grammar),
                             (flags & XMQ_FLAG_IXML_ALL_PARSES)?0:1);
 
     yaep_set_error_recovery_flag(xmq_get_yaep_grammar(ixml_grammar),
                                  (flags & XMQ_FLAG_IXML_TRY_TO_RECOVER)?1:0);
 
     YaepParseRun *run = xmq_get_yaep_parse_run(ixml_grammar);
+    run->buffer_start = start;
+    run->buffer_stop = stop;
+    run->buffer_i = start;
+
     YaepGrammar *grammar = xmq_get_yaep_grammar(ixml_grammar);
-    run->read_token = read_yaep_token;
     run->syntax_error = handle_yaep_syntax_error;
 
     // Parse source content using the yaep grammar, previously generated from the ixml source.

src/main/c/yaep/src/yaep.c

@@ -85,6 +85,7 @@
 #include "yaep.h"
 #include "xmq.h"
 #include "parts/always.h"
+#include "parts/text.h"
 
 /* Terminals are stored a in term set using bits in a bit array.
    The array consists of long ints, typedefed as term_set_el_t.
@@ -259,7 +260,7 @@ struct YaepSymb
         {
             /* The following refers for all rules with the nonterminal
                symbol is in the left hand side of the rules. */
-            YaepRule*rules;
+            YaepRule *rules;
             /* Each nonterm is given a unique integer starting from 0. */
             int nonterm_id;
             /* The following value is nonzero if nonterminal may derivate
@@ -846,6 +847,7 @@ static void symb_empty(YaepParseState *ps, YaepVocabulary *symbs);
 static void symb_finish_adding_terms(YaepParseState *ps);
 static void symb_print(FILE* f, YaepSymb *symb, bool code_p);
 static void yaep_error(YaepParseState *ps, int code, const char*format, ...);
+static int default_read_token(YaepParseRun *ps, void **attr);
 
 // Global variables /////////////////////////////////////////////////////
 
@@ -5205,6 +5207,9 @@ int yaepParse(YaepParseRun *pr, YaepGrammar *g)
    *ambiguous_p = false;
     pl_init(ps);
     tok_init_p = parse_init_p = false;
+
+    if (!ps->run.read_token) ps->run.read_token = default_read_token;
+
     if ((code = setjmp(error_longjump_buff)) != 0)
     {
         pl_fin(ps);
@@ -5635,3 +5640,21 @@ static void print_state_set(YaepParseState *ps,
         }
     }
 }
+
+static int default_read_token(YaepParseRun *ps, void **attr)
+{
+    *attr = NULL;
+    if (ps->buffer_i >= ps->buffer_stop) return -1;
+
+    int uc = 0;
+    size_t len = 0;
+    bool ok = decode_utf8(ps->buffer_i, ps->buffer_stop, &uc, &len);
+    if (!ok)
+    {
+        fprintf(stderr, "xmq: broken utf8\n");
+        exit(1);
+    }
+    ps->buffer_i += len;
+
+    return uc;
+}

src/main/c/yaep/src/yaep.h

@@ -87,9 +87,9 @@ typedef struct YaepTreeNode YaepTreeNode;
 struct YaepParseRun
 {
     void *user_data; // Points to user supplied data, useful inside the callbacked read_token and syntax_error.
-    int *buffer_start; // Points to first token.
-    int *buffer_stop; // Points to address after last token.
-    int *buffer_i; // Points to the next token to read.
+    const char *buffer_start; // Points to first token.
+    const char *buffer_stop; // Points to address after last token.
+    const char *buffer_i; // Points to the next token to read.
     // If read_token is NULL then use the built in read_token that uses buffer start,stop and i.
     int (*read_token)(YaepParseRun *ps, void **attr);
     // If syntax_error is NULL then use the built in stderr error message printer.