Skip to content

Commit ef08074

Browse files
jartjart
committed
Start porting syntax highlighter to JavaScript
1 parent 9b96502 commit ef08074

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+11879
-28
lines changed

‎llamafile/BUILD.mk

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -182,10 +182,11 @@ o/$(MODE)/llamafile/highlight_c_test: \
182182
o/$(MODE)/llamafile/is_keyword_c_pod.o \
183183
o/$(MODE)/llamafile/is_keyword_cpp.o \
184184

185-
o/$(MODE)/llamafile/highlight_python_test: \
186-
o/$(MODE)/llamafile/highlight_python_test.o \
187-
o/$(MODE)/llamafile/highlight_python.o \
188-
o/$(MODE)/llamafile/is_keyword_python.o \
185+
o/$(MODE)/llamafile/highlight_python_test: \
186+
o/$(MODE)/llamafile/highlight_python_test.o \
187+
o/$(MODE)/llamafile/highlight_python.o \
188+
o/$(MODE)/llamafile/is_keyword_python.o \
189+
o/$(MODE)/llamafile/is_keyword_python_constant.o \
189190

190191
o/$(MODE)/llamafile/high: \
191192
o/$(MODE)/llamafile/high.o \

‎llamafile/chatbot.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -405,6 +405,8 @@ static void eval_tokens(std::vector<llama_token> tokens, int n_batch) {
405405
die_out_of_context(n_eval);
406406
g_history.insert(g_history.end(), tokens.begin() + i, tokens.begin() + i + n_eval);
407407
}
408+
if (N > n_batch)
409+
clear_ephemeral();
408410
}
409411

410412
static void eval_id(int id) {

‎llamafile/flags.cpp

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
// See the License for the specific language governing permissions and
1616
// limitations under the License.
1717

18+
#include "flags.h"
1819
#include "debug.h"
1920
#include "llama.cpp/cores.h"
2021
#include "llamafile.h"
@@ -47,7 +48,7 @@ bool FLAG_trace = false;
4748
bool FLAG_unsecure = false;
4849
const char *FLAG_file = nullptr;
4950
const char *FLAG_ip_header = nullptr;
50-
const char *FLAG_listen = "0.0.0.0:8080";
51+
const char *FLAG_listen = "127.0.0.1:8080";
5152
const char *FLAG_model = nullptr;
5253
const char *FLAG_prompt = nullptr;
5354
const char *FLAG_url_prefix = "";
@@ -74,6 +75,8 @@ int FLAG_verbose = 0;
7475
int FLAG_warmup = true;
7576
int FLAG_workers;
7677

78+
std::vector<std::string> FLAG_headers;
79+
7780
static wontreturn void usage(int rc, int fd) {
7881
tinyprint(fd, "usage: ", program_invocation_name, " -m MODEL -l [HOST:]PORT\n", NULL);
7982
exit(rc);
@@ -236,6 +239,13 @@ void llamafile_get_flags(int argc, char **argv) {
236239
continue;
237240
}
238241

242+
if (!strcmp(flag, "-H") || !strcmp(flag, "--header")) {
243+
if (i == argc)
244+
missing("--header");
245+
FLAG_headers.push_back(argv[i++]);
246+
continue;
247+
}
248+
239249
if (!strcmp(flag, "--http-ibuf-size")) {
240250
if (i == argc)
241251
missing("--http-ibuf-size");

‎llamafile/flags.h

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
2+
// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
3+
//
4+
// Copyright 2024 Mozilla Foundation
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
18+
#pragma once
19+
#include <string>
20+
#include <vector>
21+
22+
extern std::vector<std::string> FLAG_headers;

‎llamafile/highlight.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ Highlight *Highlight::create(const std::string_view &lang) {
103103
return new HighlightCss;
104104

105105
if (lang == "html" || //
106+
lang == "htm" || //
106107
lang == "xhtml" || //
107108
lang == "xml")
108109
return new HighlightHtml;

‎llamafile/highlight.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ is_keyword_f is_keyword_js_constant;
6969
is_keyword_f is_keyword_java;
7070
is_keyword_f is_keyword_java_constant;
7171
is_keyword_f is_keyword_python;
72+
is_keyword_f is_keyword_python_constant;
7273
is_keyword_f is_keyword_rust;
7374
is_keyword_f is_keyword_rust_type;
7475
is_keyword_f is_keyword_rust_constant;
@@ -94,6 +95,7 @@ is_keyword_f is_keyword_lua_builtin;
9495
is_keyword_f is_keyword_lua_constant;
9596
is_keyword_f is_keyword_lisp;
9697
is_keyword_f is_keyword_ada;
98+
is_keyword_f is_keyword_ada_constant;
9799
is_keyword_f is_keyword_haskell;
98100
is_keyword_f is_keyword_perl;
99101
is_keyword_f is_keyword_shell;

‎llamafile/highlight_ada.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,10 @@ void HighlightAda::feed(std::string *r, std::string_view input) {
7171
*r += HI_KEYWORD;
7272
*r += symbol_;
7373
*r += HI_RESET;
74+
} else if (is_keyword_ada_constant(symbol_.data(), symbol_.size())) {
75+
*r += HI_CONSTANT;
76+
*r += symbol_;
77+
*r += HI_RESET;
7478
} else {
7579
*r += symbol_;
7680
}
@@ -129,6 +133,10 @@ void HighlightAda::flush(std::string *r) {
129133
*r += HI_KEYWORD;
130134
*r += symbol_;
131135
*r += HI_RESET;
136+
} else if (is_keyword_ada_constant(symbol_.data(), symbol_.size())) {
137+
*r += HI_CONSTANT;
138+
*r += symbol_;
139+
*r += HI_RESET;
132140
} else {
133141
*r += symbol_;
134142
}

‎llamafile/highlight_c.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ void HighlightC::feed(std::string *r, std::string_view input) {
120120
is_cpp_ = false;
121121
is_define_ = true;
122122
}
123+
*r += HI_RESET;
123124
} else if (is_keyword_c_constant(word_.data(), word_.size())) {
124125
*r += HI_CONSTANT;
125126
*r += word_;

‎llamafile/highlight_python.cpp

Lines changed: 22 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -23,24 +23,22 @@ enum {
2323
NORMAL,
2424
WORD,
2525
COM,
26-
27-
SQUOTE, // '
28-
SQUOTESTR, // '...
29-
SQUOTESTR_BACKSLASH, // '...
30-
SQUOTE2, // ''
31-
SQUOTE3, // '''...
26+
SQUOTE,
27+
SQUOTESTR,
28+
SQUOTESTR_BACKSLASH,
29+
SQUOTE2,
30+
SQUOTE3,
3231
SQUOTE3_BACKSLASH,
33-
SQUOTE31, // '''...'
34-
SQUOTE32, // '''...''
35-
36-
DQUOTE, // "
37-
DQUOTESTR, // "...
38-
DQUOTESTR_BACKSLASH, // "...
39-
DQUOTE2, // ""
40-
DQUOTE3, // """...
32+
SQUOTE31,
33+
SQUOTE32,
34+
DQUOTE,
35+
DQUOTESTR,
36+
DQUOTESTR_BACKSLASH,
37+
DQUOTE2,
38+
DQUOTE3,
4139
DQUOTE3_BACKSLASH,
42-
DQUOTE31, // """..."
43-
DQUOTE32, // """...""
40+
DQUOTE31,
41+
DQUOTE32,
4442
};
4543

4644
HighlightPython::HighlightPython() {
@@ -86,6 +84,10 @@ void HighlightPython::feed(std::string *r, std::string_view input) {
8684
*r += HI_KEYWORD;
8785
*r += word_;
8886
*r += HI_RESET;
87+
} else if (is_keyword_python_constant(word_.data(), word_.size())) {
88+
*r += HI_CONSTANT;
89+
*r += word_;
90+
*r += HI_RESET;
8991
} else {
9092
*r += word_;
9193
}
@@ -266,6 +268,10 @@ void HighlightPython::flush(std::string *r) {
266268
*r += HI_KEYWORD;
267269
*r += word_;
268270
*r += HI_RESET;
271+
} else if (is_keyword_python_constant(word_.data(), word_.size())) {
272+
*r += HI_CONSTANT;
273+
*r += word_;
274+
*r += HI_RESET;
269275
} else {
270276
*r += word_;
271277
}

‎llamafile/is_keyword_ada_constant.gperf

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
%{
2+
#include <string.h>
3+
#include <libc/str/tab.h>
4+
#define GPERF_DOWNCASE
5+
%}
6+
%pic
7+
%ignore-case
8+
%compare-strncmp
9+
%language=ANSI-C
10+
%readonly-tables
11+
%define lookup-function-name is_keyword_ada_constant
12+
%%
13+
True
14+
False

0 commit comments

Comments
 (0)