From 2e62eacfbcefaa22b0ebab8ebcae2d46c61c38fa Mon Sep 17 00:00:00 2001 From: Vulcan <93451215+trholding@users.noreply.github.com> Date: Mon, 21 Aug 2023 15:18:33 +0530 Subject: [PATCH] Baremetal/Portable Embedding (WIP) Embedding model, tokenizer & assets for baremetal and portable builds (WIP) --- Makefile | 8 +- incbin.h | 476 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ run.c | 32 +++- 3 files changed, 511 insertions(+), 5 deletions(-) create mode 100644 incbin.h diff --git a/Makefile b/Makefile index 47ffd85b..4f6e847d 100644 --- a/Makefile +++ b/Makefile @@ -87,11 +87,15 @@ runmkl: run.c runaccel: run.c $(CC) -D AAF -Ofast -fopenmp -march=native run.c -lm -framework Accelerate -o run -.PHONY: cosmorun -cosmorun: +.PHONY: runcosmo +runcosmo: cosmocc -Ofast -D COSMO_BLINK -D COSMO_METAL -D COSMO_ZIP run.c -lm -o run.com zip run.com out/model.bin zip run.com tokenizer.bin + +.PHONY: runboot +runboot: + cosmocc -Ofast -D COSMO_BLINK -D COSMO_METAL -D INC_BIN -D MODPATH=out/model.bin run.c -lm -o run.com # run all tests .PHONY: test diff --git a/incbin.h b/incbin.h new file mode 100644 index 00000000..3f662e15 --- /dev/null +++ b/incbin.h @@ -0,0 +1,476 @@ +/** + * @file incbin.h + * @author Dale Weiler + * @brief Utility for including binary files + * + * Facilities for including binary files into the current translation unit and + * making use from them externally in other translation units. + */ +#ifndef INCBIN_HDR +#define INCBIN_HDR +#include +#if defined(__AVX512BW__) || \ + defined(__AVX512CD__) || \ + defined(__AVX512DQ__) || \ + defined(__AVX512ER__) || \ + defined(__AVX512PF__) || \ + defined(__AVX512VL__) || \ + defined(__AVX512F__) +# define INCBIN_ALIGNMENT_INDEX 6 +#elif defined(__AVX__) || \ + defined(__AVX2__) +# define INCBIN_ALIGNMENT_INDEX 5 +#elif defined(__SSE__) || \ + defined(__SSE2__) || \ + defined(__SSE3__) || \ + defined(__SSSE3__) || \ + defined(__SSE4_1__) || \ + defined(__SSE4_2__) || \ + defined(__neon__) || \ + defined(__ARM_NEON) || \ + defined(__ALTIVEC__) +# define INCBIN_ALIGNMENT_INDEX 4 +#elif ULONG_MAX != 0xffffffffu +# define INCBIN_ALIGNMENT_INDEX 3 +# else +# define INCBIN_ALIGNMENT_INDEX 2 +#endif + +/* Lookup table of (1 << n) where `n' is `INCBIN_ALIGNMENT_INDEX' */ +#define INCBIN_ALIGN_SHIFT_0 1 +#define INCBIN_ALIGN_SHIFT_1 2 +#define INCBIN_ALIGN_SHIFT_2 4 +#define INCBIN_ALIGN_SHIFT_3 8 +#define INCBIN_ALIGN_SHIFT_4 16 +#define INCBIN_ALIGN_SHIFT_5 32 +#define INCBIN_ALIGN_SHIFT_6 64 + +/* Actual alignment value */ +#define INCBIN_ALIGNMENT \ + INCBIN_CONCATENATE( \ + INCBIN_CONCATENATE(INCBIN_ALIGN_SHIFT, _), \ + INCBIN_ALIGNMENT_INDEX) + +/* Stringize */ +#define INCBIN_STR(X) \ + #X +#define INCBIN_STRINGIZE(X) \ + INCBIN_STR(X) +/* Concatenate */ +#define INCBIN_CAT(X, Y) \ + X ## Y +#define INCBIN_CONCATENATE(X, Y) \ + INCBIN_CAT(X, Y) +/* Deferred macro expansion */ +#define INCBIN_EVAL(X) \ + X +#define INCBIN_INVOKE(N, ...) \ + INCBIN_EVAL(N(__VA_ARGS__)) +/* Variable argument count for overloading by arity */ +#define INCBIN_VA_ARG_COUNTER(_1, _2, _3, N, ...) N +#define INCBIN_VA_ARGC(...) INCBIN_VA_ARG_COUNTER(__VA_ARGS__, 3, 2, 1, 0) + +/* Green Hills uses a different directive for including binary data */ +#if defined(__ghs__) +# if (__ghs_asm == 2) +# define INCBIN_MACRO ".file" +/* Or consider the ".myrawdata" entry in the ld file */ +# else +# define INCBIN_MACRO "\tINCBIN" +# endif +#else +# define INCBIN_MACRO ".incbin" +#endif + +#ifndef _MSC_VER +# define INCBIN_ALIGN \ + __attribute__((aligned(INCBIN_ALIGNMENT))) +#else +# define INCBIN_ALIGN __declspec(align(INCBIN_ALIGNMENT)) +#endif + +#if defined(__arm__) || /* GNU C and RealView */ \ + defined(__arm) || /* Diab */ \ + defined(_ARM) /* ImageCraft */ +# define INCBIN_ARM +#endif + +#ifdef __GNUC__ +/* Utilize .balign where supported */ +# define INCBIN_ALIGN_HOST ".balign " INCBIN_STRINGIZE(INCBIN_ALIGNMENT) "\n" +# define INCBIN_ALIGN_BYTE ".balign 1\n" +#elif defined(INCBIN_ARM) +/* + * On arm assemblers, the alignment value is calculated as (1 << n) where `n' is + * the shift count. This is the value passed to `.align' + */ +# define INCBIN_ALIGN_HOST ".align " INCBIN_STRINGIZE(INCBIN_ALIGNMENT_INDEX) "\n" +# define INCBIN_ALIGN_BYTE ".align 0\n" +#else +/* We assume other inline assembler's treat `.align' as `.balign' */ +# define INCBIN_ALIGN_HOST ".align " INCBIN_STRINGIZE(INCBIN_ALIGNMENT) "\n" +# define INCBIN_ALIGN_BYTE ".align 1\n" +#endif + +/* INCBIN_CONST is used by incbin.c generated files */ +#if defined(__cplusplus) +# define INCBIN_EXTERNAL extern "C" +# define INCBIN_CONST extern const +#else +# define INCBIN_EXTERNAL extern +# define INCBIN_CONST const +#endif + +/** + * @brief Optionally override the linker section into which size and data is + * emitted. + * + * @warning If you use this facility, you might have to deal with + * platform-specific linker output section naming on your own. + */ +#if !defined(INCBIN_OUTPUT_SECTION) +# if defined(__APPLE__) +# define INCBIN_OUTPUT_SECTION ".const_data" +# else +# define INCBIN_OUTPUT_SECTION ".rodata" +# endif +#endif + +/** + * @brief Optionally override the linker section into which data is emitted. + * + * @warning If you use this facility, you might have to deal with + * platform-specific linker output section naming on your own. + */ +#if !defined(INCBIN_OUTPUT_DATA_SECTION) +# define INCBIN_OUTPUT_DATA_SECTION INCBIN_OUTPUT_SECTION +#endif + +/** + * @brief Optionally override the linker section into which size is emitted. + * + * @warning If you use this facility, you might have to deal with + * platform-specific linker output section naming on your own. + * + * @note This is useful for Harvard architectures where program memory cannot + * be directly read from the program without special instructions. With this you + * can chose to put the size variable in RAM rather than ROM. + */ +#if !defined(INCBIN_OUTPUT_SIZE_SECTION) +# define INCBIN_OUTPUT_SIZE_SECTION INCBIN_OUTPUT_SECTION +#endif + +#if defined(__APPLE__) +# include "TargetConditionals.h" +# if defined(TARGET_OS_IPHONE) && !defined(INCBIN_SILENCE_BITCODE_WARNING) +# warning "incbin is incompatible with bitcode. Using the library will break upload to App Store if you have bitcode enabled. Add `#define INCBIN_SILENCE_BITCODE_WARNING` before including this header to silence this warning." +# endif +/* The directives are different for Apple branded compilers */ +# define INCBIN_SECTION INCBIN_OUTPUT_SECTION "\n" +# define INCBIN_GLOBAL(NAME) ".globl " INCBIN_MANGLE INCBIN_STRINGIZE(INCBIN_PREFIX) #NAME "\n" +# define INCBIN_INT ".long " +# define INCBIN_MANGLE "_" +# define INCBIN_BYTE ".byte " +# define INCBIN_TYPE(...) +#else +# define INCBIN_SECTION ".section " INCBIN_OUTPUT_SECTION "\n" +# define INCBIN_GLOBAL(NAME) ".global " INCBIN_STRINGIZE(INCBIN_PREFIX) #NAME "\n" +# if defined(__ghs__) +# define INCBIN_INT ".word " +# else +# define INCBIN_INT ".int " +# endif +# if defined(__USER_LABEL_PREFIX__) +# define INCBIN_MANGLE INCBIN_STRINGIZE(__USER_LABEL_PREFIX__) +# else +# define INCBIN_MANGLE "" +# endif +# if defined(INCBIN_ARM) +/* On arm assemblers, `@' is used as a line comment token */ +# define INCBIN_TYPE(NAME) ".type " INCBIN_STRINGIZE(INCBIN_PREFIX) #NAME ", %object\n" +# elif defined(__MINGW32__) || defined(__MINGW64__) +/* Mingw doesn't support this directive either */ +# define INCBIN_TYPE(NAME) +# else +/* It's safe to use `@' on other architectures */ +# define INCBIN_TYPE(NAME) ".type " INCBIN_STRINGIZE(INCBIN_PREFIX) #NAME ", @object\n" +# endif +# define INCBIN_BYTE ".byte " +#endif + +/* List of style types used for symbol names */ +#define INCBIN_STYLE_CAMEL 0 +#define INCBIN_STYLE_SNAKE 1 + +/** + * @brief Specify the prefix to use for symbol names. + * + * @note By default this is "g". + * + * @code + * #define INCBIN_PREFIX incbin + * #include "incbin.h" + * INCBIN(Foo, "foo.txt"); + * + * // Now you have the following symbols instead: + * // const unsigned char incbinFoo[]; + * // const unsigned char *const incbinFoo; + * // const unsigned int incbinFoo; + * @endcode + */ +#if !defined(INCBIN_PREFIX) +# define INCBIN_PREFIX g +#endif + +/** + * @brief Specify the style used for symbol names. + * + * Possible options are + * - INCBIN_STYLE_CAMEL "CamelCase" + * - INCBIN_STYLE_SNAKE "snake_case" + * + * @note By default this is INCBIN_STYLE_CAMEL + * + * @code + * #define INCBIN_STYLE INCBIN_STYLE_SNAKE + * #include "incbin.h" + * INCBIN(foo, "foo.txt"); + * + * // Now you have the following symbols: + * // const unsigned char foo_data[]; + * // const unsigned char *const foo_end; + * // const unsigned int foo_size; + * @endcode + */ +#if !defined(INCBIN_STYLE) +# define INCBIN_STYLE INCBIN_STYLE_CAMEL +#endif + +/* Style lookup tables */ +#define INCBIN_STYLE_0_DATA Data +#define INCBIN_STYLE_0_END End +#define INCBIN_STYLE_0_SIZE Size +#define INCBIN_STYLE_1_DATA _data +#define INCBIN_STYLE_1_END _end +#define INCBIN_STYLE_1_SIZE _size + +/* Style lookup: returning identifier */ +#define INCBIN_STYLE_IDENT(TYPE) \ + INCBIN_CONCATENATE( \ + INCBIN_STYLE_, \ + INCBIN_CONCATENATE( \ + INCBIN_EVAL(INCBIN_STYLE), \ + INCBIN_CONCATENATE(_, TYPE))) + +/* Style lookup: returning string literal */ +#define INCBIN_STYLE_STRING(TYPE) \ + INCBIN_STRINGIZE( \ + INCBIN_STYLE_IDENT(TYPE)) \ + +/* Generate the global labels by indirectly invoking the macro with our style + * type and concatenating the name against them. */ +#define INCBIN_GLOBAL_LABELS(NAME, TYPE) \ + INCBIN_INVOKE( \ + INCBIN_GLOBAL, \ + INCBIN_CONCATENATE( \ + NAME, \ + INCBIN_INVOKE( \ + INCBIN_STYLE_IDENT, \ + TYPE))) \ + INCBIN_INVOKE( \ + INCBIN_TYPE, \ + INCBIN_CONCATENATE( \ + NAME, \ + INCBIN_INVOKE( \ + INCBIN_STYLE_IDENT, \ + TYPE))) + +/** + * @brief Externally reference binary data included in another translation unit. + * + * Produces three external symbols that reference the binary data included in + * another translation unit. + * + * The symbol names are a concatenation of `INCBIN_PREFIX' before *NAME*; with + * "Data", as well as "End" and "Size" after. An example is provided below. + * + * @param TYPE Optional array type. Omitting this picks a default of `unsigned char`. + * @param NAME The name given for the binary data + * + * @code + * INCBIN_EXTERN(Foo); + * + * // Now you have the following symbols: + * // extern const unsigned char Foo[]; + * // extern const unsigned char *const Foo; + * // extern const unsigned int Foo; + * @endcode + * + * You may specify a custom optional data type as well as the first argument. + * @code + * INCBIN_EXTERN(custom_type, Foo); + * + * // Now you have the following symbols: + * // extern const custom_type Foo[]; + * // extern const custom_type *const Foo; + * // extern const unsigned int Foo; + * @endcode + */ +#define INCBIN_EXTERN(...) \ + INCBIN_CONCATENATE(INCBIN_EXTERN_, INCBIN_VA_ARGC(__VA_ARGS__))(__VA_ARGS__) +#define INCBIN_EXTERN_1(NAME, ...) \ + INCBIN_EXTERN_2(unsigned char, NAME) +#define INCBIN_EXTERN_2(TYPE, NAME) \ + INCBIN_EXTERNAL const INCBIN_ALIGN TYPE \ + INCBIN_CONCATENATE( \ + INCBIN_CONCATENATE(INCBIN_PREFIX, NAME), \ + INCBIN_STYLE_IDENT(DATA))[]; \ + INCBIN_EXTERNAL const INCBIN_ALIGN TYPE *const \ + INCBIN_CONCATENATE( \ + INCBIN_CONCATENATE(INCBIN_PREFIX, NAME), \ + INCBIN_STYLE_IDENT(END)); \ + INCBIN_EXTERNAL const unsigned int \ + INCBIN_CONCATENATE( \ + INCBIN_CONCATENATE(INCBIN_PREFIX, NAME), \ + INCBIN_STYLE_IDENT(SIZE)) + +/** + * @brief Externally reference textual data included in another translation unit. + * + * Produces three external symbols that reference the textual data included in + * another translation unit. + * + * The symbol names are a concatenation of `INCBIN_PREFIX' before *NAME*; with + * "Data", as well as "End" and "Size" after. An example is provided below. + * + * @param NAME The name given for the textual data + * + * @code + * INCBIN_EXTERN(Foo); + * + * // Now you have the following symbols: + * // extern const char Foo[]; + * // extern const char *const Foo; + * // extern const unsigned int Foo; + * @endcode + */ +#define INCTXT_EXTERN(NAME) \ + INCBIN_EXTERN_2(char, NAME) + +/** + * @brief Include a binary file into the current translation unit. + * + * Includes a binary file into the current translation unit, producing three symbols + * for objects that encode the data and size respectively. + * + * The symbol names are a concatenation of `INCBIN_PREFIX' before *NAME*; with + * "Data", as well as "End" and "Size" after. An example is provided below. + * + * @param TYPE Optional array type. Omitting this picks a default of `unsigned char`. + * @param NAME The name to associate with this binary data (as an identifier.) + * @param FILENAME The file to include (as a string literal.) + * + * @code + * INCBIN(Icon, "icon.png"); + * + * // Now you have the following symbols: + * // const unsigned char Icon[]; + * // const unsigned char *const Icon; + * // const unsigned int Icon; + * @endcode + * + * You may specify a custom optional data type as well as the first argument. + * These macros are specialized by arity. + * @code + * INCBIN(custom_type, Icon, "icon.png"); + * + * // Now you have the following symbols: + * // const custom_type Icon[]; + * // const custom_type *const Icon; + * // const unsigned int Icon; + * @endcode + * + * @warning This must be used in global scope + * @warning The identifiers may be different if INCBIN_STYLE is not default + * + * To externally reference the data included by this in another translation unit + * please @see INCBIN_EXTERN. + */ +#ifdef _MSC_VER +# define INCBIN(NAME, FILENAME) \ + INCBIN_EXTERN(NAME) +#else +# define INCBIN(...) \ + INCBIN_CONCATENATE(INCBIN_, INCBIN_VA_ARGC(__VA_ARGS__))(__VA_ARGS__) +# if defined(__GNUC__) +# define INCBIN_1(...) _Pragma("GCC error \"Single argument INCBIN not allowed\"") +# elif defined(__clang__) +# define INCBIN_1(...) _Pragma("clang error \"Single argument INCBIN not allowed\"") +# else +# define INCBIN_1(...) /* Cannot do anything here */ +# endif +# define INCBIN_2(NAME, FILENAME) \ + INCBIN_3(unsigned char, NAME, FILENAME) +# define INCBIN_3(TYPE, NAME, FILENAME) INCBIN_COMMON(TYPE, NAME, FILENAME, /* No terminator for binary data */) +# define INCBIN_COMMON(TYPE, NAME, FILENAME, TERMINATOR) \ + __asm__(INCBIN_SECTION \ + INCBIN_GLOBAL_LABELS(NAME, DATA) \ + INCBIN_ALIGN_HOST \ + INCBIN_MANGLE INCBIN_STRINGIZE(INCBIN_PREFIX) #NAME INCBIN_STYLE_STRING(DATA) ":\n" \ + INCBIN_MACRO " \"" FILENAME "\"\n" \ + TERMINATOR \ + INCBIN_GLOBAL_LABELS(NAME, END) \ + INCBIN_ALIGN_BYTE \ + INCBIN_MANGLE INCBIN_STRINGIZE(INCBIN_PREFIX) #NAME INCBIN_STYLE_STRING(END) ":\n" \ + INCBIN_BYTE "1\n" \ + INCBIN_GLOBAL_LABELS(NAME, SIZE) \ + INCBIN_ALIGN_HOST \ + INCBIN_MANGLE INCBIN_STRINGIZE(INCBIN_PREFIX) #NAME INCBIN_STYLE_STRING(SIZE) ":\n" \ + INCBIN_INT INCBIN_MANGLE INCBIN_STRINGIZE(INCBIN_PREFIX) #NAME INCBIN_STYLE_STRING(END) " - " \ + INCBIN_MANGLE INCBIN_STRINGIZE(INCBIN_PREFIX) #NAME INCBIN_STYLE_STRING(DATA) "\n" \ + INCBIN_ALIGN_HOST \ + ".text\n" \ + ); \ + INCBIN_EXTERN(TYPE, NAME) +#endif + +/** + * @brief Include a textual file into the current translation unit. + * + * This behaves the same as INCBIN except it produces char compatible arrays + * and implicitly adds a null-terminator byte, thus the size of data included + * by this is one byte larger than that of INCBIN. + * + * Includes a textual file into the current translation unit, producing three + * symbols for objects that encode the data and size respectively. + * + * The symbol names are a concatenation of `INCBIN_PREFIX' before *NAME*; with + * "Data", as well as "End" and "Size" after. An example is provided below. + * + * @param NAME The name to associate with this binary data (as an identifier.) + * @param FILENAME The file to include (as a string literal.) + * + * @code + * INCTXT(Readme, "readme.txt"); + * + * // Now you have the following symbols: + * // const char Readme[]; + * // const char *const Readme; + * // const unsigned int Readme; + * @endcode + * + * @warning This must be used in global scope + * @warning The identifiers may be different if INCBIN_STYLE is not default + * + * To externally reference the data included by this in another translation unit + * please @see INCBIN_EXTERN. + */ +#if defined(_MSC_VER) +# define INCTXT(NAME, FILENAME) \ + INCBIN_EXTERN(NAME) +#else +# define INCTXT(NAME, FILENAME) \ + INCBIN_COMMON(char, NAME, FILENAME, INCBIN_BYTE "0\n") +#endif + +#endif \ No newline at end of file diff --git a/run.c b/run.c index 0ce5f8ae..2b1d8436 100644 --- a/run.c +++ b/run.c @@ -1,5 +1,31 @@ /* Inference for Llama-2 Transformer model in pure C -The Llama 2 Everywhere fork */ +The Llama 2 Everywhere @trholding (Vulcan) fork */ + + +// ---------------------------------------------------------------------------- +// INCBIN Support Directives + +// String substitution macro needed to pass paths to INCBIN +#define ADDPATH(FPATH) TOSTR(FPATH) +#define TOSTR(FPATH) #FPATH + +#ifdef INC_BIN // Support for embedding model and tokenizer + +#define INCBIN_PREFIX emb_ +#define INCBIN_STYLE INCBIN_STYLE_SNAKE +#include "incbin.h" + +#ifndef MODPATH +#define MODPATH out/model.bin // default model path +#endif +#ifndef TOKPATH +#define TOKPATH tokenizer.bin // default tokenizer path +#endif + +INCBIN(Model, ADDPATH(MODPATH)); // Model path is passed via makefile +INCBIN(Tokenizer, ADDPATH(TOKPATH)); // Tokenizer path is passed via makefile + +#endif // ---------------------------------------------------------------------------- // Actually Portable Executable Format Preprocessor Directives @@ -661,7 +687,7 @@ int main(int argc, char *argv[]) { char *prompt = NULL; // prompt string int buffertokens = 1; // output token buffer size - #ifdef COSMO_ZIP // if this is defined + #if defined(COSMO_ZIP) || defined(INC_BIN) // special case for embedded models // we read the embedded checkpoint from within the executable // 'checkpoint' is necessary arg checkpoint = "/zip/out/model.bin" ; @@ -726,7 +752,7 @@ int main(int argc, char *argv[]) { float* vocab_scores = (float*)malloc(config.vocab_size * sizeof(float)); unsigned int max_token_length; { - #ifdef COSMO_ZIP // if this is defined + #if defined(COSMO_ZIP) || defined(INC_BIN) // special case for embedded tokenizer // we read the embedded tokenizer.bin from within the executable FILE *file = fopen("/zip/tokenizer.bin", "rb"); #else