Skip to content

Commit 78f03bb

Browse files
nietrasnietras
authored
Initial rough port of llm.c (#2)
1 parent 3dfac5c commit 78f03bb

File tree

9 files changed

+1715
-45
lines changed

9 files changed

+1715
-45
lines changed

‎.github/workflows/comparison-bench.yml‎

Lines changed: 0 additions & 29 deletions
This file was deleted.

‎README.md‎

Lines changed: 137 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
11
# llm.cs - C# port of @karpathy [llm.c](https://github.com/karpathy/llm.c)
2-
![.NET](https://img.shields.io/badge/net7.0%20net8.0-5C2D91?logo=.NET&labelColor=gray)
2+
![.NET](https://img.shields.io/badge/net8.0-5C2D91?logo=.NET&labelColor=gray)
33
![C#](https://img.shields.io/badge/12.0-239120?logo=csharp&logoColor=white&labelColor=gray)
4-
![Lines of code](https://tokei.rs/b1/github/nietras/Llm?category=code)
5-
[![Build Status](https://github.com/nietras/Llm/actions/workflows/dotnet.yml/badge.svg?branch=main)](https://github.com/nietras/Llm/actions/workflows/dotnet.yml)
6-
[![Super-Linter](https://github.com/nietras/Llm/actions/workflows/super-linter.yml/badge.svg)](https://github.com/marketplace/actions/super-linter)
7-
[![codecov](https://codecov.io/gh/nietras/Llm/branch/main/graph/badge.svg?token=WN56CR3X0D)](https://codecov.io/gh/nietras/Llm)
8-
[![CodeQL](https://github.com/nietras/Llm/workflows/CodeQL/badge.svg)](https://github.com/nietras/Llm/actions?query=workflow%3ACodeQL)
4+
![Lines of code](https://tokei.rs/b1/github/nietras/llm.cs?category=code)
5+
[![Build Status](https://github.com/nietras/llm.cs/actions/workflows/dotnet.yml/badge.svg?branch=main)](https://github.com/nietras/llm.cs/actions/workflows/dotnet.yml)
6+
[![Super-Linter](https://github.com/nietras/llm.cs/actions/workflows/super-linter.yml/badge.svg)](https://github.com/marketplace/actions/super-linter)
7+
[![codecov](https://codecov.io/gh/nietras/llm.cs/branch/main/graph/badge.svg?token=WN56CR3X0D)](https://codecov.io/gh/nietras/llm.cs)
8+
[![CodeQL](https://github.com/nietras/llm.cs/workflows/CodeQL/badge.svg)](https://github.com/nietras/llm.cs/actions?query=workflow%3ACodeQL)
99
[![Nuget](https://img.shields.io/nuget/v/Llm?color=purple)](https://www.nuget.org/packages/Llm/)
10-
[![Release](https://img.shields.io/github/v/release/nietras/Llm)](https://github.com/nietras/Llm/releases/)
10+
[![Release](https://img.shields.io/github/v/release/nietras/llm.cs)](https://github.com/nietras/llm.cs/releases/)
1111
[![downloads](https://img.shields.io/nuget/dt/Llm)](https://www.nuget.org/packages/Llm)
12-
![Size](https://img.shields.io/github/repo-size/nietras/Llm.svg)
13-
[![License](https://img.shields.io/github/license/nietras/Llm)](https://github.com/nietras/Llm/blob/main/LICENSE)
12+
![Size](https://img.shields.io/github/repo-size/nietras/Llm.cs.svg)
13+
[![License](https://img.shields.io/github/license/nietras/Llm.cs)](https://github.com/nietras/llm.cs/blob/main/LICENSE)
1414
[![Blog](https://img.shields.io/badge/blog-nietras.com-4993DD)](https://nietras.com)
1515

1616
⭐ Please star this project if you like it. ⭐
@@ -33,6 +33,133 @@ TODO
3333
[assembly: System.Runtime.Versioning.TargetFramework(".NETCoreApp,Version=v8.0", FrameworkDisplayName=".NET 8.0")]
3434
namespace nietras.LargeLanguageModel
3535
{
36-
public static class Llm { }
36+
public static class Gpt2
37+
{
38+
public static void Train() { }
39+
public static unsafe void attention_backward(float* dinp, float* dpreatt, float* datt, float* dout, float* inp, float* att, int B, int T, int C, int NH) { }
40+
public static unsafe void attention_forward(float* output, float* preatt, float* att, float* inp, int B, int T, int C, int NH) { }
41+
public static unsafe T* calloc<T>(long size)
42+
where T : unmanaged { }
43+
public static unsafe void crossentropy_forward(float* losses, float* probs, int* targets, int B, int T, int V) { }
44+
public static unsafe void crossentropy_softmax_backward(float* dlogits, float* dlosses, float* probs, int* targets, int B, int T, int V) { }
45+
public static unsafe void encoder_backward(float* dwte, float* dwpe, float* dout, int* inp, int B, int T, int C) { }
46+
public static unsafe void encoder_forward(float* output, int* inp, float* wte, float* wpe, int B, int T, int C) { }
47+
public static unsafe void free<T>(T* ptr)
48+
where T : unmanaged { }
49+
public static unsafe void gelu_backward(float* dinp, float* inp, float* dout, int N) { }
50+
public static unsafe void gelu_forward(float* output, float* inp, int N) { }
51+
public static unsafe void gpt2_build_from_checkpoint(nietras.LargeLanguageModel.Gpt2.GPT2* model, string checkpoint_path) { }
52+
public static unsafe void layernorm_backward(float* dinp, float* dweight, float* dbias, float* dout, float* inp, float* weight, float* mean, float* rstd, int B, int T, int C) { }
53+
public static unsafe void layernorm_forward(float* output, float* mean, float* rstd, float* inp, float* weight, float* bias, int B, int T, int C) { }
54+
public static unsafe T* malloc<T>(long size)
55+
where T : unmanaged { }
56+
public static unsafe float* malloc_and_point_activations(nietras.LargeLanguageModel.Gpt2.ActivationTensors* acts, long* act_sizes) { }
57+
public static unsafe float* malloc_and_point_parameters(nietras.LargeLanguageModel.Gpt2.ParameterTensors* parameters, long* param_sizes) { }
58+
public static unsafe void matmul_backward(float* dinp, float* dweight, float* dbias, float* dout, float* inp, float* weight, int B, int T, int C, int OC) { }
59+
public static unsafe void matmul_forward(float* output, float* inp, float* weight, float* bias, int B, int T, int C, int OC) { }
60+
public static unsafe void memcpy<T>(T* dest, T* src, long size)
61+
where T : unmanaged { }
62+
public static unsafe void memset<T>(T* ptr, long size)
63+
where T : unmanaged { }
64+
public static unsafe void residual_backward(float* dinp1, float* dinp2, float* dout, int N) { }
65+
public static unsafe void residual_forward(float* output, float* inp1, float* inp2, int N) { }
66+
public static unsafe void softmax_forward(float* probs, float* logits, int B, int T, int V) { }
67+
public struct ActivationTensors
68+
{
69+
public unsafe float* att;
70+
public unsafe float* attproj;
71+
public unsafe float* atty;
72+
public unsafe float* encoded;
73+
public unsafe float* fch;
74+
public unsafe float* fch_gelu;
75+
public unsafe float* fcproj;
76+
public unsafe float* ln1;
77+
public unsafe float* ln1_mean;
78+
public unsafe float* ln1_rstd;
79+
public unsafe float* ln2;
80+
public unsafe float* ln2_mean;
81+
public unsafe float* ln2_rstd;
82+
public unsafe float* lnf;
83+
public unsafe float* lnf_mean;
84+
public unsafe float* lnf_rstd;
85+
public unsafe float* logits;
86+
public unsafe float* losses;
87+
public unsafe float* preatt;
88+
public unsafe float* probs;
89+
public unsafe float* qkv;
90+
public unsafe float* residual2;
91+
public unsafe float* residual3;
92+
}
93+
public class DataLoader : System.IDisposable
94+
{
95+
public int B;
96+
public int T;
97+
public unsafe int* batch;
98+
public long current_position;
99+
public long file_size;
100+
public unsafe int* inputs;
101+
public long num_batches;
102+
public unsafe int* targets;
103+
public System.IO.FileStream tokens_file;
104+
public DataLoader(string filename, int B, int T) { }
105+
public void Dispose() { }
106+
protected virtual void Dispose(bool disposing) { }
107+
public void dataloader_free() { }
108+
public void dataloader_next_batch() { }
109+
public void dataloader_reset() { }
110+
}
111+
public struct GPT2
112+
{
113+
[System.Runtime.CompilerServices.FixedBuffer(typeof(long), 23)]
114+
public nietras.LargeLanguageModel.Gpt2.GPT2.<act_sizes>e__FixedBuffer act_sizes;
115+
public nietras.LargeLanguageModel.Gpt2.ActivationTensors acts;
116+
public unsafe float* acts_memory;
117+
public int batch_size;
118+
public nietras.LargeLanguageModel.Gpt2.GPT2Config config;
119+
public nietras.LargeLanguageModel.Gpt2.ParameterTensors grads;
120+
public nietras.LargeLanguageModel.Gpt2.ActivationTensors grads_acts;
121+
public unsafe float* grads_acts_memory;
122+
public unsafe float* grads_memory;
123+
public unsafe int* inputs;
124+
public unsafe float* m_memory;
125+
public float mean_loss;
126+
public long num_activations;
127+
public long num_parameters;
128+
[System.Runtime.CompilerServices.FixedBuffer(typeof(long), 16)]
129+
public nietras.LargeLanguageModel.Gpt2.GPT2.<param_sizes>e__FixedBuffer param_sizes;
130+
public nietras.LargeLanguageModel.Gpt2.ParameterTensors parameters;
131+
public unsafe float* params_memory;
132+
public int seq_len;
133+
public unsafe int* targets;
134+
public unsafe float* v_memory;
135+
}
136+
public struct GPT2Config
137+
{
138+
public int channels;
139+
public int max_seq_len;
140+
public int num_heads;
141+
public int num_layers;
142+
public int vocab_size;
143+
}
144+
public struct ParameterTensors
145+
{
146+
public unsafe float* attprojb;
147+
public unsafe float* attprojw;
148+
public unsafe float* fcb;
149+
public unsafe float* fcprojb;
150+
public unsafe float* fcprojw;
151+
public unsafe float* fcw;
152+
public unsafe float* ln1b;
153+
public unsafe float* ln1w;
154+
public unsafe float* ln2b;
155+
public unsafe float* ln2w;
156+
public unsafe float* lnfb;
157+
public unsafe float* lnfw;
158+
public unsafe float* qkvb;
159+
public unsafe float* qkvw;
160+
public unsafe float* wpe;
161+
public unsafe float* wte;
162+
}
163+
}
37164
}
38165
```

‎src/Llm.Test/ReadMeTest.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ public void ReadMeTest_UpdateExampleCodeInMarkdown()
109109
[TestMethod]
110110
public void ReadMeTest_PublicApi()
111111
{
112-
var publicApi = typeof(Llm).Assembly.GeneratePublicApi();
112+
var publicApi = typeof(Gpt2).Assembly.GeneratePublicApi();
113113

114114
var readmeFilePath = s_readmeFilePath;
115115
var readmeLines = File.ReadAllLines(readmeFilePath);

0 commit comments

Comments
 (0)