Skip to content

Commit a7e55f7

Browse files
nietrasnietras
authored
Rename a few methods to C# PascalCase (#6)
1 parent 07ea337 commit a7e55f7

File tree

6 files changed

+118
-115
lines changed

6 files changed

+118
-115
lines changed

‎README.md‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -35,21 +35,21 @@ namespace nietras.LargeLanguageModel
3535
{
3636
public static class Llm
3737
{
38-
public static unsafe void attention_backward(float* dinp, float* dpreatt, float* datt, float* dout, float* inp, float* att, int B, int T, int C, int NH) { }
39-
public static unsafe void attention_forward(float* output, float* preatt, float* att, float* inp, int B, int T, int C, int NH) { }
40-
public static unsafe void crossentropy_forward(float* losses, float* probs, int* targets, int B, int T, int V) { }
41-
public static unsafe void crossentropy_softmax_backward(float* dlogits, float* dlosses, float* probs, int* targets, int B, int T, int V) { }
42-
public static unsafe void encoder_backward(float* dwte, float* dwpe, float* dout, int* inp, int B, int T, int C) { }
43-
public static unsafe void encoder_forward(float* output, int* inp, float* wte, float* wpe, int B, int T, int C) { }
44-
public static unsafe void gelu_backward(float* dinp, float* inp, float* dout, int N) { }
45-
public static unsafe void gelu_forward(float* output, float* inp, int N) { }
46-
public static unsafe void layernorm_backward(float* dinp, float* dweight, float* dbias, float* dout, float* inp, float* weight, float* mean, float* rstd, int B, int T, int C) { }
47-
public static unsafe void layernorm_forward(float* output, float* mean, float* rstd, float* inp, float* weight, float* bias, int B, int T, int C) { }
48-
public static unsafe void matmul_backward(float* dinp, float* dweight, float* dbias, float* dout, float* inp, float* weight, int B, int T, int C, int OC) { }
49-
public static unsafe void matmul_forward(float* output, float* inp, float* weight, float* bias, int B, int T, int C, int OC) { }
50-
public static unsafe void residual_backward(float* dinp1, float* dinp2, float* dout, int N) { }
51-
public static unsafe void residual_forward(float* output, float* inp1, float* inp2, int N) { }
52-
public static unsafe void softmax_forward(float* probs, float* logits, int B, int T, int V) { }
38+
public static unsafe void AttentionBackward(float* dinp, float* dpreatt, float* datt, float* dout, float* inp, float* att, int B, int T, int C, int NH) { }
39+
public static unsafe void AttentionForward(float* output, float* preatt, float* att, float* inp, int B, int T, int C, int NH) { }
40+
public static unsafe void CrossEntropyForward(float* losses, float* probs, int* targets, int B, int T, int V) { }
41+
public static unsafe void CrossEntropySoftmaxBackward(float* dlogits, float* dlosses, float* probs, int* targets, int B, int T, int V) { }
42+
public static unsafe void EncoderBackward(float* dwte, float* dwpe, float* dout, int* inp, int B, int T, int C) { }
43+
public static unsafe void EncoderForward(float* output, int* inp, float* wte, float* wpe, int B, int T, int C) { }
44+
public static unsafe void GeLUBackward(float* dinp, float* inp, float* dout, int N) { }
45+
public static unsafe void GeLUForward(float* output, float* inp, int N) { }
46+
public static unsafe void LayerNormBackward(float* dinp, float* dweight, float* dbias, float* dout, float* inp, float* weight, float* mean, float* rstd, int B, int T, int C) { }
47+
public static unsafe void LayerNormForward(float* output, float* mean, float* rstd, float* inp, float* weight, float* bias, int B, int T, int C) { }
48+
public static unsafe void MatMulBackward(float* dinp, float* dweight, float* dbias, float* dout, float* inp, float* weight, int B, int T, int C, int OC) { }
49+
public static unsafe void MatMulForward(float* output, float* inp, float* weight, float* bias, int B, int T, int C, int OC) { }
50+
public static unsafe void ResidualBackward(float* dinp1, float* dinp2, float* dout, int N) { }
51+
public static unsafe void ResidualForward(float* output, float* inp1, float* inp2, int N) { }
52+
public static unsafe void SoftmaxForward(float* probs, float* logits, int B, int T, int V) { }
5353
}
5454
}
5555
```

‎src/Llm/Extensions.cs‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,38 @@
1-
using System.Collections.Generic;
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Runtime.CompilerServices;
5+
using System.Runtime.InteropServices;
26

37
namespace nietras.LargeLanguageModel;
48

59
internal static class Extensions
610
{
11+
public static unsafe void ReadExactlyUnmanaged<T>(this FileStream file, Span<T> values)
12+
where T : unmanaged
13+
{
14+
fixed (T* ptr = values)
15+
{
16+
ReadExactlyUnmanaged(file, ptr, values.Length);
17+
}
18+
}
19+
20+
public static unsafe void ReadExactlyUnmanaged<T>(this FileStream file, T* values, long count)
21+
where T : unmanaged
22+
{
23+
Span<T> buffer = stackalloc T[(256 * 1024) / Unsafe.SizeOf<T>()];
24+
var totalReadCount = 0;
25+
while (totalReadCount < count)
26+
{
27+
var countToRead = (int)Math.Min(buffer.Length, count - totalReadCount);
28+
var bufferToRead = buffer.Slice(0, countToRead);
29+
var span = MemoryMarshal.Cast<T, byte>(bufferToRead);
30+
file.ReadExactly(span);
31+
bufferToRead.CopyTo(new Span<T>(values + totalReadCount, countToRead));
32+
totalReadCount += countToRead;
33+
}
34+
}
35+
736
public static IEnumerable<(int i0, int i1)> Enumerate(int count0, int count1)
837
{
938
for (var i0 = 0; i0 < count0; i0++)

‎src/Llm/Gpt2.Test.cs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ public static unsafe void Test()
1313
var dataDirectory = Path.Combine(location!, "../../../");
1414
// build the GPT-2 model from a checkpoint
1515
GPT2 model;
16-
gpt2_build_from_checkpoint(&model, dataDirectory + "gpt2_124M.bin");
16+
BuildFromCheckpoint(&model, dataDirectory + "gpt2_124M.bin");
1717

1818
int C = model.config.channels;
1919
int V = model.config.vocab_size;
@@ -35,7 +35,7 @@ public static unsafe void Test()
3535
Log($"seq_len: {T}");
3636

3737
ParameterTensors expected_grads;
38-
float* expected_grads_memory = malloc_and_point_parameters(&expected_grads, model.param_sizes);
38+
float* expected_grads_memory = AllocateAndPointParameters(&expected_grads, model.param_sizes);
3939

4040
// inputs and expected outputs, only used for error checking
4141
int* x = malloc<int>(B * T);
@@ -61,9 +61,9 @@ public static unsafe void Test()
6161
{
6262
stopwatch.Restart();
6363

64-
gpt2_forward(&model, x, y, B, T);
65-
gpt2_zero_grad(&model);
66-
gpt2_backward(&model);
64+
Forward(&model, x, y, B, T);
65+
ZeroGrad(&model);
66+
Backward(&model);
6767

6868
double time_elapsed_s = stopwatch.Elapsed.TotalSeconds;
6969

@@ -126,7 +126,7 @@ public static unsafe void Test()
126126
}
127127
}
128128

129-
gpt2_update(&model, 1e-4f, 0.9f, 0.999f, 1e-8f, 0.01f, step + 1);
129+
Update(&model, 1e-4f, 0.9f, 0.999f, 1e-8f, 0.01f, step + 1);
130130

131131
// print the timing information at the end
132132
Log($"step {step}: loss {model.mean_loss} (took {time_elapsed_s * 1000} ms)");
@@ -168,7 +168,7 @@ public static unsafe void Test()
168168
free(expected_logits);
169169
free(expected_loss);
170170
free(expected_grads_memory);
171-
gpt2_free(&model);
171+
Free(&model);
172172
}
173173

174174
// poor man's tensor checker

‎src/Llm/Gpt2.Train.cs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ public static unsafe void Train()
1414
var dataDirectory = Path.Combine(location!, "../../../");
1515
// build the GPT-2 model from a checkpoint
1616
GPT2 model;
17-
gpt2_build_from_checkpoint(&model, dataDirectory + "gpt2_124M.bin");
17+
BuildFromCheckpoint(&model, dataDirectory + "gpt2_124M.bin");
1818

1919
// build the DataLoaders from tokens files. for now use tiny_shakespeare if available, else tiny_stories
2020
var tiny_stories_train = dataDirectory + "TinyStories_train.bin";
@@ -51,7 +51,7 @@ public static unsafe void Train()
5151
for (int i = 0; i < val_num_batches; i++)
5252
{
5353
val_loader.dataloader_next_batch();
54-
gpt2_forward(&model, val_loader.inputs, val_loader.targets, B, T);
54+
Forward(&model, val_loader.inputs, val_loader.targets, B, T);
5555
val_loss += model.mean_loss;
5656
}
5757
val_loss /= val_num_batches;
@@ -68,7 +68,7 @@ public static unsafe void Train()
6868
// for each t, we re-compute all activations between 0 and t
6969
// leaving this alone because you want separate code for inference anyway
7070
// the inference here is just for sanity checking purposes
71-
gpt2_forward(&model, gen_tokens, null, 1, t);
71+
Forward(&model, gen_tokens, null, 1, t);
7272
float* probs = model.acts.probs + (t - 1) * model.config.vocab_size;
7373
float coin = random_f32(&rng_state);
7474
int next_token = sample_mult(probs, model.config.vocab_size, coin);
@@ -85,15 +85,15 @@ public static unsafe void Train()
8585
// do a training step
8686
stopwatch.Restart();
8787
train_loader.dataloader_next_batch();
88-
gpt2_forward(&model, train_loader.inputs, train_loader.targets, B, T);
89-
gpt2_zero_grad(&model);
90-
gpt2_backward(&model);
91-
gpt2_update(&model, 1e-4f, 0.9f, 0.999f, 1e-8f, 0.0f, step + 1);
88+
Forward(&model, train_loader.inputs, train_loader.targets, B, T);
89+
ZeroGrad(&model);
90+
Backward(&model);
91+
Update(&model, 1e-4f, 0.9f, 0.999f, 1e-8f, 0.0f, step + 1);
9292
double time_elapsed_ms = stopwatch.Elapsed.TotalMilliseconds;
9393
Log($"step {step}: train loss {model.mean_loss} (took {time_elapsed_ms} ms)");
9494
}
9595

9696
// free
97-
gpt2_free(&model);
97+
Free(&model);
9898
}
9999
}

0 commit comments

Comments
 (0)