add comments

Signed-off-by: Jon Chuang <[email protected]>
NVIDIA · Sep 6, 2023 · 3237ac7 · 3237ac7
1 parent a5dbf1e
commit 3237ac7
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/transformer_engine/common/utils.cuh b/transformer_engine/common/utils.cuh
@@ -591,6 +591,16 @@ struct DynamicReducer : public Reducer<T, 1, WARPS_M, WARPS_N> {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
+/*
+This is an implementation of the parallel Welford algorithm for incrementally computing variance
+
+A detailed reference on the exact version implemented is provided here:
+https://dbs.ifi.uni-heidelberg.de/files/Team/eschubert/publications/SSDBM18-covariance-authorcopy.pdf
+
+An introduction is provided by Wikipedia here:
+https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance?section=5#Parallel_algorithm
+*/
+
 template<typename T>
 inline __device__ void warp_chan_upd_dynamic(T &m_a, T &m2_a, T &n_a, int num_active) { // NOLINT(*)
     // Assume at least leftmost is valid and