more comment

Signed-off-by: Terry Kong <[email protected]>
NVIDIA · Nov 7, 2024 · 380e37d · 380e37d
1 parent 3d1ae26
commit 380e37d
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/nemo_aligner/data/nlp/builders.py b/nemo_aligner/data/nlp/builders.py
@@ -332,7 +332,10 @@ def collate_with_pad_to_max_batch(max_seqlen, tokenizer_eos_id, cfg, generate_ma
 
 
 def identity_collate(batch):
-    # Useful since torch's data loader's default collate will crash with ragged sequences.
+    """
+    Useful since torch's data loader's default collate will crash with ragged sequences.
+    Also, this function is needed b/c lambda functions aren't pickle-able.
+    """
     return batch