From b57ba5cd33e814b13aeede32ddcf21375350700b Mon Sep 17 00:00:00 2001 From: acmore Date: Sat, 18 Jul 2026 23:50:59 +0800 Subject: [PATCH] fix(rm): deepscaler silently zeroes all rewards for non-thinking models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_deepscaler_rule_based_reward returns 0 whenever the response contains neither "" nor "###Response". For any non-thinking model (or a thinking model with stripped tags) every sample scores 0: GRPO groups all become zero-std, advantages vanish, and training runs with no signal — with no error or warning anywhere. The symptom is deceptive: generation looks healthy (sane lengths, ~0% truncation, correct answers visible in the text) while rewards stay flat at 0. Fall back to grading the whole response instead. Measured on DeepSeek-V3.2 (non-thinking chat template): reward 0 -> 0.676 on dapo-math-17k with this one-line change. Co-Authored-By: Claude Fable 5 --- slime/rollout/rm_hub/deepscaler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/slime/rollout/rm_hub/deepscaler.py b/slime/rollout/rm_hub/deepscaler.py index 39d4de383a..014f0677de 100644 --- a/slime/rollout/rm_hub/deepscaler.py +++ b/slime/rollout/rm_hub/deepscaler.py @@ -7,7 +7,10 @@ def get_deepscaler_rule_based_reward(response, label): elif "###Response" in response: model_solution = response.split("###Response")[1] else: - return 0 + # Non-thinking models emit neither marker; grade the whole response + # instead of silently returning 0 (which zeroes every reward and + # kills the GRPO signal without any visible error). + model_solution = response model_answer = extract_answer(model_solution) if model_answer is None: