utdawn commited on
Commit
e939e1c
·
verified ·
1 Parent(s): f1d89a1

[fix] gate M2T unmasking on sampled-token confidence p_s

Browse files
Files changed (1) hide show
  1. modeling_llada2_moe.py +4 -4
modeling_llada2_moe.py CHANGED
@@ -1520,14 +1520,14 @@ class LLaDA2MoeModelLM(LLaDA2MoePreTrainedModel, GenerationMixin):
1520
  neg_inf = torch.full_like(p0, -float("inf"))
1521
 
1522
  # 3. M2T (mask -> token): threshold-gated with a per-step floor.
1523
- # Note: when temperature > 0 the gate uses greedy confidence ``p0``, not ``p_s``.
1524
- # ``threshold`` decides if a position is ready to unmask; sampling only picks which
1525
- # token (``x_s``) is written, so the final token's own probability is not constrained.
1526
  mt2_index = torch.zeros(block_length, dtype=torch.bool, device=device)
1527
  if mask_index.any():
1528
  if step_id < len(transfer_schedule):
1529
  num_need = transfer_schedule[step_id].item() + new_mask_count
1530
- mask_conf = torch.where(mask_index, p0, neg_inf)
1531
  high_conf = (mask_conf > threshold) & mask_index
1532
  if high_conf.sum().item() >= num_need:
1533
  mt2_index = high_conf
 
1520
  neg_inf = torch.full_like(p0, -float("inf"))
1521
 
1522
  # 3. M2T (mask -> token): threshold-gated with a per-step floor.
1523
+ # The gate uses ``p_s`` -- the confidence of the token that will actually be written
1524
+ # (``x_s``) -- so a position is only unmasked when the sampled token itself is
1525
+ # confident. When temperature == 0, ``p_s == p0``, so this reduces to greedy behavior.
1526
  mt2_index = torch.zeros(block_length, dtype=torch.bool, device=device)
1527
  if mask_index.any():
1528
  if step_id < len(transfer_schedule):
1529
  num_need = transfer_schedule[step_id].item() + new_mask_count
1530
+ mask_conf = torch.where(mask_index, p_s, neg_inf)
1531
  high_conf = (mask_conf > threshold) & mask_index
1532
  if high_conf.sum().item() >= num_need:
1533
  mt2_index = high_conf