3v324v23 Claude commited on
Commit
774a364
·
1 Parent(s): b53ad2f

Add YaRN/RoPE auto-configuration from training_ctx + target n_ctx

Browse files

New auto_configure_yarn() derives rope_freq_scale = training_ctx / target_n_ctx
so the effective context matches the user's target, resetting the other YaRN
knobs to llama.cpp defaults. Wired into the YaRN accordion as an
"Auto-configure from training_ctx + n_ctx" button that writes the five
gr.Number inputs back via gr.update. compute() signature unchanged — the
existing estimate()/command_preview() YaRN flow picks up the new values.

Co-Authored-By: Claude <noreply@anthropic.com>

Files changed (4) hide show
  1. app.py +34 -0
  2. tests/test_vramcalc.py +35 -0
  3. vramcalc/__init__.py +3 -1
  4. vramcalc/yarn.py +54 -0
app.py CHANGED
@@ -28,6 +28,7 @@ from vramcalc import (
28
  best_quant,
29
  min_gpu_setup,
30
  mmproj_bytes_from_tensors,
 
31
  )
32
  from vramcalc.presets import PRESETS, PRESET_NAMES
33
 
@@ -217,6 +218,26 @@ def run_min_gpu_setup(arch_fields, n_ctx, quant, gpu_vram_text, split_mode, main
217
  return f"{badge} {res.note}\n\nSubset: [{subset}]\n\nTotal: {format_bytes(res.total_bytes)}"
218
 
219
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
220
  def _arch_to_fields(m: ModelArch):
221
  return [
222
  m.name, m.architecture, m.n_layer, m.n_embd, m.n_head, m.n_head_kv,
@@ -529,6 +550,11 @@ def build_ui():
529
  with gr.Row():
530
  yarn_beta_fast = gr.Number(label="yarn_beta_fast", value=32.0)
531
  yarn_beta_slow = gr.Number(label="yarn_beta_slow", value=1.0)
 
 
 
 
 
532
 
533
  with gr.Accordion("Speculative decoding (draft model)", open=False):
534
  spec_type = gr.Dropdown(
@@ -701,6 +727,14 @@ def build_ui():
701
  outputs=[af_mingu_out],
702
  )
703
 
 
 
 
 
 
 
 
 
704
  return demo
705
 
706
 
 
28
  best_quant,
29
  min_gpu_setup,
30
  mmproj_bytes_from_tensors,
31
+ auto_configure_yarn,
32
  )
33
  from vramcalc.presets import PRESETS, PRESET_NAMES
34
 
 
218
  return f"{badge} {res.note}\n\nSubset: [{subset}]\n\nTotal: {format_bytes(res.total_bytes)}"
219
 
220
 
221
+ def auto_configure_yarn_handler(arch_fields, n_ctx):
222
+ """Auto-derive YaRN/RoPE params from training_ctx + target n_ctx.
223
+
224
+ Writes the five YaRN gr.Number inputs back via gr.update and returns a
225
+ markdown status line as the sixth output.
226
+ """
227
+ arch = _fields_to_arch(arch_fields)
228
+ cfg = auto_configure_yarn(arch.training_ctx, int(n_ctx or 8192))
229
+ badge = "✅" if cfg.scaling else "ℹ️"
230
+ status = f"{badge} {cfg.note}" if cfg.note else ""
231
+ return (
232
+ gr.update(value=cfg.rope_freq_scale),
233
+ gr.update(value=cfg.yarn_ext_factor),
234
+ gr.update(value=cfg.yarn_attn_factor),
235
+ gr.update(value=cfg.yarn_beta_fast),
236
+ gr.update(value=cfg.yarn_beta_slow),
237
+ status,
238
+ )
239
+
240
+
241
  def _arch_to_fields(m: ModelArch):
242
  return [
243
  m.name, m.architecture, m.n_layer, m.n_embd, m.n_head, m.n_head_kv,
 
550
  with gr.Row():
551
  yarn_beta_fast = gr.Number(label="yarn_beta_fast", value=32.0)
552
  yarn_beta_slow = gr.Number(label="yarn_beta_slow", value=1.0)
553
+ with gr.Row():
554
+ yarn_auto_btn = gr.Button(
555
+ "Auto-configure from training_ctx + n_ctx"
556
+ )
557
+ yarn_auto_status = gr.Markdown("")
558
 
559
  with gr.Accordion("Speculative decoding (draft model)", open=False):
560
  spec_type = gr.Dropdown(
 
727
  outputs=[af_mingu_out],
728
  )
729
 
730
+ # YaRN auto-configure: derive rope_freq_scale from training_ctx + n_ctx
731
+ yarn_auto_btn.click(
732
+ fn=auto_configure_yarn_handler,
733
+ inputs=[arch_state, n_ctx],
734
+ outputs=[rope_freq_scale, yarn_ext_factor, yarn_attn_factor,
735
+ yarn_beta_fast, yarn_beta_slow, yarn_auto_status],
736
+ )
737
+
738
  return demo
739
 
740
 
tests/test_vramcalc.py CHANGED
@@ -10,6 +10,7 @@ from vramcalc import (
10
  compute_scratch_bytes,
11
  yarn_effective_context,
12
  yarn_warnings,
 
13
  gpu_split,
14
  estimate,
15
  command_preview,
@@ -155,6 +156,40 @@ def test_yarn_warnings_no_warning_when_within():
155
  assert w == []
156
 
157
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
158
  def test_gpu_split_layer_distributes_weights_and_kv():
159
  res = gpu_split(
160
  gpu_vram_bytes=[24 << 30, 24 << 30, 16 << 30],
 
10
  compute_scratch_bytes,
11
  yarn_effective_context,
12
  yarn_warnings,
13
+ auto_configure_yarn,
14
  gpu_split,
15
  estimate,
16
  command_preview,
 
156
  assert w == []
157
 
158
 
159
+ def test_auto_configure_yarn_extends():
160
+ cfg = auto_configure_yarn(8192, 32768)
161
+ assert cfg.scaling is True
162
+ assert cfg.rope_freq_scale == 0.25
163
+ # derived scale yields the requested effective context
164
+ assert yarn_effective_context(8192, cfg.rope_freq_scale) == 32768
165
+ # remaining knobs reset to llama.cpp defaults
166
+ assert cfg.yarn_ext_factor == -1.0
167
+ assert cfg.yarn_attn_factor == 1.0
168
+ assert cfg.yarn_beta_fast == 32.0
169
+ assert cfg.yarn_beta_slow == 1.0
170
+
171
+
172
+ def test_auto_configure_yarn_within_training():
173
+ cfg = auto_configure_yarn(131072, 8192)
174
+ assert cfg.scaling is False
175
+ assert cfg.rope_freq_scale == 1.0
176
+ assert "no YaRN scaling needed" in cfg.note
177
+
178
+
179
+ def test_auto_configure_yarn_unknown_training_ctx():
180
+ cfg = auto_configure_yarn(0, 32768)
181
+ assert cfg.scaling is False
182
+ assert cfg.rope_freq_scale == 1.0
183
+ assert "unknown" in cfg.note.lower()
184
+
185
+
186
+ def test_auto_configure_yarn_invalid_target():
187
+ cfg = auto_configure_yarn(8192, 0)
188
+ assert cfg.scaling is False
189
+ assert cfg.rope_freq_scale == 1.0
190
+ assert "invalid" in cfg.note.lower()
191
+
192
+
193
  def test_gpu_split_layer_distributes_weights_and_kv():
194
  res = gpu_split(
195
  gpu_vram_bytes=[24 << 30, 24 << 30, 16 << 30],
vramcalc/__init__.py CHANGED
@@ -5,7 +5,7 @@ Pure calculation logic, importable without Gradio so it can be unit-tested.
5
 
6
  from .quant import QUANT_BPW, weight_bytes, quant_from_filename
7
  from .kv import kv_cache_bytes, cache_dtype_bytes, compute_scratch_bytes
8
- from .yarn import yarn_effective_context, yarn_warnings
9
  from .gpu import GpuBudget, GpuSpec, gpu_split, fit_gpus, GpuSplitResult
10
  from .gguf import (
11
  GGUFMetadata,
@@ -36,6 +36,8 @@ __all__ = [
36
  "compute_scratch_bytes",
37
  "yarn_effective_context",
38
  "yarn_warnings",
 
 
39
  "GpuBudget",
40
  "GpuSpec",
41
  "GpuSplitResult",
 
5
 
6
  from .quant import QUANT_BPW, weight_bytes, quant_from_filename
7
  from .kv import kv_cache_bytes, cache_dtype_bytes, compute_scratch_bytes
8
+ from .yarn import yarn_effective_context, yarn_warnings, YarnConfig, auto_configure_yarn
9
  from .gpu import GpuBudget, GpuSpec, gpu_split, fit_gpus, GpuSplitResult
10
  from .gguf import (
11
  GGUFMetadata,
 
36
  "compute_scratch_bytes",
37
  "yarn_effective_context",
38
  "yarn_warnings",
39
+ "YarnConfig",
40
+ "auto_configure_yarn",
41
  "GpuBudget",
42
  "GpuSpec",
43
  "GpuSplitResult",
vramcalc/yarn.py CHANGED
@@ -9,6 +9,60 @@ sanity check.
9
 
10
  from __future__ import annotations
11
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12
 
13
  def yarn_effective_context(training_ctx: int, rope_freq_scale: float) -> int:
14
  """Effective (interpolated) context given a rope_freq_scale.
 
9
 
10
  from __future__ import annotations
11
 
12
+ from dataclasses import dataclass
13
+
14
+
15
+ @dataclass
16
+ class YarnConfig:
17
+ """Recommended YaRN/RoPE parameters for a target context.
18
+
19
+ ``rope_freq_scale`` is the only value we actually derive; the rest are the
20
+ llama.cpp defaults (matching :class:`vramcalc.report.Inputs`) so a stale
21
+ manual tweak can't linger after auto-configuration.
22
+ """
23
+ rope_freq_scale: float = 1.0
24
+ yarn_ext_factor: float = -1.0
25
+ yarn_attn_factor: float = 1.0
26
+ yarn_beta_fast: float = 32.0
27
+ yarn_beta_slow: float = 1.0
28
+ scaling: bool = False # True when target > training_ctx (yarn needed)
29
+ note: str = "" # human-readable explanation / caveats
30
+
31
+
32
+ def auto_configure_yarn(training_ctx: int, target_n_ctx: int) -> YarnConfig:
33
+ """Derive recommended YaRN/RoPE params from training_ctx + a target n_ctx.
34
+
35
+ The llama.cpp long-context recipe is ``--rope-scaling yarn`` with
36
+ ``rope_freq_scale = training_ctx / target_n_ctx`` (so the effective context
37
+ matches the target), leaving extrapolation/attention/beta at their defaults.
38
+ Returns a :class:`YarnConfig` with everything reset to those defaults when no
39
+ extension is needed or the inputs are unusable.
40
+ """
41
+ if target_n_ctx is None or target_n_ctx <= 0:
42
+ return YarnConfig(note="Target context is invalid; nothing to configure.")
43
+ if training_ctx is None or training_ctx <= 0:
44
+ return YarnConfig(
45
+ note="Training context unknown — fetch a GGUF or load a preset, "
46
+ "or set rope_freq_scale manually.",
47
+ )
48
+ if target_n_ctx <= training_ctx:
49
+ return YarnConfig(
50
+ note=f"Target context {target_n_ctx} is within training context "
51
+ f"{training_ctx}; no YaRN scaling needed.",
52
+ )
53
+ scale = round(training_ctx / target_n_ctx, 6)
54
+ if scale <= 0:
55
+ return YarnConfig(note="Target context is invalid; nothing to configure.")
56
+ eff = yarn_effective_context(training_ctx, scale)
57
+ stretch = (1.0 / scale) if scale else 0.0
58
+ return YarnConfig(
59
+ rope_freq_scale=scale,
60
+ scaling=True,
61
+ note=f"Set rope_freq_scale={scale} so effective context ≈ {eff} "
62
+ f"(training_ctx {training_ctx} × {stretch:.1f}×). "
63
+ f"YaRN extrapolation auto (--yarn-ext-factor -1).",
64
+ )
65
+
66
 
67
  def yarn_effective_context(training_ctx: int, rope_freq_scale: float) -> int:
68
  """Effective (interpolated) context given a rope_freq_scale.