Upload GDN1 16K/32K Bridge 1B Full Fine-Tune

README.md

@@ -0,0 +1,61 @@
+---
+license: apache-2.0
+base_model: linear-moe-hub/Gated-Deltanet-1.3B
+language:
+- en
+library_name: transformers
+tags:
+- gated-deltanet
+- linear-attention
+- recurrent
+- long-context
+- research
+datasets:
+- HuggingFaceFW/fineweb-edu
+- cerebras/SlimPajama-627B
+---
+
+# GDN1 16K/32K Bridge 1B Full Fine-Tune
+
+This is a research checkpoint from the Long-GDN workspace.
+
+## Base Model
+
+- Base checkpoint: `linear-moe-hub/Gated-Deltanet-1.3B`
+- Architecture: Gated DeltaNet / linear recurrent attention
+- Base training data reported by the upstream model card: SlimPajama 100B-token sample
+- License inherited from upstream model card: Apache-2.0
+
+## Training Run
+
+- Local source path: `runs/gdn1_16k32k_bridge_from_balanced200_1b_bs10_ft/final`
+- Tokenizer source: `runs/gdn1_16k32k_bridge_from_balanced200_1b_bs10_ft/final`
+- Training mode: full fine-tuning, no LoRA/adapter
+- Hardware target: 8x NVIDIA H200
+- Sequence length: 32768
+- Approximate additional token budget: ~1B additional tokens
+- Manifest/config: `configs/gdn1_memory_mix_16k32k_bridge_recovery.json`
+
+## Intended Research Use
+
+This checkpoint is intended for research on:
+
+- long-context associative recall
+- RULER/MQAR-style state tracking
+- recurrent-state contamination during long generation
+- Reference-State Reset with Rolling Replay, a GDN/RNN adaptation of the R-SWA idea
+
+## Known Results
+
+Bridge continuation from balanced checkpoint-200. It improved some mid-length MQAR bands in checkpoint sweeps but reintroduced 32K instability; not selected as the best branch point.
+
+## Caveats
+
+Not the current best checkpoint. Use as an ablation checkpoint for 16K/32K bridge curriculum analysis.
+
+## Citation Context
+
+Relevant background papers include Gated Delta Networks, Gated DeltaNet-2,
+Log-Linear Attention, and Unlimited OCR / R-SWA. This checkpoint does not
+implement a new architecture by itself; it is part of a checkpoint-preserving
+full fine-tuning and inference-control study.

config.json

@@ -0,0 +1,38 @@
+{
+  "allow_neg_eigval": false,
+  "architectures": [
+    "GatedDeltaNetForCausalLM"
+  ],
+  "attn": null,
+  "attn_mode": "chunk",
+  "bos_token_id": 1,
+  "conv_size": 4,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "expand_v": 1,
+  "fuse_cross_entropy": true,
+  "fuse_linear_cross_entropy": false,
+  "fuse_norm": true,
+  "fuse_swiglu": true,
+  "head_dim": 256,
+  "hidden_act": "swish",
+  "hidden_ratio": 4,
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": null,
+  "max_position_embeddings": 2048,
+  "model_type": "gated_deltanet",
+  "norm_eps": 1e-06,
+  "norm_first": false,
+  "num_heads": 4,
+  "num_hidden_layers": 24,
+  "num_v_heads": null,
+  "pad_token_id": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.5.4",
+  "use_cache": false,
+  "use_gate": true,
+  "use_l2warp": false,
+  "use_short_conv": false,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.5.4",
+  "use_cache": false
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f948011d5b3f338003c62fe9550bc3fb26c53bb8dc9a2c299197a4300b1f3f86
+size 2296371864

train_plan.json

@@ -0,0 +1,27 @@
+{
+  "mode": "full_finetune",
+  "target_hardware": "8x NVIDIA H200",
+  "model_path": "/home/work/.projects/LLM-OS-Models/long-gdn/runs/gdn1_32k_balanced_recovery_1b_bs10_ft/checkpoint-200",
+  "tokenizer_path": "/home/work/.projects/LLM-OS-Models/long-gdn/runs/gdn1_32k_balanced_recovery_1b_bs10_ft/final",
+  "manifest": "/home/work/.projects/LLM-OS-Models/long-gdn/configs/gdn1_memory_mix_16k32k_bridge_recovery.json",
+  "output_dir": "/home/work/.projects/LLM-OS-Models/long-gdn/runs/gdn1_16k32k_bridge_from_balanced200_1b_bs10_ft",
+  "seq_len": 32768,
+  "per_device_batch_size": 10,
+  "grad_accum": 1,
+  "world_size_assumed": 8,
+  "global_tokens_per_step": 2621440,
+  "max_steps": 382,
+  "target_tokens": 1001390080,
+  "learning_rate": 5e-06,
+  "warmup_steps": 25,
+  "weight_decay": 0.1,
+  "optim": "adamw_torch_fused",
+  "bf16": true,
+  "gradient_checkpointing": true,
+  "override_max_position_embeddings": null,
+  "notes": [
+    "No LoRA/PEFT adapters are used.",
+    "All model parameters remain trainable unless the model implementation freezes them.",
+    "Dry-run writes this plan without loading model weights."
+  ]
+}