| { |
| "model_id": "gpt2", |
| "family": "gpt2", |
| "hidden_dim": 768, |
| "num_layers": 12, |
| "num_heads": 12, |
| "vocab_size": 50257, |
| "max_seq_len": 1024, |
| "parts": { |
| "embedding": { |
| "part_type": "embedding", |
| "layer_idx": null, |
| "module_path": "transformer.wte", |
| "input_dim": 50257, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "pos_embedding": { |
| "part_type": "embedding", |
| "layer_idx": null, |
| "module_path": "transformer.wpe", |
| "input_dim": 1024, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "attention_0": { |
| "part_type": "attention", |
| "layer_idx": 0, |
| "module_path": "transformer.h.0.attn", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": 64, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ffn_0": { |
| "part_type": "ffn", |
| "layer_idx": 0, |
| "module_path": "transformer.h.0.mlp", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "ln_1_0": { |
| "part_type": "layer_norm", |
| "layer_idx": 0, |
| "module_path": "transformer.h.0.ln_1", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ln_2_0": { |
| "part_type": "layer_norm", |
| "layer_idx": 0, |
| "module_path": "transformer.h.0.ln_2", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "layer_0": { |
| "part_type": "full_layer", |
| "layer_idx": 0, |
| "module_path": "transformer.h.0", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "attention_1": { |
| "part_type": "attention", |
| "layer_idx": 1, |
| "module_path": "transformer.h.1.attn", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": 64, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ffn_1": { |
| "part_type": "ffn", |
| "layer_idx": 1, |
| "module_path": "transformer.h.1.mlp", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "ln_1_1": { |
| "part_type": "layer_norm", |
| "layer_idx": 1, |
| "module_path": "transformer.h.1.ln_1", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ln_2_1": { |
| "part_type": "layer_norm", |
| "layer_idx": 1, |
| "module_path": "transformer.h.1.ln_2", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "layer_1": { |
| "part_type": "full_layer", |
| "layer_idx": 1, |
| "module_path": "transformer.h.1", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "attention_2": { |
| "part_type": "attention", |
| "layer_idx": 2, |
| "module_path": "transformer.h.2.attn", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": 64, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ffn_2": { |
| "part_type": "ffn", |
| "layer_idx": 2, |
| "module_path": "transformer.h.2.mlp", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "ln_1_2": { |
| "part_type": "layer_norm", |
| "layer_idx": 2, |
| "module_path": "transformer.h.2.ln_1", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ln_2_2": { |
| "part_type": "layer_norm", |
| "layer_idx": 2, |
| "module_path": "transformer.h.2.ln_2", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "layer_2": { |
| "part_type": "full_layer", |
| "layer_idx": 2, |
| "module_path": "transformer.h.2", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "attention_3": { |
| "part_type": "attention", |
| "layer_idx": 3, |
| "module_path": "transformer.h.3.attn", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": 64, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ffn_3": { |
| "part_type": "ffn", |
| "layer_idx": 3, |
| "module_path": "transformer.h.3.mlp", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "ln_1_3": { |
| "part_type": "layer_norm", |
| "layer_idx": 3, |
| "module_path": "transformer.h.3.ln_1", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ln_2_3": { |
| "part_type": "layer_norm", |
| "layer_idx": 3, |
| "module_path": "transformer.h.3.ln_2", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "layer_3": { |
| "part_type": "full_layer", |
| "layer_idx": 3, |
| "module_path": "transformer.h.3", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "attention_4": { |
| "part_type": "attention", |
| "layer_idx": 4, |
| "module_path": "transformer.h.4.attn", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": 64, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ffn_4": { |
| "part_type": "ffn", |
| "layer_idx": 4, |
| "module_path": "transformer.h.4.mlp", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "ln_1_4": { |
| "part_type": "layer_norm", |
| "layer_idx": 4, |
| "module_path": "transformer.h.4.ln_1", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ln_2_4": { |
| "part_type": "layer_norm", |
| "layer_idx": 4, |
| "module_path": "transformer.h.4.ln_2", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "layer_4": { |
| "part_type": "full_layer", |
| "layer_idx": 4, |
| "module_path": "transformer.h.4", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "attention_5": { |
| "part_type": "attention", |
| "layer_idx": 5, |
| "module_path": "transformer.h.5.attn", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": 64, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ffn_5": { |
| "part_type": "ffn", |
| "layer_idx": 5, |
| "module_path": "transformer.h.5.mlp", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "ln_1_5": { |
| "part_type": "layer_norm", |
| "layer_idx": 5, |
| "module_path": "transformer.h.5.ln_1", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ln_2_5": { |
| "part_type": "layer_norm", |
| "layer_idx": 5, |
| "module_path": "transformer.h.5.ln_2", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "layer_5": { |
| "part_type": "full_layer", |
| "layer_idx": 5, |
| "module_path": "transformer.h.5", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "attention_6": { |
| "part_type": "attention", |
| "layer_idx": 6, |
| "module_path": "transformer.h.6.attn", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": 64, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ffn_6": { |
| "part_type": "ffn", |
| "layer_idx": 6, |
| "module_path": "transformer.h.6.mlp", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "ln_1_6": { |
| "part_type": "layer_norm", |
| "layer_idx": 6, |
| "module_path": "transformer.h.6.ln_1", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ln_2_6": { |
| "part_type": "layer_norm", |
| "layer_idx": 6, |
| "module_path": "transformer.h.6.ln_2", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "layer_6": { |
| "part_type": "full_layer", |
| "layer_idx": 6, |
| "module_path": "transformer.h.6", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "attention_7": { |
| "part_type": "attention", |
| "layer_idx": 7, |
| "module_path": "transformer.h.7.attn", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": 64, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ffn_7": { |
| "part_type": "ffn", |
| "layer_idx": 7, |
| "module_path": "transformer.h.7.mlp", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "ln_1_7": { |
| "part_type": "layer_norm", |
| "layer_idx": 7, |
| "module_path": "transformer.h.7.ln_1", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ln_2_7": { |
| "part_type": "layer_norm", |
| "layer_idx": 7, |
| "module_path": "transformer.h.7.ln_2", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "layer_7": { |
| "part_type": "full_layer", |
| "layer_idx": 7, |
| "module_path": "transformer.h.7", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "attention_8": { |
| "part_type": "attention", |
| "layer_idx": 8, |
| "module_path": "transformer.h.8.attn", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": 64, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ffn_8": { |
| "part_type": "ffn", |
| "layer_idx": 8, |
| "module_path": "transformer.h.8.mlp", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "ln_1_8": { |
| "part_type": "layer_norm", |
| "layer_idx": 8, |
| "module_path": "transformer.h.8.ln_1", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ln_2_8": { |
| "part_type": "layer_norm", |
| "layer_idx": 8, |
| "module_path": "transformer.h.8.ln_2", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "layer_8": { |
| "part_type": "full_layer", |
| "layer_idx": 8, |
| "module_path": "transformer.h.8", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "attention_9": { |
| "part_type": "attention", |
| "layer_idx": 9, |
| "module_path": "transformer.h.9.attn", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": 64, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ffn_9": { |
| "part_type": "ffn", |
| "layer_idx": 9, |
| "module_path": "transformer.h.9.mlp", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "ln_1_9": { |
| "part_type": "layer_norm", |
| "layer_idx": 9, |
| "module_path": "transformer.h.9.ln_1", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ln_2_9": { |
| "part_type": "layer_norm", |
| "layer_idx": 9, |
| "module_path": "transformer.h.9.ln_2", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "layer_9": { |
| "part_type": "full_layer", |
| "layer_idx": 9, |
| "module_path": "transformer.h.9", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "attention_10": { |
| "part_type": "attention", |
| "layer_idx": 10, |
| "module_path": "transformer.h.10.attn", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": 64, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ffn_10": { |
| "part_type": "ffn", |
| "layer_idx": 10, |
| "module_path": "transformer.h.10.mlp", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "ln_1_10": { |
| "part_type": "layer_norm", |
| "layer_idx": 10, |
| "module_path": "transformer.h.10.ln_1", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ln_2_10": { |
| "part_type": "layer_norm", |
| "layer_idx": 10, |
| "module_path": "transformer.h.10.ln_2", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "layer_10": { |
| "part_type": "full_layer", |
| "layer_idx": 10, |
| "module_path": "transformer.h.10", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "attention_11": { |
| "part_type": "attention", |
| "layer_idx": 11, |
| "module_path": "transformer.h.11.attn", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": 64, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ffn_11": { |
| "part_type": "ffn", |
| "layer_idx": 11, |
| "module_path": "transformer.h.11.mlp", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "ln_1_11": { |
| "part_type": "layer_norm", |
| "layer_idx": 11, |
| "module_path": "transformer.h.11.ln_1", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "ln_2_11": { |
| "part_type": "layer_norm", |
| "layer_idx": 11, |
| "module_path": "transformer.h.11.ln_2", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "layer_11": { |
| "part_type": "full_layer", |
| "layer_idx": 11, |
| "module_path": "transformer.h.11", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": 12, |
| "head_dim": null, |
| "intermediate_dim": 3072, |
| "extra_info": {} |
| }, |
| "ln_f": { |
| "part_type": "layer_norm", |
| "layer_idx": null, |
| "module_path": "transformer.ln_f", |
| "input_dim": 768, |
| "output_dim": 768, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| }, |
| "output_head": { |
| "part_type": "output_head", |
| "layer_idx": null, |
| "module_path": "lm_head", |
| "input_dim": 768, |
| "output_dim": 50257, |
| "num_heads": null, |
| "head_dim": null, |
| "intermediate_dim": null, |
| "extra_info": {} |
| } |
| } |
| } |
