{ "model_id": "gpt2", "family": "gpt2", "hidden_dim": 768, "num_layers": 12, "num_heads": 12, "vocab_size": 50257, "max_seq_len": 1024, "parts": { "embedding": { "part_type": "embedding", "layer_idx": null, "module_path": "transformer.wte", "input_dim": 50257, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "pos_embedding": { "part_type": "embedding", "layer_idx": null, "module_path": "transformer.wpe", "input_dim": 1024, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "attention_0": { "part_type": "attention", "layer_idx": 0, "module_path": "transformer.h.0.attn", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": 64, "intermediate_dim": null, "extra_info": {} }, "ffn_0": { "part_type": "ffn", "layer_idx": 0, "module_path": "transformer.h.0.mlp", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "ln_1_0": { "part_type": "layer_norm", "layer_idx": 0, "module_path": "transformer.h.0.ln_1", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "ln_2_0": { "part_type": "layer_norm", "layer_idx": 0, "module_path": "transformer.h.0.ln_2", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "layer_0": { "part_type": "full_layer", "layer_idx": 0, "module_path": "transformer.h.0", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "attention_1": { "part_type": "attention", "layer_idx": 1, "module_path": "transformer.h.1.attn", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": 64, "intermediate_dim": null, "extra_info": {} }, "ffn_1": { "part_type": "ffn", "layer_idx": 1, "module_path": "transformer.h.1.mlp", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "ln_1_1": { "part_type": "layer_norm", "layer_idx": 1, "module_path": "transformer.h.1.ln_1", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "ln_2_1": { "part_type": "layer_norm", "layer_idx": 1, "module_path": "transformer.h.1.ln_2", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "layer_1": { "part_type": "full_layer", "layer_idx": 1, "module_path": "transformer.h.1", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "attention_2": { "part_type": "attention", "layer_idx": 2, "module_path": "transformer.h.2.attn", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": 64, "intermediate_dim": null, "extra_info": {} }, "ffn_2": { "part_type": "ffn", "layer_idx": 2, "module_path": "transformer.h.2.mlp", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "ln_1_2": { "part_type": "layer_norm", "layer_idx": 2, "module_path": "transformer.h.2.ln_1", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "ln_2_2": { "part_type": "layer_norm", "layer_idx": 2, "module_path": "transformer.h.2.ln_2", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "layer_2": { "part_type": "full_layer", "layer_idx": 2, "module_path": "transformer.h.2", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "attention_3": { "part_type": "attention", "layer_idx": 3, "module_path": "transformer.h.3.attn", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": 64, "intermediate_dim": null, "extra_info": {} }, "ffn_3": { "part_type": "ffn", "layer_idx": 3, "module_path": "transformer.h.3.mlp", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "ln_1_3": { "part_type": "layer_norm", "layer_idx": 3, "module_path": "transformer.h.3.ln_1", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "ln_2_3": { "part_type": "layer_norm", "layer_idx": 3, "module_path": "transformer.h.3.ln_2", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "layer_3": { "part_type": "full_layer", "layer_idx": 3, "module_path": "transformer.h.3", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "attention_4": { "part_type": "attention", "layer_idx": 4, "module_path": "transformer.h.4.attn", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": 64, "intermediate_dim": null, "extra_info": {} }, "ffn_4": { "part_type": "ffn", "layer_idx": 4, "module_path": "transformer.h.4.mlp", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "ln_1_4": { "part_type": "layer_norm", "layer_idx": 4, "module_path": "transformer.h.4.ln_1", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "ln_2_4": { "part_type": "layer_norm", "layer_idx": 4, "module_path": "transformer.h.4.ln_2", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "layer_4": { "part_type": "full_layer", "layer_idx": 4, "module_path": "transformer.h.4", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "attention_5": { "part_type": "attention", "layer_idx": 5, "module_path": "transformer.h.5.attn", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": 64, "intermediate_dim": null, "extra_info": {} }, "ffn_5": { "part_type": "ffn", "layer_idx": 5, "module_path": "transformer.h.5.mlp", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "ln_1_5": { "part_type": "layer_norm", "layer_idx": 5, "module_path": "transformer.h.5.ln_1", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "ln_2_5": { "part_type": "layer_norm", "layer_idx": 5, "module_path": "transformer.h.5.ln_2", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "layer_5": { "part_type": "full_layer", "layer_idx": 5, "module_path": "transformer.h.5", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "attention_6": { "part_type": "attention", "layer_idx": 6, "module_path": "transformer.h.6.attn", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": 64, "intermediate_dim": null, "extra_info": {} }, "ffn_6": { "part_type": "ffn", "layer_idx": 6, "module_path": "transformer.h.6.mlp", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "ln_1_6": { "part_type": "layer_norm", "layer_idx": 6, "module_path": "transformer.h.6.ln_1", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "ln_2_6": { "part_type": "layer_norm", "layer_idx": 6, "module_path": "transformer.h.6.ln_2", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "layer_6": { "part_type": "full_layer", "layer_idx": 6, "module_path": "transformer.h.6", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "attention_7": { "part_type": "attention", "layer_idx": 7, "module_path": "transformer.h.7.attn", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": 64, "intermediate_dim": null, "extra_info": {} }, "ffn_7": { "part_type": "ffn", "layer_idx": 7, "module_path": "transformer.h.7.mlp", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "ln_1_7": { "part_type": "layer_norm", "layer_idx": 7, "module_path": "transformer.h.7.ln_1", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "ln_2_7": { "part_type": "layer_norm", "layer_idx": 7, "module_path": "transformer.h.7.ln_2", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "layer_7": { "part_type": "full_layer", "layer_idx": 7, "module_path": "transformer.h.7", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "attention_8": { "part_type": "attention", "layer_idx": 8, "module_path": "transformer.h.8.attn", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": 64, "intermediate_dim": null, "extra_info": {} }, "ffn_8": { "part_type": "ffn", "layer_idx": 8, "module_path": "transformer.h.8.mlp", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "ln_1_8": { "part_type": "layer_norm", "layer_idx": 8, "module_path": "transformer.h.8.ln_1", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "ln_2_8": { "part_type": "layer_norm", "layer_idx": 8, "module_path": "transformer.h.8.ln_2", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "layer_8": { "part_type": "full_layer", "layer_idx": 8, "module_path": "transformer.h.8", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "attention_9": { "part_type": "attention", "layer_idx": 9, "module_path": "transformer.h.9.attn", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": 64, "intermediate_dim": null, "extra_info": {} }, "ffn_9": { "part_type": "ffn", "layer_idx": 9, "module_path": "transformer.h.9.mlp", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "ln_1_9": { "part_type": "layer_norm", "layer_idx": 9, "module_path": "transformer.h.9.ln_1", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "ln_2_9": { "part_type": "layer_norm", "layer_idx": 9, "module_path": "transformer.h.9.ln_2", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "layer_9": { "part_type": "full_layer", "layer_idx": 9, "module_path": "transformer.h.9", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "attention_10": { "part_type": "attention", "layer_idx": 10, "module_path": "transformer.h.10.attn", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": 64, "intermediate_dim": null, "extra_info": {} }, "ffn_10": { "part_type": "ffn", "layer_idx": 10, "module_path": "transformer.h.10.mlp", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "ln_1_10": { "part_type": "layer_norm", "layer_idx": 10, "module_path": "transformer.h.10.ln_1", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "ln_2_10": { "part_type": "layer_norm", "layer_idx": 10, "module_path": "transformer.h.10.ln_2", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "layer_10": { "part_type": "full_layer", "layer_idx": 10, "module_path": "transformer.h.10", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "attention_11": { "part_type": "attention", "layer_idx": 11, "module_path": "transformer.h.11.attn", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": 64, "intermediate_dim": null, "extra_info": {} }, "ffn_11": { "part_type": "ffn", "layer_idx": 11, "module_path": "transformer.h.11.mlp", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "ln_1_11": { "part_type": "layer_norm", "layer_idx": 11, "module_path": "transformer.h.11.ln_1", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "ln_2_11": { "part_type": "layer_norm", "layer_idx": 11, "module_path": "transformer.h.11.ln_2", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "layer_11": { "part_type": "full_layer", "layer_idx": 11, "module_path": "transformer.h.11", "input_dim": 768, "output_dim": 768, "num_heads": 12, "head_dim": null, "intermediate_dim": 3072, "extra_info": {} }, "ln_f": { "part_type": "layer_norm", "layer_idx": null, "module_path": "transformer.ln_f", "input_dim": 768, "output_dim": 768, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} }, "output_head": { "part_type": "output_head", "layer_idx": null, "module_path": "lm_head", "input_dim": 768, "output_dim": 50257, "num_heads": null, "head_dim": null, "intermediate_dim": null, "extra_info": {} } } }