File size: 3,523 Bytes
56ba74f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 | {
"auto_map": {
"AutoConfig": "config_molmoe.MolmoConfig",
"AutoModelForCausalLM": "modeling_molmoe.MolmoForCausalLM"
},
"activation_type": "swiglu",
"additional_vocab_size": 128,
"alibi": false,
"alibi_bias_max": 8.0,
"always_start_with_space": true,
"architectures": [
"OLMoForCausalLM"
],
"attention_dropout": 0.0,
"attention_layer_norm": true,
"attention_layer_norm_with_affine": true,
"attention_type": "sdpa",
"attn_logit_softcapping": null,
"bias_for_layer_norm": false,
"block_group_size": 1,
"block_type": "moe",
"clip_qkv": null,
"crop_mode": "overlap-and-resize-c2",
"d_model": 2048,
"default_inference_len": 65,
"do_random_scale": false,
"embedding_dropout": 0.0,
"embedding_size": 50304,
"final_logit_softcapping": null,
"fix_image_input_idx": 2,
"float32_attention": true,
"gin_bindings": null,
"head_dim": null,
"image_feature_dropout": 0.0,
"image_padding_embed": "pad_and_partial_pad",
"image_pooling_2d": "attention-meanq",
"image_pooling_h": 2,
"image_pooling_w": 2,
"image_projector": "mlp",
"include_bias": false,
"init_cutoff_factor": 3.0,
"init_device": "meta",
"init_fn": "normal",
"init_std": 0.02,
"initializer_range": 0.02,
"layer_norm_eps": 1e-05,
"layer_norm_type": "rms",
"layer_norm_with_affine": true,
"llm_load_path": null,
"loss_token_weighting": "root_subsegments",
"low_cpu_fsdp": true,
"max_crops": 12,
"max_position_embeddings": 32768,
"max_sequence_length": 4096,
"message_formatting": "role",
"mlp_hidden_size": null,
"mlp_ratio": 1,
"model_type": "molmo",
"moe_capacity_factor": 1.25,
"moe_dropless": true,
"moe_interleave": false,
"moe_lbl_in_fp32": false,
"moe_log_expert_assignment": false,
"moe_loss_weight": 0.0,
"moe_mlp_impl": "sparse",
"moe_num_experts": 64,
"moe_shared_expert": false,
"moe_top_k": 8,
"moe_zloss_weight": 0.0,
"multi_query_attention": null,
"n_heads": 16,
"n_kv_heads": null,
"n_layers": 16,
"new_embedding_init_range": 0.02,
"norm_after": false,
"normalize_input_embeds": false,
"overlap_margins": [
4,
4
],
"pad_to": null,
"pad_token_id": 1,
"pad_tokenizer": false,
"precision": "amp_bf16",
"prompt_override": null,
"prompt_type": "uber_model",
"qkv_bias": false,
"query_pre_attn_scalar": 224,
"residual_dropout": 0.1,
"response_attention_dropout": 0.0,
"response_residual_dropout": 0.0,
"rope": true,
"rope_full_precision": true,
"rope_impl": "llama",
"rope_theta": 10000.0,
"scale_logits": false,
"system_prompt_kind": "demo_or_style",
"transformers_version": "4.45.0.dev0",
"unconditioned": false,
"use_cache": true,
"use_cls_feature": false,
"use_col_tokens": true,
"use_position_ids": true,
"vision_backbone": {
"attention_dropout": 0.0,
"fsdp_wrap": false,
"image_default_input_size": [
336,
336
],
"image_dropout_rate": 0.0,
"image_emb_dim": 1024,
"image_head_dim": 64,
"image_mlp_activations": "quick_gelu",
"image_mlp_dim": 4096,
"image_model_type": "openai",
"image_norm_eps": 1e-05,
"image_num_heads": 16,
"image_num_key_value_heads": 16,
"image_num_layers": 23,
"image_num_pos": 577,
"image_patch_size": 14,
"image_pos_patch_size": 14,
"initializer_range": 0.02,
"residual_dropout": 0.0,
"resize_mode": "default"
},
"vit_layers": [
-2,
-9
],
"vit_load_path": null,
"vocab_size": 50280,
"weight_tying": false
}
|