Hugging Face's logo

RMDWLLC
/
Jah-6.0

Text Generation
Safetensors
English
deepseek_v41
rmdw
jah
uncensored
fp8
long-context
vision
tool-calling
8-bit precision
Model card Files
xet
 Community
restokes92 commited on
Commit
36ad9d6
·
verified ·
1 Parent(s): 28b56e2

Upload config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. config.json +170 -0
config.json ADDED
@@ -0,0 +1,170 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "DeepseekV41ForCausalLM"
4
+ ],
5
+ "model_type": "deepseek_v41",
6
+ "dtype": "bfloat16",
7
+ "transformers_version": "5.6.0",
8
+ "bos_token_id": 0,
9
+ "eos_token_id": 1,
10
+ "pad_token_id": 2,
11
+ "image_token_id": 129264,
12
+ "quantization_config": {
13
+ "quant_method": "fp8",
14
+ "activation_scheme": "dynamic",
15
+ "weight_block_size": [
16
+ 32,
17
+ 32
18
+ ],
19
+ "scale_fmt": "ue8m0",
20
+ "expert_dtype": "fp4"
21
+ },
22
+ "text_config": {
23
+ "model_type": "deepseek_v41_text",
24
+ "vocab_size": 129280,
25
+ "hidden_size": 5120,
26
+ "moe_intermediate_size": 2304,
27
+ "num_hidden_layers": 40,
28
+ "num_attention_heads": 64,
29
+ "num_key_value_heads": 1,
30
+ "head_dim": 512,
31
+ "qk_rope_head_dim": 64,
32
+ "q_lora_rank": 1280,
33
+ "o_lora_rank": 1024,
34
+ "o_groups": 8,
35
+ "hidden_act": "silu",
36
+ "swiglu_limit": 10.0,
37
+ "rms_norm_eps": 1e-20,
38
+ "attention_bias": false,
39
+ "attention_dropout": 0.0,
40
+ "initializer_range": 0.02,
41
+ "use_cache": true,
42
+ "tie_word_embeddings": false,
43
+ "max_position_embeddings": 1048576,
44
+ "rope_theta": 10000,
45
+ "rope_scaling": {
46
+ "rope_type": "yarn",
47
+ "factor": 16,
48
+ "beta_fast": 32,
49
+ "beta_slow": 1,
50
+ "original_max_position_embeddings": 65536
51
+ },
52
+ "n_routed_experts": 384,
53
+ "n_shared_experts": 1,
54
+ "num_experts_per_tok": 6,
55
+ "scoring_func": "sqrtsoftplus",
56
+ "topk_method": "noaux_tc",
57
+ "norm_topk_prob": true,
58
+ "routed_scaling_factor": 1.5,
59
+ "sliding_window": 128,
60
+ "compress_ratios": [
61
+ 0,
62
+ 0,
63
+ 2,
64
+ 2,
65
+ 2,
66
+ 2,
67
+ 2,
68
+ 2,
69
+ 2,
70
+ 2,
71
+ 2,
72
+ 2,
73
+ 2,
74
+ 2,
75
+ 2,
76
+ 2,
77
+ 2,
78
+ 2,
79
+ 2,
80
+ 2,
81
+ 1,
82
+ 1,
83
+ 1,
84
+ 1,
85
+ 1,
86
+ 1,
87
+ 1,
88
+ 1,
89
+ 1,
90
+ 1,
91
+ 1,
92
+ 1,
93
+ 1,
94
+ 1,
95
+ 1,
96
+ 1,
97
+ 1,
98
+ 1,
99
+ 1,
100
+ 1,
101
+ 0,
102
+ 0,
103
+ 0
104
+ ],
105
+ "compress_rope_theta": 160000,
106
+ "kv_source_layer_ids": [
107
+ 2,
108
+ 8,
109
+ 14,
110
+ 20
111
+ ],
112
+ "index_source_layer_ids": [
113
+ 2,
114
+ 8,
115
+ 14,
116
+ 20,
117
+ 24,
118
+ 28,
119
+ 32,
120
+ 36
121
+ ],
122
+ "index_n_heads": 32,
123
+ "index_head_dim": 128,
124
+ "index_topk": 512,
125
+ "candidate_source_layer_id": 20,
126
+ "candidate_topk_blocks": 2048,
127
+ "candidate_block_size": 8,
128
+ "hc_mult": 4,
129
+ "hc_sinkhorn_iters": 20,
130
+ "hc_eps": 1e-06,
131
+ "engram_layer_ids": [
132
+ 1,
133
+ 14
134
+ ],
135
+ "engram_num_embeddings": [
136
+ 384006168,
137
+ 384016682
138
+ ],
139
+ "engram_max_ngram_size": 4,
140
+ "engram_vocab_size": 16000000,
141
+ "engram_n_heads": 8,
142
+ "engram_head_dim": 256,
143
+ "engram_pad_token_id": 2,
144
+ "engram_compressed_vocab_size": 99092,
145
+ "num_nextn_predict_layers": 3,
146
+ "dspark_block_size": 5,
147
+ "dspark_noise_token_id": 128799,
148
+ "dspark_target_layer_ids": [
149
+ 37,
150
+ 38,
151
+ 39
152
+ ],
153
+ "dspark_markov_rank": 256,
154
+ "dspark_n_routed_experts": 128,
155
+ "dspark_num_experts_per_tok": 3
156
+ },
157
+ "vision_config": {
158
+ "model_type": "deepseek_v41_vision",
159
+ "num_hidden_layers": 32,
160
+ "hidden_size": 1024,
161
+ "num_attention_heads": 16,
162
+ "intermediate_size": 2816,
163
+ "patch_size": 14,
164
+ "rope_theta": 10000,
165
+ "downsample_ratio": 3,
166
+ "max_image_tokens": 1024,
167
+ "min_pixels": 295936,
168
+ "max_wh_ratio": null
169
+ }
170
+ }