Hugging Face's logo

Phonsiri
/
Thai-Legal-Gemma-4B-CPT

Text Generation
Transformers
TensorBoard
Safetensors
Thai
English
gemma4
image-text-to-text
legal
thai
continued-pretraining
cpt
law
gemma
causal-lm
Model card Files
xet
 Metrics Community
Phonsiri commited on
Commit
9ea0bb6
·
verified ·
1 Parent(s): 1cbfede

Training in progress, step 20, checkpoint

Browse files
Files changed (10) hide show
  1. checkpoint-20/config.json +198 -0
  2. checkpoint-20/generation_config.json +13 -0
  3. checkpoint-20/model.safetensors +3 -0
  4. checkpoint-20/optimizer.pt +3 -0
  5. checkpoint-20/rng_state.pth +3 -0
  6. checkpoint-20/scheduler.pt +3 -0
  7. checkpoint-20/tokenizer.json +3 -0
  8. checkpoint-20/tokenizer_config.json +55 -0
  9. checkpoint-20/trainer_state.json +174 -0
  10. checkpoint-20/training_args.bin +3 -0
checkpoint-20/config.json ADDED
@@ -0,0 +1,198 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Gemma4ForConditionalGeneration"
4
+ ],
5
+ "audio_config": {
6
+ "_name_or_path": "",
7
+ "architectures": null,
8
+ "attention_chunk_size": 12,
9
+ "attention_context_left": 13,
10
+ "attention_context_right": 0,
11
+ "attention_invalid_logits_value": -1000000000.0,
12
+ "attention_logit_cap": 50.0,
13
+ "chunk_size_feed_forward": 0,
14
+ "conv_kernel_size": 5,
15
+ "dtype": "bfloat16",
16
+ "gradient_clipping": 10000000000.0,
17
+ "hidden_act": "silu",
18
+ "hidden_size": 1024,
19
+ "id2label": {
20
+ "0": "LABEL_0",
21
+ "1": "LABEL_1"
22
+ },
23
+ "initializer_range": 0.02,
24
+ "is_encoder_decoder": false,
25
+ "label2id": {
26
+ "LABEL_0": 0,
27
+ "LABEL_1": 1
28
+ },
29
+ "model_type": "gemma4_audio",
30
+ "num_attention_heads": 8,
31
+ "num_hidden_layers": 12,
32
+ "output_attentions": false,
33
+ "output_hidden_states": false,
34
+ "output_proj_dims": 1536,
35
+ "problem_type": null,
36
+ "residual_weight": 0.5,
37
+ "return_dict": true,
38
+ "rms_norm_eps": 1e-06,
39
+ "subsampling_conv_channels": [
40
+ 128,
41
+ 32
42
+ ],
43
+ "use_clipped_linears": true
44
+ },
45
+ "audio_token_id": 258881,
46
+ "boa_token_id": 256000,
47
+ "boi_token_id": 255999,
48
+ "bos_token_id": 2,
49
+ "dtype": "bfloat16",
50
+ "eoa_token_id": 258883,
51
+ "eoa_token_index": 258883,
52
+ "eoi_token_id": 258882,
53
+ "eos_token_id": 1,
54
+ "image_token_id": 258880,
55
+ "initializer_range": 0.02,
56
+ "model_type": "gemma4",
57
+ "pad_token_id": 0,
58
+ "text_config": {
59
+ "attention_bias": false,
60
+ "attention_dropout": 0.0,
61
+ "attention_k_eq_v": false,
62
+ "bos_token_id": 2,
63
+ "dtype": "bfloat16",
64
+ "enable_moe_block": false,
65
+ "eos_token_id": 1,
66
+ "expert_intermediate_size": null,
67
+ "final_logit_softcapping": 30.0,
68
+ "global_head_dim": 512,
69
+ "head_dim": 256,
70
+ "hidden_activation": "gelu_pytorch_tanh",
71
+ "hidden_size": 2560,
72
+ "hidden_size_per_layer_input": 256,
73
+ "initializer_range": 0.02,
74
+ "intermediate_size": 10240,
75
+ "layer_types": [
76
+ "sliding_attention",
77
+ "sliding_attention",
78
+ "sliding_attention",
79
+ "sliding_attention",
80
+ "sliding_attention",
81
+ "full_attention",
82
+ "sliding_attention",
83
+ "sliding_attention",
84
+ "sliding_attention",
85
+ "sliding_attention",
86
+ "sliding_attention",
87
+ "full_attention",
88
+ "sliding_attention",
89
+ "sliding_attention",
90
+ "sliding_attention",
91
+ "sliding_attention",
92
+ "sliding_attention",
93
+ "full_attention",
94
+ "sliding_attention",
95
+ "sliding_attention",
96
+ "sliding_attention",
97
+ "sliding_attention",
98
+ "sliding_attention",
99
+ "full_attention",
100
+ "sliding_attention",
101
+ "sliding_attention",
102
+ "sliding_attention",
103
+ "sliding_attention",
104
+ "sliding_attention",
105
+ "full_attention",
106
+ "sliding_attention",
107
+ "sliding_attention",
108
+ "sliding_attention",
109
+ "sliding_attention",
110
+ "sliding_attention",
111
+ "full_attention",
112
+ "sliding_attention",
113
+ "sliding_attention",
114
+ "sliding_attention",
115
+ "sliding_attention",
116
+ "sliding_attention",
117
+ "full_attention"
118
+ ],
119
+ "max_position_embeddings": 131072,
120
+ "model_type": "gemma4_text",
121
+ "moe_intermediate_size": null,
122
+ "num_attention_heads": 8,
123
+ "num_experts": null,
124
+ "num_global_key_value_heads": null,
125
+ "num_hidden_layers": 42,
126
+ "num_key_value_heads": 2,
127
+ "num_kv_shared_layers": 18,
128
+ "pad_token_id": 0,
129
+ "rms_norm_eps": 1e-06,
130
+ "rope_parameters": {
131
+ "full_attention": {
132
+ "partial_rotary_factor": 0.25,
133
+ "rope_theta": 1000000.0,
134
+ "rope_type": "proportional"
135
+ },
136
+ "sliding_attention": {
137
+ "rope_theta": 10000.0,
138
+ "rope_type": "default"
139
+ }
140
+ },
141
+ "sliding_window": 512,
142
+ "tie_word_embeddings": true,
143
+ "top_k_experts": null,
144
+ "use_bidirectional_attention": null,
145
+ "use_cache": true,
146
+ "use_double_wide_mlp": false,
147
+ "vocab_size": 262144,
148
+ "vocab_size_per_layer_input": 262144
149
+ },
150
+ "tie_word_embeddings": true,
151
+ "transformers_version": "5.7.0",
152
+ "use_cache": false,
153
+ "video_token_id": 258884,
154
+ "vision_config": {
155
+ "_name_or_path": "",
156
+ "architectures": null,
157
+ "attention_bias": false,
158
+ "attention_dropout": 0.0,
159
+ "chunk_size_feed_forward": 0,
160
+ "default_output_length": 280,
161
+ "dtype": "bfloat16",
162
+ "global_head_dim": 64,
163
+ "head_dim": 64,
164
+ "hidden_activation": "gelu_pytorch_tanh",
165
+ "hidden_size": 768,
166
+ "id2label": {
167
+ "0": "LABEL_0",
168
+ "1": "LABEL_1"
169
+ },
170
+ "initializer_range": 0.02,
171
+ "intermediate_size": 3072,
172
+ "is_encoder_decoder": false,
173
+ "label2id": {
174
+ "LABEL_0": 0,
175
+ "LABEL_1": 1
176
+ },
177
+ "max_position_embeddings": 131072,
178
+ "model_type": "gemma4_vision",
179
+ "num_attention_heads": 12,
180
+ "num_hidden_layers": 16,
181
+ "num_key_value_heads": 12,
182
+ "output_attentions": false,
183
+ "output_hidden_states": false,
184
+ "patch_size": 16,
185
+ "pooling_kernel_size": 3,
186
+ "position_embedding_size": 10240,
187
+ "problem_type": null,
188
+ "return_dict": true,
189
+ "rms_norm_eps": 1e-06,
190
+ "rope_parameters": {
191
+ "rope_theta": 100.0,
192
+ "rope_type": "default"
193
+ },
194
+ "standardize": false,
195
+ "use_clipped_linears": true
196
+ },
197
+ "vision_soft_tokens_per_image": 280
198
+ }
checkpoint-20/generation_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 2,
3
+ "do_sample": true,
4
+ "eos_token_id": [
5
+ 1,
6
+ 1
7
+ ],
8
+ "pad_token_id": 0,
9
+ "temperature": 1.0,
10
+ "top_k": 64,
11
+ "top_p": 0.95,
12
+ "transformers_version": "5.7.0"
13
+ }
checkpoint-20/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c497c7cb5ae08f5d330119df7ed74b8dc511cac57ce985f64dbc14d7fdf06db9
3
+ size 15882477500
checkpoint-20/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e8fa98630d2a45d5ec5281dd52a4f7616df1bde5d03c9ec638ff3650c9d58447
3
+ size 29852609572
checkpoint-20/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
3
+ size 14645
checkpoint-20/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:afd22c1bf329a6fbd284d23e548d59e0abc4082e4d246c1055453c11250c80a9
3
+ size 1465
checkpoint-20/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12bac982b793c44b03d52a250a9f0d0b666813da566b910c24a6da0695fd11e6
3
+ size 32170070
checkpoint-20/tokenizer_config.json ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "audio_token": "<|audio|>",
3
+ "backend": "tokenizers",
4
+ "boa_token": "<|audio>",
5
+ "boi_token": "<|image>",
6
+ "bos_token": "<bos>",
7
+ "eoa_token": "<audio|>",
8
+ "eoc_token": "<channel|>",
9
+ "eoi_token": "<image|>",
10
+ "eos_token": "<eos>",
11
+ "eot_token": "<turn|>",
12
+ "escape_token": "<|\"|>",
13
+ "etc_token": "<tool_call|>",
14
+ "etd_token": "<tool|>",
15
+ "etr_token": "<tool_response|>",
16
+ "extra_special_tokens": [
17
+ "<|video|>"
18
+ ],
19
+ "image_token": "<|image|>",
20
+ "is_local": false,
21
+ "local_files_only": false,
22
+ "mask_token": "<mask>",
23
+ "model_max_length": 1000000000000000019884624838656,
24
+ "model_specific_special_tokens": {
25
+ "audio_token": "<|audio|>",
26
+ "boa_token": "<|audio>",
27
+ "boi_token": "<|image>",
28
+ "eoa_token": "<audio|>",
29
+ "eoc_token": "<channel|>",
30
+ "eoi_token": "<image|>",
31
+ "eot_token": "<turn|>",
32
+ "escape_token": "<|\"|>",
33
+ "etc_token": "<tool_call|>",
34
+ "etd_token": "<tool|>",
35
+ "etr_token": "<tool_response|>",
36
+ "image_token": "<|image|>",
37
+ "soc_token": "<|channel>",
38
+ "sot_token": "<|turn>",
39
+ "stc_token": "<|tool_call>",
40
+ "std_token": "<|tool>",
41
+ "str_token": "<|tool_response>",
42
+ "think_token": "<|think|>"
43
+ },
44
+ "pad_token": "<pad>",
45
+ "padding_side": "left",
46
+ "processor_class": "Gemma4Processor",
47
+ "soc_token": "<|channel>",
48
+ "sot_token": "<|turn>",
49
+ "stc_token": "<|tool_call>",
50
+ "std_token": "<|tool>",
51
+ "str_token": "<|tool_response>",
52
+ "think_token": "<|think|>",
53
+ "tokenizer_class": "GemmaTokenizer",
54
+ "unk_token": "<unk>"
55
+ }
checkpoint-20/trainer_state.json ADDED
@@ -0,0 +1,174 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.20970714724554576,
6
+ "eval_steps": 1,
7
+ "global_step": 20,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.010485357362277289,
14
+ "grad_norm": 498.0,
15
+ "learning_rate": 0.0,
16
+ "loss": 1.6033425331115723,
17
+ "step": 1
18
+ },
19
+ {
20
+ "epoch": 0.020970714724554577,
21
+ "grad_norm": 139.0,
22
+ "learning_rate": 4e-08,
23
+ "loss": 1.5843058824539185,
24
+ "step": 2
25
+ },
26
+ {
27
+ "epoch": 0.03145607208683186,
28
+ "grad_norm": 158.0,
29
+ "learning_rate": 8e-08,
30
+ "loss": 1.624650239944458,
31
+ "step": 3
32
+ },
33
+ {
34
+ "epoch": 0.041941429449109155,
35
+ "grad_norm": 201.0,
36
+ "learning_rate": 1.2000000000000002e-07,
37
+ "loss": 1.5930713415145874,
38
+ "step": 4
39
+ },
40
+ {
41
+ "epoch": 0.05242678681138644,
42
+ "grad_norm": 1392.0,
43
+ "learning_rate": 1.6e-07,
44
+ "loss": 1.6284583806991577,
45
+ "step": 5
46
+ },
47
+ {
48
+ "epoch": 0.06291214417366373,
49
+ "grad_norm": 644.0,
50
+ "learning_rate": 2.0000000000000002e-07,
51
+ "loss": 1.5755040645599365,
52
+ "step": 6
53
+ },
54
+ {
55
+ "epoch": 0.07339750153594102,
56
+ "grad_norm": 358.0,
57
+ "learning_rate": 2.4000000000000003e-07,
58
+ "loss": 1.5862637758255005,
59
+ "step": 7
60
+ },
61
+ {
62
+ "epoch": 0.08388285889821831,
63
+ "grad_norm": 1012.0,
64
+ "learning_rate": 2.8e-07,
65
+ "loss": 1.6306817531585693,
66
+ "step": 8
67
+ },
68
+ {
69
+ "epoch": 0.0943682162604956,
70
+ "grad_norm": 187.0,
71
+ "learning_rate": 3.2e-07,
72
+ "loss": 1.5844253301620483,
73
+ "step": 9
74
+ },
75
+ {
76
+ "epoch": 0.10485357362277288,
77
+ "grad_norm": 342.0,
78
+ "learning_rate": 3.6e-07,
79
+ "loss": 1.5895334482192993,
80
+ "step": 10
81
+ },
82
+ {
83
+ "epoch": 0.11533893098505017,
84
+ "grad_norm": 816.0,
85
+ "learning_rate": 4.0000000000000003e-07,
86
+ "loss": 1.5861637592315674,
87
+ "step": 11
88
+ },
89
+ {
90
+ "epoch": 0.12582428834732745,
91
+ "grad_norm": 260.0,
92
+ "learning_rate": 4.4e-07,
93
+ "loss": 1.57806396484375,
94
+ "step": 12
95
+ },
96
+ {
97
+ "epoch": 0.13630964570960474,
98
+ "grad_norm": 326.0,
99
+ "learning_rate": 4.800000000000001e-07,
100
+ "loss": 1.5836868286132812,
101
+ "step": 13
102
+ },
103
+ {
104
+ "epoch": 0.14679500307188204,
105
+ "grad_norm": 668.0,
106
+ "learning_rate": 5.2e-07,
107
+ "loss": 1.5597259998321533,
108
+ "step": 14
109
+ },
110
+ {
111
+ "epoch": 0.15728036043415933,
112
+ "grad_norm": 256.0,
113
+ "learning_rate": 5.6e-07,
114
+ "loss": 1.5852601528167725,
115
+ "step": 15
116
+ },
117
+ {
118
+ "epoch": 0.16776571779643662,
119
+ "grad_norm": 205.0,
120
+ "learning_rate": 6.000000000000001e-07,
121
+ "loss": 1.5899027585983276,
122
+ "step": 16
123
+ },
124
+ {
125
+ "epoch": 0.1782510751587139,
126
+ "grad_norm": 188.0,
127
+ "learning_rate": 6.4e-07,
128
+ "loss": 1.5692049264907837,
129
+ "step": 17
130
+ },
131
+ {
132
+ "epoch": 0.1887364325209912,
133
+ "grad_norm": 426.0,
134
+ "learning_rate": 6.800000000000001e-07,
135
+ "loss": 1.55263090133667,
136
+ "step": 18
137
+ },
138
+ {
139
+ "epoch": 0.1992217898832685,
140
+ "grad_norm": 245.0,
141
+ "learning_rate": 7.2e-07,
142
+ "loss": 1.5602704286575317,
143
+ "step": 19
144
+ },
145
+ {
146
+ "epoch": 0.20970714724554576,
147
+ "grad_norm": 158.0,
148
+ "learning_rate": 7.6e-07,
149
+ "loss": 1.5857173204421997,
150
+ "step": 20
151
+ }
152
+ ],
153
+ "logging_steps": 1,
154
+ "max_steps": 96,
155
+ "num_input_tokens_seen": 0,
156
+ "num_train_epochs": 1,
157
+ "save_steps": 5,
158
+ "stateful_callbacks": {
159
+ "TrainerControl": {
160
+ "args": {
161
+ "should_epoch_stop": false,
162
+ "should_evaluate": false,
163
+ "should_log": false,
164
+ "should_save": true,
165
+ "should_training_stop": false
166
+ },
167
+ "attributes": {}
168
+ }
169
+ },
170
+ "total_flos": 1.120241531706409e+18,
171
+ "train_batch_size": 1,
172
+ "trial_name": null,
173
+ "trial_params": null
174
+ }
checkpoint-20/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:874ac4b635f5e0e0130ccc11de5bfc6cee2eac14aa65567d5f9737b0b5cc556a
3
+ size 5265