Text Generation
PEFT
Safetensors
English
qlora
lora
structured-output
HamadaMayu commited on
Commit
cb5d95e
·
verified ·
1 Parent(s): 375e1f3

Upload LoRA adapter (README written by author)

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
README.md CHANGED
@@ -1,3 +1,65 @@
1
- ---
2
- license: apache-2.0
3
- ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: Qwen/Qwen3-4B-Instruct-2507
3
+ datasets:
4
+ - u-10bei/structured_data_with_cot_dataset_512_v4
5
+ language:
6
+ - en
7
+ license: apache-2.0
8
+ library_name: peft
9
+ pipeline_tag: text-generation
10
+ tags:
11
+ - qlora
12
+ - lora
13
+ - structured-output
14
+ ---
15
+
16
+ qwen3-4b-structured-output-lora-v4-2epoch
17
+
18
+ This repository provides a **LoRA adapter** fine-tuned from
19
+ **Qwen/Qwen3-4B-Instruct-2507** using **QLoRA (4-bit, Unsloth)**.
20
+
21
+ This repository contains **LoRA adapter weights only**.
22
+ The base model must be loaded separately.
23
+
24
+ ## Training Objective
25
+
26
+ This adapter is trained to improve **structured output accuracy**
27
+ (JSON / YAML / XML / TOML / CSV).
28
+
29
+ Loss is applied only to the final assistant output,
30
+ while intermediate reasoning (Chain-of-Thought) is masked.
31
+
32
+ ## Training Configuration
33
+
34
+ - Base model: Qwen/Qwen3-4B-Instruct-2507
35
+ - Method: QLoRA (4-bit)
36
+ - Max sequence length: 2048
37
+ - Epochs: 2
38
+ - Learning rate: 2e-06
39
+ - LoRA: r=64, alpha=128
40
+
41
+ ## Usage
42
+
43
+ ```python
44
+ from transformers import AutoModelForCausalLM, AutoTokenizer
45
+ from peft import PeftModel
46
+ import torch
47
+
48
+ base = "Qwen/Qwen3-4B-Instruct-2507"
49
+ adapter = "your_id/your-repo"
50
+
51
+ tokenizer = AutoTokenizer.from_pretrained(base)
52
+ model = AutoModelForCausalLM.from_pretrained(
53
+ base,
54
+ torch_dtype=torch.float16,
55
+ device_map="auto",
56
+ )
57
+ model = PeftModel.from_pretrained(model, adapter)
58
+ ```
59
+
60
+ ## Sources & Terms (IMPORTANT)
61
+
62
+ Training data: u-10bei/structured_data_with_cot_dataset_512_v4
63
+
64
+ Dataset License: MIT License. This dataset is used and distributed under the terms of the MIT License.
65
+ Compliance: Users must comply with the MIT license (including copyright notice) and the base model's original terms of use.
adapter_config.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "Qwen/Qwen3-4B-Instruct-2507",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": false,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 128,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.0,
22
+ "megatron_config": null,
23
+ "megatron_core": "megatron.core",
24
+ "modules_to_save": null,
25
+ "peft_type": "LORA",
26
+ "peft_version": "0.18.1",
27
+ "qalora_group_size": 16,
28
+ "r": 64,
29
+ "rank_pattern": {},
30
+ "revision": null,
31
+ "target_modules": [
32
+ "up_proj",
33
+ "o_proj",
34
+ "v_proj",
35
+ "k_proj",
36
+ "q_proj",
37
+ "gate_proj",
38
+ "down_proj"
39
+ ],
40
+ "target_parameters": null,
41
+ "task_type": "CAUSAL_LM",
42
+ "trainable_token_indices": null,
43
+ "use_dora": false,
44
+ "use_qalora": false,
45
+ "use_rslora": false
46
+ }
adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3237ce4db46fa09af432eb7882f4f6fa5896bd36fb23086a4f8047c7f36e901
3
+ size 528550256
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
3
+ size 11422650
tokenizer_config.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": null,
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "<|im_end|>",
7
+ "errors": "replace",
8
+ "extra_special_tokens": [
9
+ "<|im_start|>",
10
+ "<|im_end|>",
11
+ "<|object_ref_start|>",
12
+ "<|object_ref_end|>",
13
+ "<|box_start|>",
14
+ "<|box_end|>",
15
+ "<|quad_start|>",
16
+ "<|quad_end|>",
17
+ "<|vision_start|>",
18
+ "<|vision_end|>",
19
+ "<|vision_pad|>",
20
+ "<|image_pad|>",
21
+ "<|video_pad|>"
22
+ ],
23
+ "is_local": false,
24
+ "model_max_length": 1010000,
25
+ "pad_token": "<|endoftext|>",
26
+ "split_special_tokens": false,
27
+ "tokenizer_class": "Qwen2Tokenizer",
28
+ "unk_token": null
29
+ }