File size: 1,361 Bytes
f320b60 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | # Comparison run — 林路 persona LoRA on Qwen3.6-27B (dense VLM, text tower only).
# Dense model: standard attention everywhere, so the full q/k/v/o + MLP set is
# targetable (no fused-expert name clash).
# Run: CUDA_VISIBLE_DEVICES=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
# llamafactory-cli train train_lora_27b.yaml
### model
model_name_or_path: ./Qwen3.6-27B
trust_remote_code: true
### method
stage: sft
do_train: true
finetuning_type: lora
lora_rank: 32
lora_alpha: 64
lora_dropout: 0.05
lora_target: q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj
freeze_vision_tower: true
### dataset
dataset: lin_lu_train
eval_dataset: lin_lu_val
dataset_dir: ./data
template: qwen
cutoff_len: 8192
overwrite_cache: true
preprocessing_num_workers: 8
### output
output_dir: ./out/lin_lu_lora_27b
logging_steps: 2
save_steps: 4 # ~10 min between checkpoints; host crashes lose little
save_total_limit: 3
plot_loss: true
overwrite_output_dir: true
report_to: none
### train
per_device_train_batch_size: 1
gradient_accumulation_steps: 8
learning_rate: 1.0e-4
num_train_epochs: 5.0
lr_scheduler_type: cosine
warmup_ratio: 0.05
bf16: true
gradient_checkpointing: true
### eval
per_device_eval_batch_size: 1
eval_strategy: steps
eval_steps: 4
load_best_model_at_end: true
metric_for_best_model: eval_loss
greater_is_better: false
|