| { |
| "project": "VK Education Vision-Language Modeling", |
| "author": { |
| "name": "Ибрагимов Далгат Магомедалиевич", |
| "institution": "МАИ, институт 8", |
| "group": "М8О-308Б-32" |
| }, |
| "primary_hf_artifact": "https://huggingface.co/lockR/vk-vlm-gqa-ru-qwen35-08b-lora", |
| "primary_run": "gqa_ru_qwen35_0_8b_lora_fast_v1", |
| "base_model": "Qwen/Qwen3.5-0.8B", |
| "adapter_type": "LoRA", |
| "dataset": { |
| "name": "deepvk/GQA-ru", |
| "source": "https://huggingface.co/datasets/deepvk/GQA-ru", |
| "train_samples": 38019, |
| "validation_samples": 1981, |
| "testdev_samples": 12216, |
| "usage": "Image-question-answer records were used for multimodal VLM LoRA fine-tuning and official lmms-eval evaluation." |
| }, |
| "training": { |
| "multimodal": true, |
| "vision_encoder_trainable": false, |
| "lora_scope": "language_model attention projection layers", |
| "seed": 42, |
| "epochs": 1.0, |
| "batch_size": 8, |
| "learning_rate": 0.0002, |
| "mixed_precision": "bf16", |
| "max_seq_length": 1024, |
| "image_resolution": 336, |
| "lora_r": 16, |
| "lora_alpha": 32, |
| "lora_dropout": 0.05, |
| "target_modules": [ |
| "q_proj", |
| "k_proj", |
| "v_proj", |
| "o_proj" |
| ], |
| "best_checkpoint": "checkpoint-4560", |
| "best_metric_name": "eval_loss", |
| "best_metric_value": 0.4337001144886017 |
| }, |
| "train_metrics": { |
| "train_runtime": 6219.1947, |
| "train_samples_per_second": 6.113, |
| "train_steps_per_second": 0.764, |
| "train_loss": 0.04432422036801592, |
| "epoch": 1.0, |
| "eval_loss": 0.4337001144886017, |
| "eval_runtime": 116.0177, |
| "eval_samples_per_second": 17.075, |
| "eval_steps_per_second": 1.069, |
| "metrics_path": "runs/gqa_ru_qwen35_0_8b_lora_fast_v1/train_metrics.json" |
| }, |
| "official_benchmark_full": { |
| "runner": "lmms-eval", |
| "task": "gqa-ru", |
| "dataset_path": "deepvk/GQA-ru", |
| "dataset_name": "testdev_balanced_instructions", |
| "split": "testdev", |
| "metric": "exact_match", |
| "metric_higher_is_better": true, |
| "prompt_suffix": "Ответь одним словом.", |
| "samples": 12216, |
| "limit": null, |
| "model_backend": "qwen3_5", |
| "model_args_common": "enable_thinking=False", |
| "base_exact_match": 0.2861820563195809, |
| "adapter_exact_match": 0.48321872953503603, |
| "exact_match_delta": 0.19703667321545515, |
| "relative_improvement": 0.6885011441647597, |
| "base_stderr": 0.004089480999753636, |
| "adapter_stderr": 0.004521458266039995, |
| "base_correct": 3496, |
| "adapter_correct": 5903, |
| "correct_delta": 2407, |
| "base_results_path": "runs/lmms_eval/gqa_ru_qwen35_base_full/Qwen__Qwen3.5-0.8B/20260604_141134_results.json", |
| "adapter_results_path": "runs/lmms_eval/gqa_ru_qwen35_lora_full/artifacts__merged_qwen35_gqa_ru_full/20260604_145224_results.json" |
| }, |
| "secondary_experiment": { |
| "base_model": "Qwen/Qwen2.5-VL-3B-Instruct", |
| "status": "Full GQA-ru training and full base-vs-adapter evaluation are in progress.", |
| "smoke_hf_artifact": "https://huggingface.co/lockR/vk-vlm-gqa-ru-qwen25vl-3b-lora-smoke", |
| "smoke_limit_100_base_exact_match": 0.39, |
| "smoke_limit_100_adapter_exact_match": 0.48 |
| }, |
| "limitations": [ |
| "The vision encoder was frozen; LoRA was trained only in language model attention projection layers.", |
| "MMBench-ru has not yet been measured.", |
| "The Qwen2.5-VL full experiment is still running and is not included in the primary result." |
| ] |
| } |
|
|
