/vast/users/muhammad.haris/anaconda3/envs/polyalign/lib/python3.11/site-packages/torch/nn/modules/module.py:1357: UserWarning: expandable_segments not supported on this platform (Triggered internally at /pytorch/c10/hip/HIPAllocatorConfig.h:36.) return t.to( { "input_root": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_raw", "output_root": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_scored", "splits": { "train": [ { "dataset": "cmrc2018", "input_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_raw/cmrc2018/train.jsonl", "output_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_scored/cmrc2018/train.jsonl", "records": 5867, "critic_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/critic/bundle", "score_diagnostics": { "score_min": -0.0466718413, "score_max": 1.1740882397, "score_range": 1.220760081, "score_std": 0.1314062483, "equal_score_records": 0, "distinct_text_pairs": 5867, "distinct_text_equal_score_records": 0, "all_scores_tied": false } }, { "dataset": "coig_cqia", "input_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_raw/coig_cqia/train.jsonl", "output_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_scored/coig_cqia/train.jsonl", "records": 9380, "critic_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/critic/bundle", "score_diagnostics": { "score_min": -0.0026426115, "score_max": 2.2379500866, "score_range": 2.2405926981, "score_std": 0.4729376128, "equal_score_records": 0, "distinct_text_pairs": 9380, "distinct_text_equal_score_records": 0, "all_scores_tied": false } }, { "dataset": "drcd", "input_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_raw/drcd/train.jsonl", "output_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_scored/drcd/train.jsonl", "records": 5865, "critic_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/critic/bundle", "score_diagnostics": { "score_min": 0.0004258901, "score_max": 1.058147192, "score_range": 1.0577213019, "score_std": 0.0536777405, "equal_score_records": 0, "distinct_text_pairs": 5865, "distinct_text_equal_score_records": 0, "all_scores_tied": false } }, { "dataset": "dureader", "input_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_raw/dureader/train.jsonl", "output_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_scored/dureader/train.jsonl", "records": 5264, "critic_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/critic/bundle", "score_diagnostics": { "score_min": -0.0216317102, "score_max": 1.3106007576, "score_range": 1.3322324678, "score_std": 0.1393853748, "equal_score_records": 0, "distinct_text_pairs": 5264, "distinct_text_equal_score_records": 0, "all_scores_tied": false } }, { "dataset": "hc3_chinese", "input_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_raw/hc3_chinese/train.jsonl", "output_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_scored/hc3_chinese/train.jsonl", "records": 19983, "critic_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/critic/bundle", "score_diagnostics": { "score_min": -0.1189319193, "score_max": 1.9853544235, "score_range": 2.1042863429, "score_std": 0.3768150072, "equal_score_records": 0, "distinct_text_pairs": 19983, "distinct_text_equal_score_records": 0, "all_scores_tied": false } }, { "dataset": "oasst2_zh", "input_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_raw/oasst2_zh/train.jsonl", "output_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_scored/oasst2_zh/train.jsonl", "records": 3773, "critic_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/critic/bundle", "score_diagnostics": { "score_min": -0.0031202063, "score_max": 2.7128031254, "score_range": 2.7159233317, "score_std": 0.3419253835, "equal_score_records": 0, "distinct_text_pairs": 3773, "distinct_text_equal_score_records": 0, "all_scores_tied": false } } ], "val": [ { "dataset": "cmrc2018", "input_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_raw/cmrc2018/dev.jsonl", "output_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_scored/cmrc2018/dev.jsonl", "records": 2421, "critic_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/critic/bundle", "score_diagnostics": { "score_min": -0.0493306108, "score_max": 1.2184919119, "score_range": 1.2678225227, "score_std": 0.1214584971, "equal_score_records": 0, "distinct_text_pairs": 2421, "distinct_text_equal_score_records": 0, "all_scores_tied": false } }, { "dataset": "coig_cqia", "input_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_raw/coig_cqia/dev.jsonl", "output_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_scored/coig_cqia/dev.jsonl", "records": 506, "critic_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/critic/bundle", "score_diagnostics": { "score_min": -0.0006540432, "score_max": 1.8188692331, "score_range": 1.8195232763, "score_std": 0.2749469174, "equal_score_records": 0, "distinct_text_pairs": 506, "distinct_text_equal_score_records": 0, "all_scores_tied": false } }, { "dataset": "drcd", "input_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_raw/drcd/dev.jsonl", "output_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_scored/drcd/dev.jsonl", "records": 1566, "critic_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/critic/bundle", "score_diagnostics": { "score_min": 0.0007924479, "score_max": 0.623061657, "score_range": 0.622269209, "score_std": 0.0715070959, "equal_score_records": 0, "distinct_text_pairs": 1566, "distinct_text_equal_score_records": 0, "all_scores_tied": false } }, { "dataset": "dureader", "input_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_raw/dureader/dev.jsonl", "output_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_scored/dureader/dev.jsonl", "records": 1238, "critic_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/critic/bundle", "score_diagnostics": { "score_min": -0.004366355, "score_max": 1.0542331934, "score_range": 1.0585995484, "score_std": 0.151149901, "equal_score_records": 0, "distinct_text_pairs": 1238, "distinct_text_equal_score_records": 0, "all_scores_tied": false } }, { "dataset": "hc3_chinese", "input_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_raw/hc3_chinese/dev.jsonl", "output_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_scored/hc3_chinese/dev.jsonl", "records": 1152, "critic_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/critic/bundle", "score_diagnostics": { "score_min": -0.0591590889, "score_max": 1.6256961823, "score_range": 1.6848552711, "score_std": 0.2465027639, "equal_score_records": 0, "distinct_text_pairs": 1152, "distinct_text_equal_score_records": 0, "all_scores_tied": false } }, { "dataset": "oasst2_zh", "input_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_raw/oasst2_zh/dev.jsonl", "output_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_scored/oasst2_zh/dev.jsonl", "records": 525, "critic_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/critic/bundle", "score_diagnostics": { "score_min": -0.0008399463, "score_max": 1.4422352314, "score_range": 1.4430751777, "score_std": 0.1786839623, "equal_score_records": 0, "distinct_text_pairs": 525, "distinct_text_equal_score_records": 0, "all_scores_tied": false } } ], "test": [ { "dataset": "cmrc2018", "input_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_raw/cmrc2018/test.jsonl", "output_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_scored/cmrc2018/test.jsonl", "records": 651, "critic_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/critic/bundle", "score_diagnostics": { "score_min": -0.0340934582, "score_max": 1.1238440275, "score_range": 1.1579374857, "score_std": 0.1315003839, "equal_score_records": 0, "distinct_text_pairs": 651, "distinct_text_equal_score_records": 0, "all_scores_tied": false } }, { "dataset": "coig_cqia", "input_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_raw/coig_cqia/test.jsonl", "output_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_scored/coig_cqia/test.jsonl", "records": 572, "critic_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/critic/bundle", "score_diagnostics": { "score_min": 0.0007630251, "score_max": 0.9438989758, "score_range": 0.9431359507, "score_std": 0.1890997684, "equal_score_records": 0, "distinct_text_pairs": 572, "distinct_text_equal_score_records": 0, "all_scores_tied": false } }, { "dataset": "drcd", "input_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_raw/drcd/test.jsonl", "output_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_scored/drcd/test.jsonl", "records": 893, "critic_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/critic/bundle", "score_diagnostics": { "score_min": 0.000575081, "score_max": 0.7704350948, "score_range": 0.7698600139, "score_std": 0.0490631256, "equal_score_records": 0, "distinct_text_pairs": 893, "distinct_text_equal_score_records": 0, "all_scores_tied": false } }, { "dataset": "hc3_chinese", "input_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_raw/hc3_chinese/test.jsonl", "output_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_scored/hc3_chinese/test.jsonl", "records": 1058, "critic_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/critic/bundle", "score_diagnostics": { "score_min": -0.0278274715, "score_max": 0.9147658944, "score_range": 0.9425933659, "score_std": 0.1929577156, "equal_score_records": 0, "distinct_text_pairs": 1058, "distinct_text_equal_score_records": 0, "all_scores_tied": false } }, { "dataset": "oasst2_zh", "input_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_raw/oasst2_zh/test.jsonl", "output_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/pairs_scored/oasst2_zh/test.jsonl", "records": 224, "critic_path": "/vast/users/muhammad.haris/Sathira/PolyAlign/data/chinese/merged_sft_dedup/hdpo_work/qwen25_1_5b/critic/bundle", "score_diagnostics": { "score_min": 0.0009332732, "score_max": 0.9446465969, "score_range": 0.9437133237, "score_std": 0.1509780647, "equal_score_records": 0, "distinct_text_pairs": 224, "distinct_text_equal_score_records": 0, "all_scores_tied": false } } ] } }