data/scripts_hsichen/dpo2sft_data.py

import json

data = json.load(open('/home/hsichen/LLaMA-Factory/data/chaiting/pk_2087.json','r'))

converted_data = []

for item in data:
    try:
        bot_name = item['conversations'][2]['value'].split(':')[0]
        item['chosen']['value'] = bot_name + ': ' + item['chosen']['value']
        converted_data.append({
            'conversations': item['conversations'] + [item['chosen']]
        })
    except:
        print(item['conversations'])
        print('###############\n\n')
        continue
    

output = '/home/hsichen/LLaMA-Factory/data/chaiting/sft_2087.json'
with open(output, 'w') as f:
    json.dump(converted_data, f, ensure_ascii=False)
print(f"Converted data saved to {output}, length: {len(converted_data)}")