Upload curated V1 reward snapshots

data/rewards.mock.jsonl

@@ -0,0 +1,30 @@
+{"task_id":"math-001","prompt":"Compute 17 + 25. Return only the number.","domain":"math","split":"train","tags":["arithmetic"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[0.9999993184666667,-6.691333333195264e-7,-6.687333333053781e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"42","quality":1.0,"cost_usd":0.0,"latency_ms":1.0222999999550666,"prompt_tokens":8,"completion_tokens":1,"generation_id":"mock-math_expert-math-001","error":null,"utility":0.9999993184666667},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0036999999792897,"prompt_tokens":8,"completion_tokens":5,"generation_id":"mock-code_expert-math-001","error":null,"utility":-6.691333333195264e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0030999999580672,"prompt_tokens":8,"completion_tokens":5,"generation_id":"mock-general_expert-math-001","error":null,"utility":-6.687333333053781e-7}]}
+{"task_id":"math-002","prompt":"Compute 9 multiplied by 8. Return only the number.","domain":"math","split":"train","tags":["arithmetic"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[0.9999993280666667,-6.679333332770815e-7,-6.673999999754111e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"72","quality":1.0,"cost_usd":0.0,"latency_ms":1.0078999999004736,"prompt_tokens":9,"completion_tokens":1,"generation_id":"mock-math_expert-math-002","error":null,"utility":0.9999993280666667},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0018999999156222,"prompt_tokens":9,"completion_tokens":5,"generation_id":"mock-code_expert-math-002","error":null,"utility":-6.679333332770815e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0010999999631167,"prompt_tokens":9,"completion_tokens":5,"generation_id":"mock-general_expert-math-002","error":null,"utility":-6.673999999754111e-7}]}
+{"task_id":"math-003","prompt":"What is the square root of 144? Return only the number.","domain":"math","split":"train","tags":["arithmetic"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[0.9999993296,-6.677333332700073e-7,-6.67333333255859e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"12","quality":1.0,"cost_usd":0.0,"latency_ms":1.005599999894912,"prompt_tokens":11,"completion_tokens":1,"generation_id":"mock-math_expert-math-003","error":null,"utility":0.9999993296},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.001599999905011,"prompt_tokens":11,"completion_tokens":5,"generation_id":"mock-code_expert-math-003","error":null,"utility":-6.677333332700073e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0009999998837884,"prompt_tokens":11,"completion_tokens":5,"generation_id":"mock-general_expert-math-003","error":null,"utility":-6.67333333255859e-7}]}
+{"task_id":"math-004","prompt":"Solve 3x + 5 = 20. Return only x.","domain":"math","split":"train","tags":["algebra"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[0.9999993298,-6.675333332629332e-7,-6.674666666949634e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"5","quality":1.0,"cost_usd":0.0,"latency_ms":1.0053000001116743,"prompt_tokens":9,"completion_tokens":1,"generation_id":"mock-math_expert-math-004","error":null,"utility":0.9999993298},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0012999998943997,"prompt_tokens":9,"completion_tokens":5,"generation_id":"mock-code_expert-math-004","error":null,"utility":-6.675333332629332e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.001200000042445,"prompt_tokens":9,"completion_tokens":5,"generation_id":"mock-general_expert-math-004","error":null,"utility":-6.674666666949634e-7}]}
+{"task_id":"math-005","prompt":"A triangle has angles 50 and 60 degrees. Return the third angle as a number.","domain":"math","split":"train","tags":["geometry"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[0.9999993302666667,-6.675333332629332e-7,-6.67199999968337e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"70","quality":1.0,"cost_usd":0.0,"latency_ms":1.0046000000111235,"prompt_tokens":15,"completion_tokens":1,"generation_id":"mock-math_expert-math-005","error":null,"utility":0.9999993302666667},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0012999998943997,"prompt_tokens":15,"completion_tokens":5,"generation_id":"mock-code_expert-math-005","error":null,"utility":-6.675333332629332e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0007999999525055,"prompt_tokens":15,"completion_tokens":5,"generation_id":"mock-general_expert-math-005","error":null,"utility":-6.67199999968337e-7}]}
+{"task_id":"math-006","prompt":"Compute 2 to the power 10. Return only the number.","domain":"math","split":"train","tags":["arithmetic"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[0.9999993297333334,-6.677333332700073e-7,-6.672666666878893e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"1024","quality":1.0,"cost_usd":0.0,"latency_ms":1.005399999963629,"prompt_tokens":10,"completion_tokens":1,"generation_id":"mock-math_expert-math-006","error":null,"utility":0.9999993297333334},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.001599999905011,"prompt_tokens":10,"completion_tokens":5,"generation_id":"mock-code_expert-math-006","error":null,"utility":-6.677333332700073e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0009000000318338,"prompt_tokens":10,"completion_tokens":5,"generation_id":"mock-general_expert-math-006","error":null,"utility":-6.672666666878893e-7}]}
+{"task_id":"code-001","prompt":"Which Python keyword starts a function definition? Answer with the keyword.","domain":"code","split":"train","tags":["python"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.700000000673754e-7,0.9999993325333333,-6.673333334074414e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.005000000101063,"prompt_tokens":11,"completion_tokens":5,"generation_id":"mock-math_expert-code-001","error":null,"utility":-6.700000000673754e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"def","quality":1.0,"cost_usd":0.0,"latency_ms":1.001200000042445,"prompt_tokens":11,"completion_tokens":1,"generation_id":"mock-code_expert-code-001","error":null,"utility":0.9999993325333333},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.001000000111162,"prompt_tokens":11,"completion_tokens":5,"generation_id":"mock-general_expert-code-001","error":null,"utility":-6.673333334074414e-7}]}
+{"task_id":"code-002","prompt":"What is the conventional filename for Python package build metadata using PEP 621?","domain":"code","split":"train","tags":["python","packaging"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.700666667869276e-7,0.9999993324,-6.673333334074414e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0051000001803914,"prompt_tokens":13,"completion_tokens":5,"generation_id":"mock-math_expert-code-002","error":null,"utility":-6.700666667869276e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"pyproject.toml","quality":1.0,"cost_usd":0.0,"latency_ms":1.001399999973728,"prompt_tokens":13,"completion_tokens":1,"generation_id":"mock-code_expert-code-002","error":null,"utility":0.9999993324},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.001000000111162,"prompt_tokens":13,"completion_tokens":5,"generation_id":"mock-general_expert-code-002","error":null,"utility":-6.673333334074414e-7}]}
+{"task_id":"code-003","prompt":"Which Git command displays the working tree status?","domain":"code","split":"train","tags":["git"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.698000000603012e-7,0.9999993324,-6.675333332629332e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0047000000904518,"prompt_tokens":8,"completion_tokens":5,"generation_id":"mock-math_expert-code-003","error":null,"utility":-6.698000000603012e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"git status","quality":1.0,"cost_usd":0.0,"latency_ms":1.001399999973728,"prompt_tokens":8,"completion_tokens":2,"generation_id":"mock-code_expert-code-003","error":null,"utility":0.9999993324},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0012999998943997,"prompt_tokens":8,"completion_tokens":5,"generation_id":"mock-general_expert-code-003","error":null,"utility":-6.675333332629332e-7}]}
+{"task_id":"code-004","prompt":"Which SQL keyword removes duplicate rows from a SELECT result?","domain":"code","split":"train","tags":["sql"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.700000000673754e-7,0.9999993323333333,-6.672666666878893e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.005000000101063,"prompt_tokens":10,"completion_tokens":5,"generation_id":"mock-math_expert-code-004","error":null,"utility":-6.700000000673754e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"DISTINCT","quality":1.0,"cost_usd":0.0,"latency_ms":1.0015000000530563,"prompt_tokens":10,"completion_tokens":1,"generation_id":"mock-code_expert-code-004","error":null,"utility":0.9999993323333333},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0009000000318338,"prompt_tokens":10,"completion_tokens":5,"generation_id":"mock-general_expert-code-004","error":null,"utility":-6.672666666878893e-7}]}
+{"task_id":"code-005","prompt":"Name the HTTP method normally used to retrieve a resource.","domain":"code","split":"train","tags":["http"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.698000000603012e-7,0.9999993324,-6.673333334074414e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0047000000904518,"prompt_tokens":10,"completion_tokens":5,"generation_id":"mock-math_expert-code-005","error":null,"utility":-6.698000000603012e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"GET","quality":1.0,"cost_usd":0.0,"latency_ms":1.001399999973728,"prompt_tokens":10,"completion_tokens":1,"generation_id":"mock-code_expert-code-005","error":null,"utility":0.9999993324},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.001000000111162,"prompt_tokens":10,"completion_tokens":5,"generation_id":"mock-general_expert-code-005","error":null,"utility":-6.673333334074414e-7}]}
+{"task_id":"code-006","prompt":"Which shell command prints the current working directory?","domain":"code","split":"train","tags":["shell"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.704000000815237e-7,0.9999993322666666,-6.673999999754111e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0056000001222856,"prompt_tokens":8,"completion_tokens":5,"generation_id":"mock-math_expert-code-006","error":null,"utility":-6.704000000815237e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"pwd","quality":1.0,"cost_usd":0.0,"latency_ms":1.0016000001323846,"prompt_tokens":8,"completion_tokens":1,"generation_id":"mock-code_expert-code-006","error":null,"utility":0.9999993322666666},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0010999999631167,"prompt_tokens":8,"completion_tokens":5,"generation_id":"mock-general_expert-code-006","error":null,"utility":-6.673999999754111e-7}]}
+{"task_id":"general-001","prompt":"What is the capital of France?","domain":"general","split":"train","tags":["geography"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.703333333619716e-7,-6.675333332629332e-7,0.9999993326000001],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0055000000429573,"prompt_tokens":6,"completion_tokens":5,"generation_id":"mock-math_expert-general-001","error":null,"utility":-6.703333333619716e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0012999998943997,"prompt_tokens":6,"completion_tokens":5,"generation_id":"mock-code_expert-general-001","error":null,"utility":-6.675333332629332e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"Paris","quality":1.0,"cost_usd":0.0,"latency_ms":1.0010999999631167,"prompt_tokens":6,"completion_tokens":1,"generation_id":"mock-general_expert-general-001","error":null,"utility":0.9999993326000001}]}
+{"task_id":"general-002","prompt":"Which planet is known as the Red Planet?","domain":"general","split":"train","tags":["science"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.699999999157929e-7,-6.675333334145155e-7,0.9999993326666667],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0049999998736894,"prompt_tokens":8,"completion_tokens":5,"generation_id":"mock-math_expert-general-002","error":null,"utility":-6.699999999157929e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0013000001217733,"prompt_tokens":8,"completion_tokens":5,"generation_id":"mock-code_expert-general-002","error":null,"utility":-6.675333334145155e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"Mars","quality":1.0,"cost_usd":0.0,"latency_ms":1.0009999998837884,"prompt_tokens":8,"completion_tokens":1,"generation_id":"mock-general_expert-general-002","error":null,"utility":0.9999993326666667}]}
+{"task_id":"general-003","prompt":"Who wrote Hamlet?","domain":"general","split":"train","tags":["literature"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.698666667798534e-7,-6.674666666949634e-7,0.9999993326000001],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.00480000016978,"prompt_tokens":3,"completion_tokens":5,"generation_id":"mock-math_expert-general-003","error":null,"utility":-6.698666667798534e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.001200000042445,"prompt_tokens":3,"completion_tokens":5,"generation_id":"mock-code_expert-general-003","error":null,"utility":-6.674666666949634e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"Shakespeare","quality":1.0,"cost_usd":0.0,"latency_ms":1.0010999999631167,"prompt_tokens":3,"completion_tokens":1,"generation_id":"mock-general_expert-general-003","error":null,"utility":0.9999993326000001}]}
+{"task_id":"general-004","prompt":"What gas do plants absorb from the atmosphere during photosynthesis?","domain":"general","split":"train","tags":["science"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.700000000673754e-7,-6.673999999754111e-7,0.9999993326000001],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.005000000101063,"prompt_tokens":10,"completion_tokens":5,"generation_id":"mock-math_expert-general-004","error":null,"utility":-6.700000000673754e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0010999999631167,"prompt_tokens":10,"completion_tokens":5,"generation_id":"mock-code_expert-general-004","error":null,"utility":-6.673999999754111e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"carbon dioxide","quality":1.0,"cost_usd":0.0,"latency_ms":1.0010999999631167,"prompt_tokens":10,"completion_tokens":2,"generation_id":"mock-general_expert-general-004","error":null,"utility":0.9999993326000001}]}
+{"task_id":"general-005","prompt":"What is the largest ocean on Earth?","domain":"general","split":"train","tags":["geography"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.701333333548973e-7,-6.675333332629332e-7,0.9999993325333335],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.005200000032346,"prompt_tokens":7,"completion_tokens":5,"generation_id":"mock-math_expert-general-005","error":null,"utility":-6.701333333548973e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0012999998943997,"prompt_tokens":7,"completion_tokens":5,"generation_id":"mock-code_expert-general-005","error":null,"utility":-6.675333332629332e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"Pacific","quality":1.0,"cost_usd":0.0,"latency_ms":1.0011999998150714,"prompt_tokens":7,"completion_tokens":1,"generation_id":"mock-general_expert-general-005","error":null,"utility":0.9999993325333335}]}
+{"task_id":"general-006","prompt":"Which language is primarily spoken in Brazil?","domain":"general","split":"train","tags":["language"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.698000000603012e-7,-6.67466666543381e-7,0.9999993325999998],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0047000000904518,"prompt_tokens":7,"completion_tokens":5,"generation_id":"mock-math_expert-general-006","error":null,"utility":-6.698000000603012e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0011999998150714,"prompt_tokens":7,"completion_tokens":5,"generation_id":"mock-code_expert-general-006","error":null,"utility":-6.67466666543381e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"Portuguese","quality":1.0,"cost_usd":0.0,"latency_ms":1.0011000001904904,"prompt_tokens":7,"completion_tokens":1,"generation_id":"mock-general_expert-general-006","error":null,"utility":0.9999993325999998}]}
+{"task_id":"math-val-001","prompt":"Compute 11 squared. Return only the number.","domain":"math","split":"validation","tags":["arithmetic"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[0.9999993298,-6.676666667020376e-7,-6.672666666878893e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"121","quality":1.0,"cost_usd":0.0,"latency_ms":1.0053000001116743,"prompt_tokens":7,"completion_tokens":1,"generation_id":"mock-math_expert-math-val-001","error":null,"utility":0.9999993298},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0015000000530563,"prompt_tokens":7,"completion_tokens":5,"generation_id":"mock-code_expert-math-val-001","error":null,"utility":-6.676666667020376e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0009000000318338,"prompt_tokens":7,"completion_tokens":5,"generation_id":"mock-general_expert-math-val-001","error":null,"utility":-6.672666666878893e-7}]}
+{"task_id":"math-val-002","prompt":"Solve 2x = 18. Return only x.","domain":"math","split":"validation","tags":["algebra"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[0.9999993299333334,-6.676666667020376e-7,-6.67333333255859e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"9","quality":1.0,"cost_usd":0.0,"latency_ms":1.0050999999530177,"prompt_tokens":7,"completion_tokens":1,"generation_id":"mock-math_expert-math-val-002","error":null,"utility":0.9999993299333334},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0015000000530563,"prompt_tokens":7,"completion_tokens":5,"generation_id":"mock-code_expert-math-val-002","error":null,"utility":-6.676666667020376e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0009999998837884,"prompt_tokens":7,"completion_tokens":5,"generation_id":"mock-general_expert-math-val-002","error":null,"utility":-6.67333333255859e-7}]}
+{"task_id":"code-val-001","prompt":"Which Python keyword creates an anonymous function?","domain":"code","split":"validation","tags":["python"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.69733333340749e-7,0.9999993324,-6.67333333255859e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0046000000111235,"prompt_tokens":7,"completion_tokens":5,"generation_id":"mock-math_expert-code-val-001","error":null,"utility":-6.69733333340749e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"lambda","quality":1.0,"cost_usd":0.0,"latency_ms":1.001399999973728,"prompt_tokens":7,"completion_tokens":1,"generation_id":"mock-code_expert-code-val-001","error":null,"utility":0.9999993324},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0009999998837884,"prompt_tokens":7,"completion_tokens":5,"generation_id":"mock-general_expert-code-val-001","error":null,"utility":-6.67333333255859e-7}]}
+{"task_id":"code-val-002","prompt":"Which HTTP status code means Not Found? Return the number.","domain":"code","split":"validation","tags":["http"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.700666666353452e-7,0.9999993324,-6.673999999754111e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0050999999530177,"prompt_tokens":10,"completion_tokens":5,"generation_id":"mock-math_expert-code-val-002","error":null,"utility":-6.700666666353452e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"404","quality":1.0,"cost_usd":0.0,"latency_ms":1.001399999973728,"prompt_tokens":10,"completion_tokens":1,"generation_id":"mock-code_expert-code-val-002","error":null,"utility":0.9999993324},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0010999999631167,"prompt_tokens":10,"completion_tokens":5,"generation_id":"mock-general_expert-code-val-002","error":null,"utility":-6.673999999754111e-7}]}
+{"task_id":"general-val-001","prompt":"What is the capital of Japan?","domain":"general","split":"validation","tags":["geography"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.699999999157929e-7,-6.674666666949634e-7,0.9999993326666666],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0049999998736894,"prompt_tokens":6,"completion_tokens":5,"generation_id":"mock-math_expert-general-val-001","error":null,"utility":-6.699999999157929e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.001200000042445,"prompt_tokens":6,"completion_tokens":5,"generation_id":"mock-code_expert-general-val-001","error":null,"utility":-6.674666666949634e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"Tokyo","quality":1.0,"cost_usd":0.0,"latency_ms":1.001000000111162,"prompt_tokens":6,"completion_tokens":1,"generation_id":"mock-general_expert-general-val-001","error":null,"utility":0.9999993326666666}]}
+{"task_id":"general-val-002","prompt":"Which element has the chemical symbol O?","domain":"general","split":"validation","tags":["science"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.69866666628271e-7,-6.673999999754111e-7,0.9999993326666667],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0047999999424064,"prompt_tokens":7,"completion_tokens":5,"generation_id":"mock-math_expert-general-val-002","error":null,"utility":-6.69866666628271e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0010999999631167,"prompt_tokens":7,"completion_tokens":5,"generation_id":"mock-code_expert-general-val-002","error":null,"utility":-6.673999999754111e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"oxygen","quality":1.0,"cost_usd":0.0,"latency_ms":1.0009999998837884,"prompt_tokens":7,"completion_tokens":1,"generation_id":"mock-general_expert-general-val-002","error":null,"utility":0.9999993326666667}]}
+{"task_id":"math-test-001","prompt":"Compute 15 percent of 200. Return only the number.","domain":"math","split":"test","tags":["percent"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[0.99999933,-6.676666667020376e-7,-6.674666666949634e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"30","quality":1.0,"cost_usd":0.0,"latency_ms":1.005000000101063,"prompt_tokens":9,"completion_tokens":1,"generation_id":"mock-math_expert-math-test-001","error":null,"utility":0.99999933},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0015000000530563,"prompt_tokens":9,"completion_tokens":5,"generation_id":"mock-code_expert-math-test-001","error":null,"utility":-6.676666667020376e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.001200000042445,"prompt_tokens":9,"completion_tokens":5,"generation_id":"mock-general_expert-math-test-001","error":null,"utility":-6.674666666949634e-7}]}
+{"task_id":"math-test-002","prompt":"Compute 7 cubed. Return only the number.","domain":"math","split":"test","tags":["arithmetic"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[0.9999993301333332,-6.676666667020376e-7,-6.67333333255859e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"343","quality":1.0,"cost_usd":0.0,"latency_ms":1.00480000016978,"prompt_tokens":7,"completion_tokens":1,"generation_id":"mock-math_expert-math-test-002","error":null,"utility":0.9999993301333332},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0015000000530563,"prompt_tokens":7,"completion_tokens":5,"generation_id":"mock-code_expert-math-test-002","error":null,"utility":-6.676666667020376e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0009999998837884,"prompt_tokens":7,"completion_tokens":5,"generation_id":"mock-general_expert-math-test-002","error":null,"utility":-6.67333333255859e-7}]}
+{"task_id":"code-test-001","prompt":"Which command initializes a new Git repository?","domain":"code","split":"test","tags":["git"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.698000000603012e-7,0.9999993324,-6.672666666878893e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0047000000904518,"prompt_tokens":7,"completion_tokens":5,"generation_id":"mock-math_expert-code-test-001","error":null,"utility":-6.698000000603012e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"git init","quality":1.0,"cost_usd":0.0,"latency_ms":1.001399999973728,"prompt_tokens":7,"completion_tokens":2,"generation_id":"mock-code_expert-code-test-001","error":null,"utility":0.9999993324},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0009000000318338,"prompt_tokens":7,"completion_tokens":5,"generation_id":"mock-general_expert-code-test-001","error":null,"utility":-6.672666666878893e-7}]}
+{"task_id":"code-test-002","prompt":"Which Python built-in returns the number of items in a collection?","domain":"code","split":"test","tags":["python"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.698666667798534e-7,0.9999993324,-6.67333333255859e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.00480000016978,"prompt_tokens":11,"completion_tokens":5,"generation_id":"mock-math_expert-code-test-002","error":null,"utility":-6.698666667798534e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"len","quality":1.0,"cost_usd":0.0,"latency_ms":1.001399999973728,"prompt_tokens":11,"completion_tokens":1,"generation_id":"mock-code_expert-code-test-002","error":null,"utility":0.9999993324},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0009999998837884,"prompt_tokens":11,"completion_tokens":5,"generation_id":"mock-general_expert-code-test-002","error":null,"utility":-6.67333333255859e-7}]}
+{"task_id":"general-test-001","prompt":"What is the capital of Canada?","domain":"general","split":"test","tags":["geography"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.69733333340749e-7,-6.675999999824853e-7,0.9999993326666666],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0046000000111235,"prompt_tokens":6,"completion_tokens":5,"generation_id":"mock-math_expert-general-test-001","error":null,"utility":-6.69733333340749e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.001399999973728,"prompt_tokens":6,"completion_tokens":5,"generation_id":"mock-code_expert-general-test-001","error":null,"utility":-6.675999999824853e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"Ottawa","quality":1.0,"cost_usd":0.0,"latency_ms":1.001000000111162,"prompt_tokens":6,"completion_tokens":1,"generation_id":"mock-general_expert-general-test-001","error":null,"utility":0.9999993326666666}]}
+{"task_id":"general-test-002","prompt":"Which instrument measures temperature?","domain":"general","split":"test","tags":["science"],"metadata":{},"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[-6.696666667727793e-7,-6.674666666949634e-7,0.9999993326666666],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"incorrect mock answer from math_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0045000001591688,"prompt_tokens":4,"completion_tokens":5,"generation_id":"mock-math_expert-general-test-002","error":null,"utility":-6.696666667727793e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.001200000042445,"prompt_tokens":4,"completion_tokens":5,"generation_id":"mock-code_expert-general-test-002","error":null,"utility":-6.674666666949634e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"thermometer","quality":1.0,"cost_usd":0.0,"latency_ms":1.001000000111162,"prompt_tokens":4,"completion_tokens":1,"generation_id":"mock-general_expert-general-test-002","error":null,"utility":0.9999993326666666}]}

data/rewards.openrouter-smoke.jsonl

@@ -0,0 +1 @@
+{"task_id":"math-001","prompt":"Compute 17 + 25. Return only the number.","domain":"math","split":"train","tags":["arithmetic"],"metadata":{},"worker_ids":["qwen_worker","deepseek_worker","gemini_worker"],"rewards":[1.0,1.0,1.0],"results":[{"worker_id":"qwen_worker","requested_model":"qwen/qwen3-8b","served_model":"qwen/qwen3-8b","response":"42","quality":1.0,"cost_usd":0.000159952,"latency_ms":5603.121014999488,"prompt_tokens":41,"completion_tokens":341,"generation_id":"gen-1785923963-ro7vM7ZUUQYoVoK5SFwC","error":null,"utility":1.0},{"worker_id":"deepseek_worker","requested_model":"deepseek/deepseek-v4-flash-0731","served_model":"deepseek/deepseek-v4-flash-0731","response":"42","quality":1.0,"cost_usd":0.00001064,"latency_ms":1512.0638320004218,"prompt_tokens":30,"completion_tokens":23,"generation_id":"gen-1785923963-OBp0xXrdiZDHhXOgkmkr","error":null,"utility":1.0},{"worker_id":"gemini_worker","requested_model":"google/gemini-3.1-flash-lite","served_model":"google/gemini-3.1-flash-lite","response":"42","quality":1.0,"cost_usd":0.00001075,"latency_ms":1517.3159320002014,"prompt_tokens":31,"completion_tokens":2,"generation_id":"gen-1785923963-ImgmfYNr0vCsY5Wm0ZnJ","error":null,"utility":1.0}]}

data/rewards.repeat-mock.jsonl

@@ -0,0 +1 @@
+{"task_id":"math-001","prompt":"Compute 17 + 25. Return only the number.","domain":"math","split":"train","tags":["arithmetic"],"metadata":{},"repetitions":3,"worker_ids":["math_expert","code_expert","general_expert"],"rewards":[0.9999993272222222,-6.674222227205368e-7,-6.671777772232114e-7],"results":[{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"42","quality":1.0,"cost_usd":0.0,"latency_ms":1.0230000005103648,"prompt_tokens":8,"completion_tokens":1,"generation_id":"mock-math_expert-math-001","error":null,"utility":0.9999993179999996},{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"42","quality":1.0,"cost_usd":0.0,"latency_ms":1.002900000254158,"prompt_tokens":8,"completion_tokens":1,"generation_id":"mock-math_expert-math-001","error":null,"utility":0.9999993313999999},{"worker_id":"math_expert","requested_model":"mock/math-expert","served_model":"mock/math-expert","response":"42","quality":1.0,"cost_usd":0.0,"latency_ms":1.0015999994502636,"prompt_tokens":8,"completion_tokens":1,"generation_id":"mock-math_expert-math-001","error":null,"utility":0.999999332266667},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0015000005078036,"prompt_tokens":8,"completion_tokens":5,"generation_id":"mock-code_expert-math-001","error":null,"utility":-6.676666670052025e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.001100001099985,"prompt_tokens":8,"completion_tokens":5,"generation_id":"mock-code_expert-math-001","error":null,"utility":-6.674000007333235e-7},{"worker_id":"code_expert","requested_model":"mock/code-expert","served_model":"mock/code-expert","response":"incorrect mock answer from code_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0008000006346265,"prompt_tokens":8,"completion_tokens":5,"generation_id":"mock-code_expert-math-001","error":null,"utility":-6.672000004230844e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.000799998815637,"prompt_tokens":8,"completion_tokens":5,"generation_id":"mock-general_expert-math-001","error":null,"utility":-6.671999992104247e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.0006999998731771,"prompt_tokens":8,"completion_tokens":5,"generation_id":"mock-general_expert-math-001","error":null,"utility":-6.671333332487848e-7},{"worker_id":"general_expert","requested_model":"mock/general-expert","served_model":"mock/general-expert","response":"incorrect mock answer from general_expert","quality":0.0,"cost_usd":0.0,"latency_ms":1.000799998815637,"prompt_tokens":8,"completion_tokens":5,"generation_id":"mock-general_expert-math-001","error":null,"utility":-6.671999992104247e-7}]}

data/rewards.schema.example.jsonl

@@ -0,0 +1,2 @@
+{"task_id":"example-1","prompt":"The exact task text seen by workers.","domain":"code","split":"train","tags":["python"],"metadata":{},"worker_ids":["worker_a","worker_b"],"rewards":[0.95,0.42],"results":[]}
+