| { |
| "settings": { |
| "stock-final-c1": { |
| "run_output_tokens_s": [ |
| 104.53250339797597, |
| 104.30634725774433 |
| ], |
| "aggregate_output_tokens_s": 104.419302873155, |
| "completed_requests_s": 0.40788790184826174, |
| "requests": 32, |
| "input_tokens": 7524, |
| "output_tokens": 8192, |
| "peak_sampled_vram_bytes": 21532827648, |
| "telemetry_samples": 158, |
| "e2e_s": { |
| "n": 32, |
| "mean": 2.45127607462382, |
| "p50": 2.450226479471894, |
| "p90": 2.455984840623569, |
| "p95": 2.4575956087763187, |
| "p99": 2.457689605660271, |
| "min": 2.4440573300234973, |
| "max": 2.4577168679679744 |
| }, |
| "ttft_s": { |
| "n": 32, |
| "mean": 0.08590561302480637, |
| "p50": 0.08608527900651097, |
| "p90": 0.0866674986667931, |
| "p95": 0.08668633470661007, |
| "p99": 0.0869625799235655, |
| "min": 0.08462987397797406, |
| "max": 0.08708392601693049 |
| }, |
| "tpot_s": { |
| "n": 32, |
| "mean": 0.009275962594505936, |
| "p50": 0.009272217666661805, |
| "p90": 0.009295140871750738, |
| "p95": 0.009297683427724348, |
| "p99": 0.009298357449595214, |
| "min": 0.009252656690374601, |
| "max": 0.00929861241180048 |
| }, |
| "output_tokens_s": { |
| "n": 32, |
| "mean": 104.43562038582411, |
| "p50": 104.48013787206841, |
| "p90": 104.58091092417986, |
| "p95": 104.63651680403437, |
| "p99": 104.71985722296866, |
| "min": 104.16171339201462, |
| "max": 104.74386048773201 |
| }, |
| "decode_tokens_s": { |
| "n": 32, |
| "mean": 107.80757244235349, |
| "p50": 107.85071689616248, |
| "p90": 107.97120816019948, |
| "p95": 108.01162285149796, |
| "p99": 108.06927831302553, |
| "min": 107.54460746433084, |
| "max": 108.07876539200603 |
| }, |
| "telemetry": { |
| "temp1_input": { |
| "n": 158, |
| "mean": 58405.06329113924, |
| "p50": 58000.0, |
| "p90": 60000.0, |
| "p95": 60000.0, |
| "p99": 60000.0, |
| "min": 39000, |
| "max": 60000 |
| }, |
| "temp2_input": { |
| "n": 158, |
| "mean": 83784.81012658228, |
| "p50": 84000.0, |
| "p90": 85000.0, |
| "p95": 85000.0, |
| "p99": 86000.0, |
| "min": 51000, |
| "max": 86000 |
| }, |
| "temp3_input": { |
| "n": 158, |
| "mean": 73082.27848101266, |
| "p50": 74000.0, |
| "p90": 76000.0, |
| "p95": 76000.0, |
| "p99": 76000.0, |
| "min": 43000, |
| "max": 76000 |
| }, |
| "freq1_input": { |
| "n": 158, |
| "mean": 3154449367.088608, |
| "p50": 3190000000.0, |
| "p90": 3195000000.0, |
| "p95": 3196150000.0, |
| "p99": 3199430000.0, |
| "min": 1151000000, |
| "max": 3201000000 |
| }, |
| "freq2_input": { |
| "n": 158, |
| "mean": 1257151898.734177, |
| "p50": 1258000000.0, |
| "p90": 1258000000.0, |
| "p95": 1258000000.0, |
| "p99": 1258000000.0, |
| "min": 1124000000, |
| "max": 1258000000 |
| }, |
| "power1_average": { |
| "n": 158, |
| "mean": 296196202.5316456, |
| "p50": 299000000.0, |
| "p90": 300000000.0, |
| "p95": 300000000.0, |
| "p99": 302000000.0, |
| "min": 76000000, |
| "max": 302000000 |
| } |
| } |
| }, |
| "paiton-final-c1": { |
| "run_output_tokens_s": [ |
| 126.7482161946917, |
| 126.61164669857416 |
| ], |
| "aggregate_output_tokens_s": 126.67989463885475, |
| "completed_requests_s": 0.49484333843302636, |
| "requests": 32, |
| "input_tokens": 7524, |
| "output_tokens": 8192, |
| "peak_sampled_vram_bytes": 21563035648, |
| "telemetry_samples": 130, |
| "e2e_s": { |
| "n": 32, |
| "mean": 2.0204912018471077, |
| "p50": 2.0204291350091808, |
| "p90": 2.0219247136905323, |
| "p95": 2.0233954027033176, |
| "p99": 2.0276054632267915, |
| "min": 2.0170277759898454, |
| "max": 2.0289841779740527 |
| }, |
| "ttft_s": { |
| "n": 32, |
| "mean": 0.08601805344005697, |
| "p50": 0.08624113749829121, |
| "p90": 0.08684968375018798, |
| "p95": 0.08692604756506625, |
| "p99": 0.08708030231355224, |
| "min": 0.08451331598917022, |
| "max": 0.08710829500341788 |
| }, |
| "tpot_s": { |
| "n": 32, |
| "mean": 0.007586169209439415, |
| "p50": 0.007583975192163979, |
| "p90": 0.0075936817341283255, |
| "p95": 0.007597326572320205, |
| "p99": 0.007611751111756152, |
| "min": 0.007574955266717748, |
| "max": 0.007617579866657216 |
| }, |
| "output_tokens_s": { |
| "n": 32, |
| "mean": 126.70200492982798, |
| "p50": 126.70575566781284, |
| "p90": 126.82481146128272, |
| "p95": 126.90434004800989, |
| "p99": 126.91782245956921, |
| "min": 126.171511231604, |
| "max": 126.91942225454451 |
| }, |
| "decode_tokens_s": { |
| "n": 32, |
| "mean": 131.82178280828202, |
| "p50": 131.85967507508565, |
| "p90": 131.94757658227738, |
| "p95": 131.95905111349347, |
| "p99": 132.000468370036, |
| "min": 131.2782823035353, |
| "max": 132.01679797240226 |
| }, |
| "telemetry": { |
| "temp1_input": { |
| "n": 130, |
| "mean": 58407.692307692305, |
| "p50": 58000.0, |
| "p90": 60000.0, |
| "p95": 60000.0, |
| "p99": 60000.0, |
| "min": 45000, |
| "max": 60000 |
| }, |
| "temp2_input": { |
| "n": 130, |
| "mean": 83253.84615384616, |
| "p50": 83000.0, |
| "p90": 85000.0, |
| "p95": 85000.0, |
| "p99": 85710.0, |
| "min": 63000, |
| "max": 86000 |
| }, |
| "temp3_input": { |
| "n": 130, |
| "mean": 73000, |
| "p50": 74000.0, |
| "p90": 75000.0, |
| "p95": 76000.0, |
| "p99": 76000.0, |
| "min": 50000, |
| "max": 76000 |
| }, |
| "freq1_input": { |
| "n": 130, |
| "mean": 3227776923.076923, |
| "p50": 3265000000.0, |
| "p90": 3279100000.0, |
| "p95": 3283000000.0, |
| "p99": 3288130000.0, |
| "min": 2211000000, |
| "max": 3293000000 |
| }, |
| "freq2_input": { |
| "n": 130, |
| "mean": 1256969230.7692308, |
| "p50": 1258000000.0, |
| "p90": 1258000000.0, |
| "p95": 1258000000.0, |
| "p99": 1258000000.0, |
| "min": 1124000000, |
| "max": 1258000000 |
| }, |
| "power1_average": { |
| "n": 130, |
| "mean": 294484615.38461536, |
| "p50": 297000000.0, |
| "p90": 299100000.0, |
| "p95": 300000000.0, |
| "p99": 301000000.0, |
| "min": 187000000, |
| "max": 301000000 |
| } |
| } |
| }, |
| "stock-final-c2": { |
| "run_output_tokens_s": [ |
| 101.29990196967496, |
| 101.91403696866031 |
| ], |
| "aggregate_output_tokens_s": 101.60604147722242, |
| "completed_requests_s": 0.3968985995204001, |
| "requests": 32, |
| "input_tokens": 7524, |
| "output_tokens": 8192, |
| "peak_sampled_vram_bytes": 21559439360, |
| "telemetry_samples": 162, |
| "e2e_s": { |
| "n": 32, |
| "mean": 5.037850869217436, |
| "p50": 5.021176078473218, |
| "p90": 5.141629918292165, |
| "p95": 5.142242345429258, |
| "p99": 5.142400043794186, |
| "min": 4.897444790985901, |
| "max": 5.142469321959652 |
| }, |
| "ttft_s": { |
| "n": 32, |
| "mean": 0.17167586559116899, |
| "p50": 0.16551451152190566, |
| "p90": 0.1681803280953318, |
| "p95": 0.2726705030450829, |
| "p99": 0.4010564773745136, |
| "min": 0.13795678497990593, |
| "max": 0.40174011199269444 |
| }, |
| "tpot_s": { |
| "n": 32, |
| "mean": 0.01908303922990693, |
| "p50": 0.019040175582346157, |
| "p90": 0.019494635844260786, |
| "p95": 0.0195414327638904, |
| "p99": 0.019592890214257162, |
| "min": 0.018565450615578277, |
| "max": 0.019595309949078252 |
| }, |
| "output_tokens_s": { |
| "n": 32, |
| "mean": 50.82458721518752, |
| "p50": 50.984072669266574, |
| "p90": 51.410113251573286, |
| "p95": 51.76038175886279, |
| "p99": 52.24342206711506, |
| "min": 49.781531784120695, |
| "max": 52.27215638473075 |
| }, |
| "decode_tokens_s": { |
| "n": 32, |
| "mean": 52.414180313215645, |
| "p50": 52.521024706967715, |
| "p90": 53.51770768960344, |
| "p95": 53.79832409114189, |
| "p99": 53.84681548466302, |
| "min": 51.03301061088222, |
| "max": 53.86393737357884 |
| }, |
| "telemetry": { |
| "temp1_input": { |
| "n": 162, |
| "mean": 58753.08641975309, |
| "p50": 59000.0, |
| "p90": 59000.0, |
| "p95": 59000.0, |
| "p99": 59000.0, |
| "min": 58000, |
| "max": 59000 |
| }, |
| "temp2_input": { |
| "n": 162, |
| "mean": 82444.44444444444, |
| "p50": 83000.0, |
| "p90": 83000.0, |
| "p95": 83000.0, |
| "p99": 83000.0, |
| "min": 73000, |
| "max": 83000 |
| }, |
| "temp3_input": { |
| "n": 162, |
| "mean": 73197.53086419753, |
| "p50": 73000.0, |
| "p90": 74000.0, |
| "p95": 74000.0, |
| "p99": 74000.0, |
| "min": 72000, |
| "max": 75000 |
| }, |
| "freq1_input": { |
| "n": 162, |
| "mean": 3240216049.382716, |
| "p50": 3280000000.0, |
| "p90": 3285000000.0, |
| "p95": 3285950000.0, |
| "p99": 3286390000.0, |
| "min": 1941000000, |
| "max": 3288000000 |
| }, |
| "freq2_input": { |
| "n": 162, |
| "mean": 1257172839.506173, |
| "p50": 1258000000.0, |
| "p90": 1258000000.0, |
| "p95": 1258000000.0, |
| "p99": 1258000000.0, |
| "min": 1124000000, |
| "max": 1258000000 |
| }, |
| "power1_average": { |
| "n": 162, |
| "mean": 292740740.7407407, |
| "p50": 295000000.0, |
| "p90": 298000000.0, |
| "p95": 299000000.0, |
| "p99": 300390000.0, |
| "min": 168000000, |
| "max": 302000000 |
| } |
| } |
| }, |
| "paiton-final-c2": { |
| "run_output_tokens_s": [ |
| 172.9216995128759, |
| 172.72773043576697 |
| ], |
| "aggregate_output_tokens_s": 172.82466054923114, |
| "completed_requests_s": 0.6750963302704341, |
| "requests": 32, |
| "input_tokens": 7524, |
| "output_tokens": 8192, |
| "peak_sampled_vram_bytes": 21563035648, |
| "telemetry_samples": 96, |
| "e2e_s": { |
| "n": 32, |
| "mean": 2.961399451123725, |
| "p50": 2.961522576515563, |
| "p90": 2.9646852039149962, |
| "p95": 2.964837046136381, |
| "p99": 2.96495436332887, |
| "min": 2.9523183770361356, |
| "max": 2.9649825469823554 |
| }, |
| "ttft_s": { |
| "n": 32, |
| "mean": 0.15564254137825628, |
| "p50": 0.15792722153128125, |
| "p90": 0.1674851266143378, |
| "p95": 0.16758953527896664, |
| "p99": 0.16771430715918542, |
| "min": 0.13778713200008497, |
| "max": 0.1677317310241051 |
| }, |
| "tpot_s": { |
| "n": 32, |
| "mean": 0.011002968273511642, |
| "p50": 0.010995687492073095, |
| "p90": 0.011052764856402634, |
| "p95": 0.011055382690209822, |
| "p99": 0.011057267765044326, |
| "min": 0.010952511007724075, |
| "max": 0.011057785427307381 |
| }, |
| "output_tokens_s": { |
| "n": 32, |
| "mean": 86.44569888506376, |
| "p50": 86.44202211812255, |
| "p90": 86.5361364181114, |
| "p95": 86.5907744343187, |
| "p99": 86.69320834533765, |
| "min": 86.34114904337191, |
| "max": 86.71151525906944 |
| }, |
| "decode_tokens_s": { |
| "n": 32, |
| "mean": 90.8871974927982, |
| "p50": 90.94656225027069, |
| "p90": 91.29696533357941, |
| "p95": 91.30182692577216, |
| "p99": 91.30392198020051, |
| "min": 90.43523834314854, |
| "max": 91.30455862155868 |
| }, |
| "telemetry": { |
| "temp1_input": { |
| "n": 96, |
| "mean": 57979.166666666664, |
| "p50": 58000.0, |
| "p90": 58000.0, |
| "p95": 58000.0, |
| "p99": 58000.0, |
| "min": 57000, |
| "max": 58000 |
| }, |
| "temp2_input": { |
| "n": 96, |
| "mean": 84729.16666666667, |
| "p50": 85000.0, |
| "p90": 85000.0, |
| "p95": 85000.0, |
| "p99": 85000.0, |
| "min": 73000, |
| "max": 85000 |
| }, |
| "temp3_input": { |
| "n": 96, |
| "mean": 75916.66666666667, |
| "p50": 76000.0, |
| "p90": 76000.0, |
| "p95": 76000.0, |
| "p99": 76000.0, |
| "min": 74000, |
| "max": 76000 |
| }, |
| "freq1_input": { |
| "n": 96, |
| "mean": 3050895833.3333335, |
| "p50": 3090000000.0, |
| "p90": 3093000000.0, |
| "p95": 3094250000.0, |
| "p99": 3101000000.0, |
| "min": 1703000000, |
| "max": 3101000000 |
| }, |
| "freq2_input": { |
| "n": 96, |
| "mean": 1256604166.6666667, |
| "p50": 1258000000.0, |
| "p90": 1258000000.0, |
| "p95": 1258000000.0, |
| "p99": 1258000000.0, |
| "min": 1124000000, |
| "max": 1258000000 |
| }, |
| "power1_average": { |
| "n": 96, |
| "mean": 297468750, |
| "p50": 299000000.0, |
| "p90": 300000000.0, |
| "p95": 301000000.0, |
| "p99": 301050000.0, |
| "min": 140000000, |
| "max": 302000000 |
| } |
| } |
| } |
| }, |
| "comparisons": { |
| "1": { |
| "throughput_gain_percent": 21.31846426205426, |
| "mean_latency_change_percent": -17.573902720966338, |
| "common_generated_prefix_tokens": { |
| "n": 32, |
| "mean": 129.8125, |
| "p50": 115.0, |
| "p90": 256.0, |
| "p95": 256.0, |
| "p99": 256.0, |
| "min": 13, |
| "max": 256 |
| }, |
| "common_prefix_logprob_abs_difference": { |
| "n": 4154, |
| "mean": 0.00962247806280263, |
| "p50": 1.6689248241164023e-06, |
| "p90": 0.027043122053146403, |
| "p95": 0.06337008178234087, |
| "p99": 0.14746004521847003, |
| "min": 0.0, |
| "max": 0.3880656957626343 |
| }, |
| "identical_tokens_and_logprobs_requests": 0, |
| "scope": "Common-prefix logprobs only; not a full-vocabulary or teacher-forced numerical equivalence test." |
| }, |
| "2": { |
| "throughput_gain_percent": 70.092898056632, |
| "mean_latency_change_percent": -41.217008442654844, |
| "common_generated_prefix_tokens": { |
| "n": 32, |
| "mean": 89.75, |
| "p50": 78.0, |
| "p90": 236.60000000000014, |
| "p95": 248.85, |
| "p99": 256.0, |
| "min": 4, |
| "max": 256 |
| }, |
| "common_prefix_logprob_abs_difference": { |
| "n": 2872, |
| "mean": 0.009687470123258648, |
| "p50": 3.278164513176307e-06, |
| "p90": 0.03029351122677327, |
| "p95": 0.06604001820087418, |
| "p99": 0.1236238181591034, |
| "min": 0.0, |
| "max": 0.4361233711242676 |
| }, |
| "identical_tokens_and_logprobs_requests": 0, |
| "scope": "Common-prefix logprobs only; not a full-vocabulary or teacher-forced numerical equivalence test." |
| } |
| } |
| } |
|
|
