{ "variant": "nar", "upstream": { "repo": "ibm-granite/granite-speech-4.1-2b-nar", "url": "https://huggingface.co/ibm-granite/granite-speech-4.1-2b-nar", "license": "Apache-2.0" }, "topology": "encoder + editor + embed_tokens (NAR bidirectional)", "graphs": [ { "name": "fp32/encoder.onnx", "sidecar": "fp32/encoder.onnx_data", "precision": "fp32", "size_bytes": 1068009, "sidecar_size_bytes": 2491061620, "opset": 20, "ir_version": 10, "ai_onnx_only": true, "inputs": [ { "name": "input_features", "shape": [ "B", "T", 160 ], "dtype": "float32" }, { "name": "attention_mask", "shape": [ "B", "T" ], "dtype": "int64" } ], "outputs": [ { "name": "char_logits", "shape": [ "B", "T_enc", 348 ], "dtype": "float32" }, { "name": "bpe_logits_dense", "shape": [ "B", "T_bpe", "V_bpe" ], "dtype": "float32" }, { "name": "bpe_mask", "shape": [ "B", "T_bpe" ], "dtype": "float32" }, { "name": "audio_embeds", "shape": [ "B", "T_audio", 2048 ], "dtype": "float32" }, { "name": "audio_lengths", "shape": [ "B" ], "dtype": "int64" } ] }, { "name": "int8/encoder.onnx", "sidecar": "int8/encoder.onnx_data", "precision": "int8-weights-only", "size_bytes": 3392476, "sidecar_size_bytes": 632400244, "opset": 20, "ir_version": 10, "ai_onnx_only": true, "inputs": [ { "name": "input_features", "shape": [ "B", "T", 160 ], "dtype": "float32" }, { "name": "attention_mask", "shape": [ "B", "T" ], "dtype": "int64" } ], "outputs": [ { "name": "char_logits", "shape": [ "B", "T_enc", 348 ], "dtype": "float32" }, { "name": "bpe_logits_dense", "shape": [ "B", "T_bpe", "V_bpe" ], "dtype": "float32" }, { "name": "bpe_mask", "shape": [ "B", "T_bpe" ], "dtype": "float32" }, { "name": "audio_embeds", "shape": [ "B", "T_audio", 2048 ], "dtype": "float32" }, { "name": "audio_lengths", "shape": [ "B" ], "dtype": "int64" } ] }, { "name": "fp16w/encoder.onnx", "sidecar": "fp16w/encoder.onnx_data", "precision": "fp16-weights-fp32-compute", "size_bytes": 1415006, "sidecar_size_bytes": 1245531506, "opset": 20, "ir_version": 10, "ai_onnx_only": true, "inputs": [ { "name": "input_features", "shape": [ "B", "T", 160 ], "dtype": "float32" }, { "name": "attention_mask", "shape": [ "B", "T" ], "dtype": "int64" } ], "outputs": [ { "name": "char_logits", "shape": [ "B", "T_enc", 348 ], "dtype": "float32" }, { "name": "bpe_logits_dense", "shape": [ "B", "T_bpe", "V_bpe" ], "dtype": "float32" }, { "name": "bpe_mask", "shape": [ "B", "T_bpe" ], "dtype": "float32" }, { "name": "audio_embeds", "shape": [ "B", "T_audio", 2048 ], "dtype": "float32" }, { "name": "audio_lengths", "shape": [ "B" ], "dtype": "int64" } ] }, { "name": "fp32/editor.onnx", "sidecar": "fp32/editor.onnx_data", "precision": "fp32", "size_bytes": 1838919, "sidecar_size_bytes": 6527000576, "opset": 20, "ir_version": 10, "ai_onnx_only": true, "inputs": [ { "name": "inputs_embeds", "shape": [ 1, "N", 2048 ], "dtype": "float32" }, { "name": "position_ids", "shape": [ 1, "N" ], "dtype": "int64" }, { "name": "attention_mask", "shape": [ 1, 1, "N", "N" ], "dtype": "float32" } ], "outputs": [ { "name": "logits", "shape": [ "B_out", "N", 100352 ], "dtype": "float32" } ] }, { "name": "int8/editor.onnx", "sidecar": "int8/editor.onnx_data", "precision": "int8-weights-only", "size_bytes": 6415968, "sidecar_size_bytes": 1632247808, "opset": 20, "ir_version": 10, "ai_onnx_only": true, "inputs": [ { "name": "inputs_embeds", "shape": [ 1, "N", 2048 ], "dtype": "float32" }, { "name": "position_ids", "shape": [ 1, "N" ], "dtype": "int64" }, { "name": "attention_mask", "shape": [ 1, 1, "N", "N" ], "dtype": "float32" } ], "outputs": [ { "name": "logits", "shape": [ "B_out", "N", 100352 ], "dtype": "float32" } ] }, { "name": "fp16w/editor.onnx", "sidecar": "fp16w/editor.onnx_data", "precision": "fp16-weights-fp32-compute", "size_bytes": 2571913, "sidecar_size_bytes": 3263500288, "opset": 20, "ir_version": 10, "ai_onnx_only": true, "inputs": [ { "name": "inputs_embeds", "shape": [ 1, "N", 2048 ], "dtype": "float32" }, { "name": "position_ids", "shape": [ 1, "N" ], "dtype": "int64" }, { "name": "attention_mask", "shape": [ 1, 1, "N", "N" ], "dtype": "float32" } ], "outputs": [ { "name": "logits", "shape": [ "B_out", "N", 100352 ], "dtype": "float32" } ] }, { "name": "fp32/embed_tokens.onnx", "sidecar": "fp32/embed_tokens.onnx_data", "precision": "fp32", "size_bytes": 336, "sidecar_size_bytes": 822083584, "opset": 20, "ir_version": 10, "ai_onnx_only": true, "inputs": [ { "name": "input_ids", "shape": [ "B", "N" ], "dtype": "int64" } ], "outputs": [ { "name": "inputs_embeds", "shape": [ "B", "N", 2048 ], "dtype": "float32" } ] }, { "name": "int8/embed_tokens.onnx", "sidecar": "int8/embed_tokens.onnx_data", "precision": "int8-weights-only", "size_bytes": 991, "sidecar_size_bytes": 205922304, "opset": 20, "ir_version": 10, "ai_onnx_only": true, "inputs": [ { "name": "input_ids", "shape": [ "B", "N" ], "dtype": "int64" } ], "outputs": [ { "name": "inputs_embeds", "shape": [ "B", "N", 2048 ], "dtype": "float32" } ] }, { "name": "fp16w/embed_tokens.onnx", "sidecar": "fp16w/embed_tokens.onnx_data", "precision": "fp16-weights-fp32-compute", "size_bytes": 432, "sidecar_size_bytes": 411041792, "opset": 20, "ir_version": 10, "ai_onnx_only": true, "inputs": [ { "name": "input_ids", "shape": [ "B", "N" ], "dtype": "int64" } ], "outputs": [ { "name": "inputs_embeds", "shape": [ "B", "N", 2048 ], "dtype": "float32" } ] } ], "parity": { "fp32": { "encoder": { "argmax_only": false, "ok": true, "char_logits": { "max_abs_err": 0.0007429122924804688, "mean_abs_err": 2.1003936126362532e-05, "argmax_mismatches": 0, "argmax_total": 843 }, "bpe_logits_dense": { "max_abs_err": 0.002044081687927246, "mean_abs_err": 3.075435233768076e-05, "argmax_mismatches": 0, "argmax_total": 211 }, "audio_embeds": { "max_abs_err": 2.384185791015625e-05, "mean_abs_err": 2.855549382729805e-06 }, "attention_mask_sum": 843 }, "editor": { "argmax_only": false, "ok": true, "max_abs_err": 0.00147247314453125, "mean_abs_err": 4.470655039767735e-05, "argmax_mismatches": 0, "argmax_total": 257, "transcript_match": true, "top1_match": 257, "top5_set_match": 257 } }, "int8": { "encoder": { "argmax_only": true, "ok": true, "char_logits": { "max_abs_err": 7.266112327575684, "mean_abs_err": 0.5243923664093018, "argmax_mismatches": 76, "argmax_total": 843 }, "bpe_logits_dense": { "max_abs_err": 1.8408704996109009, "mean_abs_err": 0.20938457548618317, "argmax_mismatches": 0, "argmax_total": 211 }, "audio_embeds": { "max_abs_err": 0.7828707695007324, "mean_abs_err": 0.06720388680696487 }, "attention_mask_sum": 843 }, "editor": { "argmax_only": true, "ok": true, "max_abs_err": 94.53038024902344, "mean_abs_err": 8.335189819335938, "argmax_mismatches": 15, "argmax_total": 257, "transcript_match": true, "top1_match": 242, "top5_set_match": 24 } } }, "multi_clip_parity": { "rows": [ { "name": "is-it-more-wood", "duration_s": 46.9, "fp32_byte_exact_vs_pt": true, "int8_byte_exact_vs_pt": false, "int8_wer_vs_pt": 0.0432, "int8_wer_norm_vs_pt": 0.0205, "int8_vs_fp32_lev": 13, "fp16w_byte_exact_vs_pt": true, "fp16w_wer_vs_pt": 0.0, "fp16w_wer_norm_vs_pt": 0.0, "fp32_transcript": "well, hello, sam. guess who? yeah, it's robert clotworthy, the narrator of your favorite television show, the curse of oak island. yes, i'm the is it possible? could it be? and what else do we say on oak island? a couple of words. they're not coming to me. oh, yeah, more wood. but let's not forget it is an island named after a tree. well, here's the question. why am i reaching out to you? is it possible that i'm reaching out to you because it's your birthday? could it be that emma let the cat out of the bag? well, the answer to those questions is yes. and she said, well, she contacted me. she said, robert, you know, sam is an amazing boyfriend. in fact, she used the word great. she said he is a great boyfriend.", "int8_transcript": "well, hello, sam. guess who? yeah, it's robert clotworthy, the narrator of your favorite television show, the curse of oak island. yes, i'm the, is it possible? could it be? and what else do we say on oak island? couple of words. they're not coming to me. oh, yeah, more would. let's not forget, it is an island named after a tree. well, here's the question. why am i reaching out to you? is it possible that i'm reaching out to you because it's your birthday? could it be that emma let the cat out of the bag? well, the answer to those questions is yes. and she said... well, she contacted me. she said, robert, you know, sam is an amazing boyfriend. in fact, she used the word great. she said he is a great boyfriend.", "fp16w_transcript": "well, hello, sam. guess who? yeah, it's robert clotworthy, the narrator of your favorite television show, the curse of oak island. yes, i'm the is it possible? could it be? and what else do we say on oak island? a couple of words. they're not coming to me. oh, yeah, more wood. but let's not forget it is an island named after a tree. well, here's the question. why am i reaching out to you? is it possible that i'm reaching out to you because it's your birthday? could it be that emma let the cat out of the bag? well, the answer to those questions is yes. and she said, well, she contacted me. she said, robert, you know, sam is an amazing boyfriend. in fact, she used the word great. she said he is a great boyfriend.", "pt_transcript": "well, hello, sam. guess who? yeah, it's robert clotworthy, the narrator of your favorite television show, the curse of oak island. yes, i'm the is it possible? could it be? and what else do we say on oak island? a couple of words. they're not coming to me. oh, yeah, more wood. but let's not forget it is an island named after a tree. well, here's the question. why am i reaching out to you? is it possible that i'm reaching out to you because it's your birthday? could it be that emma let the cat out of the bag? well, the answer to those questions is yes. and she said, well, she contacted me. she said, robert, you know, sam is an amazing boyfriend. in fact, she used the word great. she said he is a great boyfriend." }, { "name": "two-speakers-1", "duration_s": 93.8, "fp32_byte_exact_vs_pt": false, "int8_byte_exact_vs_pt": false, "int8_wer_vs_pt": 0.0351, "int8_wer_norm_vs_pt": 0.0172, "int8_vs_fp32_lev": 16, "fp16w_byte_exact_vs_pt": false, "fp16w_wer_vs_pt": 0.0035, "fp16w_wer_norm_vs_pt": 0.0034, "fp32_transcript": "today, it is a true honor to speak with demisavas, who is the ceo of deepmind. demis, welcome to the podcast. thanks for having me. first question, given your neuroscience background, how do you think about intelligence? specifically, do you think it's like one higher level general reasoning circuit, or do you think it's thousands of independent subkills and heuristics? well, it's interesting because intelligence is so broad and what we use it for is so sort of generally applicable. i think that suggests that you know there must be some sort of high level common things, you know common kind of algorithmic themes, i think, around how the brain processes the world around us. so of course, then there are specialized parts of the brain that do specific things, but i think there are probably some underlying principles that underpin all of that. yeah. how do you make sense of the fact that in these llms, though, when you give them a lot of data in any specific domain, they tend to get asymmetrically better in that domain? wouldn't we expect a sort of like general improvement across all the different areas? well, i think you, first of all, i think you do actually sometimes get surprising improvement in other domains when you improve in a specific domain. so, for example, when these large models sort of improve at coding, that can actually improve their general reasoning. so there is some evidence of some transfer, although i think we would like a lot more evidence of that. but also, you know, that's how the human brain learns too, is if we experience and practice a lot of things like chess or, you know, writing.", "int8_transcript": "today, it is a true honor to speak with demis avas, who is the ceo of deepmind. demis, welcome to the podcast. thanks for having me. first question, given your neuroscience background, how do you think about intelligence? specifically, do you think it's like one higher level general reasoning circuit, or do you think it's thousands of independent subskills and heuristics? well, it's interesting because intelligence is so broad and, know what we use it for is so sort of generally applicable. i think that suggests that, you know there must be some sort of high level common things you know common kind of algorithmic themes, i think, around how the brain processes the world around us. so, of course, then there are specialized parts of the brain that do specific things, but i think there are probably some underlying principles that underpin all of that. yeah. how do you make sense of the fact that in these lms, though, when you give them a lot of data in any specific domain, they tend to get asymmetrically better in that domain? wouldn't we expect a sort of like general improvement across all the different areas? well, i think, first of all, i think you do actually sometimes get surprising improvement in other domains when you improve in a specific domain. so, for example, when these large models sort of improve at coding, that can actually improve their general reasoning. so there is some evidence of some transfer, although i think we would like a lot more evidence of that. but also, you know, that's how the human brain learns too, is if we experience and practice a lot of things like chess or, you know, writing.", "fp16w_transcript": "today, it is a true honor to speak with demisavas, who is the ceo of deepmind. demis, welcome to the podcast. thanks for having me. first question, given your neuroscience background, how do you think about intelligence? specifically, do you think it's like one higher level general reasoning circuit, or do you think it's thousands of independent subkills and heuristics? well, it's interesting because intelligence is so broad and what we use it for is so sort of generally applicable. i think that suggests that you know there must be some sort of high level common things, you know common kind of algorithmic themes, i think, around how the brain processes the world around us. so of course, then there are specialized parts of the brain that do specific things, but i think there are probably some underlying principles that underpin all of that. yeah. how do you make sense of the fact that in these llms, though, when you give them a lot of data in any specific domain, they tend to get asymmetrically better in that domain? wouldn't we expect a sort of like general improvement across all the different areas? well, i think you, first of all, i think you do actually sometimes get surprising improvement in other domains when you improve in a specific domain. so, for example, when these large models sort of improve at coding, that can actually improve their general reasoning. so there is some evidence of some transfer, although i think we would like a lot more evidence of that. but also, you know, that's how the human brain learns too, is if we experience and practice a lot of things like chess or, you know, writing.", "pt_transcript": "today, it is a true honor to speak with demisavas, who is the ceo of deepmind. demis, welcome to the podcast. thanks for having me. first question, given your neuroscience background, how do you think about intelligence? specifically, do you think it's like one higher level general reasoning circuit, or do you think it's thousands of independent subkills and heuristics? well, it's interesting because intelligence is so broad and what we use it for is so sort of generally applicable. i think that suggests that you know there must be some sort of high level common things, you know common kind of algorithmic themes, i think, around how the brain processes the world around us. so of course, then there are specialized parts of the brain that do specific things, but i think there are probably some underlying principles that underpin all of that. yeah. how do you make sense of the fact that in these lms, though, when you give them a lot of data in any specific domain, they tend to get asymmetrically better in that domain? wouldn't we expect a sort of like general improvement across all the different areas? well, i think you, first of all, i think you do actually sometimes get surprising improvement in other domains when you improve in a specific domain. so, for example, when these large models sort of improve at coding, that can actually improve their general reasoning. so there is some evidence of some transfer, although i think we would like a lot more evidence of that. but also, you know, that's how the human brain learns too, is if we experience and practice a lot of things like chess or, you know, writing." }, { "name": "two-speakers-2", "duration_s": 38.8, "fp32_byte_exact_vs_pt": true, "int8_byte_exact_vs_pt": false, "int8_wer_vs_pt": 0.0505, "int8_wer_norm_vs_pt": 0.0096, "int8_vs_fp32_lev": 7, "fp16w_byte_exact_vs_pt": true, "fp16w_wer_vs_pt": 0.0, "fp16w_wer_norm_vs_pt": 0.0, "fp32_transcript": "for the first time ever, we may have things more intelligent than us. you believe they can understand? yes. you believe they are intelligent? yes. you believe these systems have experiences of their own and can make decisions based on those experiences? in the same sense as people do, yes. are they conscious? i think they probably don't have much self--awareness at present. so in that sense, i don't think they're conscious. will they have self-awareness? yes.. oh, yes, i think they will in time. and so human beings will be the second most intelligent beings on the planet.", "int8_transcript": "for the first time ever, we may have things more intelligent than us. you believe they can understand? yes. you believe they are intelligent? yes. you believe these systems have experiences of their own and can make decisions based on those experiences. in the same sense as people do, yes. are they conscious? i think they probably don't have much self--awareness at present. so in that sense, i don't think they're conscious. will they have self--awareness? yes. i yes. i think they will in time. and so human beings will be the second most intelligent beings on the planet.", "fp16w_transcript": "for the first time ever, we may have things more intelligent than us. you believe they can understand? yes. you believe they are intelligent? yes. you believe these systems have experiences of their own and can make decisions based on those experiences? in the same sense as people do, yes. are they conscious? i think they probably don't have much self--awareness at present. so in that sense, i don't think they're conscious. will they have self-awareness? yes.. oh, yes, i think they will in time. and so human beings will be the second most intelligent beings on the planet.", "pt_transcript": "for the first time ever, we may have things more intelligent than us. you believe they can understand? yes. you believe they are intelligent? yes. you believe these systems have experiences of their own and can make decisions based on those experiences? in the same sense as people do, yes. are they conscious? i think they probably don't have much self--awareness at present. so in that sense, i don't think they're conscious. will they have self-awareness? yes.. oh, yes, i think they will in time. and so human beings will be the second most intelligent beings on the planet." } ] }, "toolchain": { "transformers": "5.8.0", "torch": "2.11.0", "onnx": "1.21.0", "onnxruntime": "1.25.1", "exporter": "torch.onnx.export TorchScript path (dynamo=False)" }, "ort_compatibility": "ort 2.0-rc.x (Rust crate); validated against onnxruntime 1.17 - 1.25", "audio_token_id": 100352 }