| schema_version: 'v1.2' |
| pipeline: |
| workflow: |
| manifest: |
| capabilities: |
| - workflow_ssa |
| - linear_effects |
| - nested_control_flow |
| - typed_emit |
| - emit_valid_length |
| - loop_induction_values |
| - serving_service_contract |
| - bounded_state_recurrence |
| inputs: |
| request.input_ids: |
| contract: |
| dtype: int64 |
| rank: 2 |
| shape: |
| - batch |
| - sequence_len |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| role: |
| kind: runtime |
| version: '1.0' |
| role: prompt_tokens |
| source: |
| kind: request |
| required: true |
| request.max_iterations: |
| contract: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - 1 |
| role: |
| kind: runtime |
| version: '1.0' |
| role: max_output_tokens |
| source: |
| kind: request |
| required: true |
| package.one_token: |
| contract: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| role: |
| kind: opaque |
| source: |
| kind: literal |
| required: false |
| default: 1 |
| package.one_step: |
| contract: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - 1 |
| role: |
| kind: opaque |
| source: |
| kind: literal |
| required: false |
| default: 1 |
| package.max_context: |
| contract: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - 1 |
| role: |
| kind: opaque |
| source: |
| kind: literal |
| required: false |
| default: 32768 |
| request.prompt_lengths: |
| contract: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| role: |
| kind: opaque |
| source: |
| kind: application |
| name: prompt_lengths |
| required: false |
| default: -1 |
| request.eos_ids: |
| contract: |
| dtype: int64 |
| rank: 2 |
| shape: |
| - batch |
| - num_eos |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| role: |
| kind: runtime |
| version: '1.0' |
| role: eos_token_ids |
| source: |
| kind: request |
| required: true |
| request.eos_lengths: |
| contract: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| role: |
| kind: runtime |
| version: '1.0' |
| role: eos_token_lengths |
| source: |
| kind: request |
| required: true |
| request.row_max_iterations: |
| contract: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| role: |
| kind: opaque |
| source: |
| kind: application |
| name: row_max_iterations |
| required: false |
| default: -1 |
| request.temperature: |
| contract: |
| dtype: float32 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| role: |
| kind: runtime |
| version: '1.0' |
| role: sampling_temperature |
| source: |
| kind: request |
| required: false |
| default: 1.0 |
| request.top_k: |
| contract: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| role: |
| kind: runtime |
| version: '1.0' |
| role: sampling_top_k |
| source: |
| kind: request |
| required: false |
| default: 1 |
| request.top_p: |
| contract: |
| dtype: float32 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| role: |
| kind: runtime |
| version: '1.0' |
| role: sampling_top_p |
| source: |
| kind: request |
| required: false |
| default: 1.0 |
| request.min_p: |
| contract: |
| dtype: float32 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| role: |
| kind: runtime |
| version: '1.0' |
| role: sampling_min_p |
| source: |
| kind: request |
| required: false |
| default: 0.0 |
| request.seed: |
| contract: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| role: |
| kind: runtime |
| version: '1.0' |
| role: seed |
| source: |
| kind: request |
| required: false |
| default: 0 |
| request.rng_counter: |
| contract: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| role: |
| kind: opaque |
| source: |
| kind: application |
| name: rng_counter |
| required: false |
| default: 0 |
| package.active: |
| contract: |
| dtype: bool |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| role: |
| kind: opaque |
| source: |
| kind: literal |
| required: false |
| default: true |
| package.not_done: |
| contract: |
| dtype: bool |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| role: |
| kind: opaque |
| source: |
| kind: literal |
| required: false |
| default: false |
| package.cache_lengths: |
| contract: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| role: |
| kind: opaque |
| source: |
| kind: literal |
| required: false |
| default: 0 |
| package.zero_batch: |
| contract: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| role: |
| kind: opaque |
| source: |
| kind: literal |
| required: false |
| default: 0 |
| outputs: |
| tokens: |
| contract: |
| dtype: int64 |
| rank: 2 |
| shape: |
| - batch |
| - generated_sequence |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| role: tokens |
| stage: pre_adapter |
| components: |
| model: |
| implementation: |
| kind: onnx |
| artifact: model.onnx |
| ports: |
| roles: |
| input_ids: token_ids |
| attention_mask: attention_mask |
| logits: logits |
| token_sampler: |
| implementation: |
| kind: onnx |
| artifact: policies/token_sampler.onnx |
| ports: |
| inputs: |
| logits: |
| dtype: float32 |
| rank: 2 |
| shape: |
| - batch |
| - vocabulary |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| temperature: |
| dtype: float32 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| top_k: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| top_p: |
| dtype: float32 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| min_p: |
| dtype: float32 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| seed: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| counter: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| active: |
| dtype: bool |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| done: |
| dtype: bool |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| outputs: |
| token: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| next_counter: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| contract: |
| id: onnx-genai.token-sampler |
| version: '2' |
| bindings: |
| logits: logits |
| token: token |
| temperature: temperature |
| top_k: top_k |
| top_p: top_p |
| min_p: min_p |
| active: active |
| done: done |
| seed: seed |
| counter: counter |
| next_counter: next_counter |
| parameters: |
| mode: seeded_stochastic |
| batching: per_row |
| inactive_rows: preserve |
| application_overridable: true |
| termination: |
| implementation: |
| kind: onnx |
| artifact: policies/termination.onnx |
| ports: |
| inputs: |
| tokens: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| eos_ids: |
| dtype: int64 |
| rank: 2 |
| shape: |
| - batch |
| - num_eos |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| eos_lengths: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| iteration: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - 1 |
| max_iterations: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| active: |
| dtype: bool |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| outputs: |
| done: |
| dtype: bool |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| next_active: |
| dtype: bool |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| continue: |
| dtype: bool |
| rank: 1 |
| shape: |
| - 1 |
| contract: |
| id: onnx-genai.termination-predicate |
| version: '2' |
| bindings: |
| tokens: tokens |
| eos_ids: eos_ids |
| iteration: iteration |
| max_iterations: max_iterations |
| eos_lengths: eos_lengths |
| active: active |
| done: done |
| next_active: next_active |
| continue: continue |
| parameters: |
| batching: per_row |
| inactive_rows: preserve |
| token_state_update: |
| implementation: |
| kind: onnx |
| artifact: policies/token_state_update.onnx |
| ports: |
| inputs: |
| current: |
| dtype: int64 |
| rank: 2 |
| shape: |
| - batch |
| - 1 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| update: |
| dtype: int64 |
| rank: 2 |
| shape: |
| - batch |
| - 1 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| active: |
| dtype: bool |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| done: |
| dtype: bool |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| outputs: |
| next: |
| dtype: int64 |
| rank: 2 |
| shape: |
| - batch |
| - 1 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| contract: |
| id: onnx-genai.state-update |
| version: '2' |
| bindings: |
| current: current |
| update: update |
| active: active |
| done: done |
| next: next |
| parameters: |
| batching: per_row |
| inactive_rows: preserve |
| last_token_logits: |
| implementation: |
| kind: onnx |
| artifact: policies/last_token_logits.onnx |
| ports: |
| inputs: |
| logits: |
| dtype: float16 |
| rank: 3 |
| shape: |
| - batch |
| - sequence |
| - vocabulary |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| outputs: |
| last_logits: |
| dtype: float32 |
| rank: 2 |
| shape: |
| - batch |
| - vocabulary |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| decoder_state_initializer: |
| implementation: |
| kind: onnx |
| artifact: policies/decoder_state_initializer.onnx |
| ports: |
| inputs: |
| prompt_tokens: |
| dtype: int64 |
| rank: 2 |
| shape: |
| - batch |
| - prompt_sequence |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| prompt_lengths: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| max_iterations: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - 1 |
| outputs: |
| attention_mask: |
| dtype: int64 |
| rank: 2 |
| shape: |
| - batch |
| - capacity |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| body_attention_mask: |
| dtype: int64 |
| rank: 2 |
| shape: |
| - batch |
| - capacity |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| token_slot: |
| dtype: int64 |
| rank: 2 |
| shape: |
| - batch |
| - 1 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| generated_lengths: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| cache_lengths: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.0.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.0.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.1.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.1.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.10.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.10.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.11.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.11.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.12.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.12.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.13.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.13.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.14.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.14.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.15.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.15.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.16.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.16.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.17.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.17.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.18.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.18.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.19.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.19.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.2.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.2.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.20.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.20.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.21.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.21.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.22.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.22.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.23.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.23.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.3.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.3.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.4.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.4.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.5.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.5.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.6.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.6.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.7.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.7.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.8.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.8.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.9.key: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| past_key_values.9.value: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - capacity |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| decoder_step_update: |
| implementation: |
| kind: onnx |
| artifact: policies/decoder_step_update.onnx |
| ports: |
| inputs: |
| attention_mask: |
| dtype: int64 |
| rank: 2 |
| shape: |
| - batch |
| - context |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| logical_length: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| outputs: |
| next_attention_mask: |
| dtype: int64 |
| rank: 2 |
| shape: |
| - batch |
| - context |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| cache_length_update: |
| implementation: |
| kind: onnx |
| artifact: policies/cache_length_update.onnx |
| ports: |
| inputs: |
| left: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| right: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| active: |
| dtype: bool |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| done: |
| dtype: bool |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| outputs: |
| total: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| termination_batch_initializer: |
| implementation: |
| kind: onnx |
| artifact: policies/termination_batch_initializer.onnx |
| ports: |
| inputs: |
| input_eos_ids: |
| dtype: int64 |
| rank: 2 |
| shape: |
| - batch |
| - num_eos |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| input_eos_lengths: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| input_max_iterations: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| fallback_max_iterations: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - 1 |
| active: |
| dtype: bool |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| outputs: |
| row_eos_ids: |
| dtype: int64 |
| rank: 2 |
| shape: |
| - batch |
| - num_eos |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| eos_lengths: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| max_iterations: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| token_to_slot: |
| implementation: |
| kind: onnx |
| artifact: policies/token_to_slot.onnx |
| ports: |
| inputs: |
| token: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| outputs: |
| slot: |
| dtype: int64 |
| rank: 2 |
| shape: |
| - batch |
| - 1 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| generated_length_update: |
| implementation: |
| kind: onnx |
| artifact: policies/generated_length_update.onnx |
| ports: |
| inputs: |
| left: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| right: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| active: |
| dtype: bool |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| done: |
| dtype: bool |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| outputs: |
| total: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| state: |
| token: |
| contract: |
| dtype: int64 |
| rank: 2 |
| shape: |
| - batch |
| - 1 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: initializer.token_slot |
| recurrence: |
| kind: invariant |
| logits: |
| contract: |
| dtype: float32 |
| rank: 2 |
| shape: |
| - batch |
| - 151936 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.last_logits |
| recurrence: |
| kind: invariant |
| generated_lengths: |
| contract: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| class: semantic |
| scope: invocation |
| initializer: initializer.generated_lengths |
| recurrence: |
| kind: invariant |
| active: |
| contract: |
| dtype: bool |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| class: semantic |
| scope: invocation |
| initializer: package.active |
| recurrence: |
| kind: invariant |
| done: |
| contract: |
| dtype: bool |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| class: semantic |
| scope: invocation |
| initializer: package.not_done |
| recurrence: |
| kind: invariant |
| accepted_len: |
| contract: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| class: semantic |
| scope: invocation |
| initializer: package.zero_batch |
| recurrence: |
| kind: invariant |
| cache_lengths: |
| contract: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| class: semantic |
| scope: invocation |
| initializer: initializer.cache_lengths |
| recurrence: |
| kind: invariant |
| rng_counter: |
| contract: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - batch |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| class: semantic |
| initializer: request.rng_counter |
| recurrence: |
| kind: invariant |
| attention_mask: |
| contract: |
| dtype: int64 |
| rank: 2 |
| shape: |
| - batch |
| - context |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: initializer.attention_mask |
| recurrence: |
| kind: invariant |
| cache_0: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.0.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_1: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.0.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_2: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.1.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_3: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.1.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_4: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.2.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_5: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.2.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_6: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.3.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_7: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.3.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_8: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.4.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_9: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.4.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_10: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.5.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_11: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.5.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_12: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.6.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_13: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.6.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_14: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.7.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_15: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.7.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_16: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.8.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_17: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.8.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_18: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.9.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_19: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.9.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_20: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.10.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_21: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.10.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_22: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.11.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_23: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.11.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_24: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.12.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_25: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.12.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_26: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.13.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_27: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.13.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_28: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.14.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_29: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.14.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_30: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.15.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_31: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.15.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_32: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.16.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_33: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.16.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_34: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.17.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_35: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.17.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_36: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.18.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_37: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.18.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_38: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.19.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_39: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.19.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_40: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.20.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_41: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.20.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_42: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.21.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_43: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.21.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_44: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.22.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_45: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.22.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_46: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.23.key |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| cache_47: |
| contract: |
| dtype: float16 |
| rank: 4 |
| shape: |
| - batch |
| - 2 |
| - past_sequence_len |
| - 64 |
| batch_layout: |
| kind: request_aligned |
| axis: 0 |
| scope: invocation |
| initializer: decoder.setup.present.23.value |
| recurrence: |
| kind: bounded |
| axis: 2 |
| max: package.max_context |
| management: runtime |
| release_boundary: invocation |
| service_group: decoder_cache |
| serving: |
| active: active |
| done: done |
| accepted_len: accepted_len |
| state_service: |
| groups: |
| decoder_cache: |
| kind: full_attention |
| sequence_axis: 2 |
| layout: bnsh |
| logical_lengths: cache_lengths |
| aliasing: permitted |
| reuse: |
| prefix_reusable: true |
| evictable_prefix: false |
| ports: |
| model: |
| cache_0: |
| input: past_key_values.0.key |
| output: present.0.key |
| role: key |
| layer: 0 |
| cache_1: |
| input: past_key_values.0.value |
| output: present.0.value |
| role: value |
| layer: 0 |
| cache_2: |
| input: past_key_values.1.key |
| output: present.1.key |
| role: key |
| layer: 1 |
| cache_3: |
| input: past_key_values.1.value |
| output: present.1.value |
| role: value |
| layer: 1 |
| cache_4: |
| input: past_key_values.2.key |
| output: present.2.key |
| role: key |
| layer: 2 |
| cache_5: |
| input: past_key_values.2.value |
| output: present.2.value |
| role: value |
| layer: 2 |
| cache_6: |
| input: past_key_values.3.key |
| output: present.3.key |
| role: key |
| layer: 3 |
| cache_7: |
| input: past_key_values.3.value |
| output: present.3.value |
| role: value |
| layer: 3 |
| cache_8: |
| input: past_key_values.4.key |
| output: present.4.key |
| role: key |
| layer: 4 |
| cache_9: |
| input: past_key_values.4.value |
| output: present.4.value |
| role: value |
| layer: 4 |
| cache_10: |
| input: past_key_values.5.key |
| output: present.5.key |
| role: key |
| layer: 5 |
| cache_11: |
| input: past_key_values.5.value |
| output: present.5.value |
| role: value |
| layer: 5 |
| cache_12: |
| input: past_key_values.6.key |
| output: present.6.key |
| role: key |
| layer: 6 |
| cache_13: |
| input: past_key_values.6.value |
| output: present.6.value |
| role: value |
| layer: 6 |
| cache_14: |
| input: past_key_values.7.key |
| output: present.7.key |
| role: key |
| layer: 7 |
| cache_15: |
| input: past_key_values.7.value |
| output: present.7.value |
| role: value |
| layer: 7 |
| cache_16: |
| input: past_key_values.8.key |
| output: present.8.key |
| role: key |
| layer: 8 |
| cache_17: |
| input: past_key_values.8.value |
| output: present.8.value |
| role: value |
| layer: 8 |
| cache_18: |
| input: past_key_values.9.key |
| output: present.9.key |
| role: key |
| layer: 9 |
| cache_19: |
| input: past_key_values.9.value |
| output: present.9.value |
| role: value |
| layer: 9 |
| cache_20: |
| input: past_key_values.10.key |
| output: present.10.key |
| role: key |
| layer: 10 |
| cache_21: |
| input: past_key_values.10.value |
| output: present.10.value |
| role: value |
| layer: 10 |
| cache_22: |
| input: past_key_values.11.key |
| output: present.11.key |
| role: key |
| layer: 11 |
| cache_23: |
| input: past_key_values.11.value |
| output: present.11.value |
| role: value |
| layer: 11 |
| cache_24: |
| input: past_key_values.12.key |
| output: present.12.key |
| role: key |
| layer: 12 |
| cache_25: |
| input: past_key_values.12.value |
| output: present.12.value |
| role: value |
| layer: 12 |
| cache_26: |
| input: past_key_values.13.key |
| output: present.13.key |
| role: key |
| layer: 13 |
| cache_27: |
| input: past_key_values.13.value |
| output: present.13.value |
| role: value |
| layer: 13 |
| cache_28: |
| input: past_key_values.14.key |
| output: present.14.key |
| role: key |
| layer: 14 |
| cache_29: |
| input: past_key_values.14.value |
| output: present.14.value |
| role: value |
| layer: 14 |
| cache_30: |
| input: past_key_values.15.key |
| output: present.15.key |
| role: key |
| layer: 15 |
| cache_31: |
| input: past_key_values.15.value |
| output: present.15.value |
| role: value |
| layer: 15 |
| cache_32: |
| input: past_key_values.16.key |
| output: present.16.key |
| role: key |
| layer: 16 |
| cache_33: |
| input: past_key_values.16.value |
| output: present.16.value |
| role: value |
| layer: 16 |
| cache_34: |
| input: past_key_values.17.key |
| output: present.17.key |
| role: key |
| layer: 17 |
| cache_35: |
| input: past_key_values.17.value |
| output: present.17.value |
| role: value |
| layer: 17 |
| cache_36: |
| input: past_key_values.18.key |
| output: present.18.key |
| role: key |
| layer: 18 |
| cache_37: |
| input: past_key_values.18.value |
| output: present.18.value |
| role: value |
| layer: 18 |
| cache_38: |
| input: past_key_values.19.key |
| output: present.19.key |
| role: key |
| layer: 19 |
| cache_39: |
| input: past_key_values.19.value |
| output: present.19.value |
| role: value |
| layer: 19 |
| cache_40: |
| input: past_key_values.20.key |
| output: present.20.key |
| role: key |
| layer: 20 |
| cache_41: |
| input: past_key_values.20.value |
| output: present.20.value |
| role: value |
| layer: 20 |
| cache_42: |
| input: past_key_values.21.key |
| output: present.21.key |
| role: key |
| layer: 21 |
| cache_43: |
| input: past_key_values.21.value |
| output: present.21.value |
| role: value |
| layer: 21 |
| cache_44: |
| input: past_key_values.22.key |
| output: present.22.key |
| role: key |
| layer: 22 |
| cache_45: |
| input: past_key_values.22.value |
| output: present.22.value |
| role: value |
| layer: 22 |
| cache_46: |
| input: past_key_values.23.key |
| output: present.23.key |
| role: key |
| layer: 23 |
| cache_47: |
| input: past_key_values.23.value |
| output: present.23.value |
| role: value |
| layer: 23 |
| steps: |
| - kind: loop |
| setup: |
| - kind: invoke |
| component: decoder_state_initializer |
| inputs: |
| prompt_tokens: request.input_ids |
| prompt_lengths: request.prompt_lengths |
| max_iterations: request.max_iterations |
| outputs: |
| attention_mask: initializer.attention_mask |
| body_attention_mask: initializer.body_attention_mask |
| token_slot: initializer.token_slot |
| generated_lengths: initializer.generated_lengths |
| cache_lengths: initializer.cache_lengths |
| past_key_values.0.key: initializer.past_key_values.0.key |
| past_key_values.0.value: initializer.past_key_values.0.value |
| past_key_values.1.key: initializer.past_key_values.1.key |
| past_key_values.1.value: initializer.past_key_values.1.value |
| past_key_values.10.key: initializer.past_key_values.10.key |
| past_key_values.10.value: initializer.past_key_values.10.value |
| past_key_values.11.key: initializer.past_key_values.11.key |
| past_key_values.11.value: initializer.past_key_values.11.value |
| past_key_values.12.key: initializer.past_key_values.12.key |
| past_key_values.12.value: initializer.past_key_values.12.value |
| past_key_values.13.key: initializer.past_key_values.13.key |
| past_key_values.13.value: initializer.past_key_values.13.value |
| past_key_values.14.key: initializer.past_key_values.14.key |
| past_key_values.14.value: initializer.past_key_values.14.value |
| past_key_values.15.key: initializer.past_key_values.15.key |
| past_key_values.15.value: initializer.past_key_values.15.value |
| past_key_values.16.key: initializer.past_key_values.16.key |
| past_key_values.16.value: initializer.past_key_values.16.value |
| past_key_values.17.key: initializer.past_key_values.17.key |
| past_key_values.17.value: initializer.past_key_values.17.value |
| past_key_values.18.key: initializer.past_key_values.18.key |
| past_key_values.18.value: initializer.past_key_values.18.value |
| past_key_values.19.key: initializer.past_key_values.19.key |
| past_key_values.19.value: initializer.past_key_values.19.value |
| past_key_values.2.key: initializer.past_key_values.2.key |
| past_key_values.2.value: initializer.past_key_values.2.value |
| past_key_values.20.key: initializer.past_key_values.20.key |
| past_key_values.20.value: initializer.past_key_values.20.value |
| past_key_values.21.key: initializer.past_key_values.21.key |
| past_key_values.21.value: initializer.past_key_values.21.value |
| past_key_values.22.key: initializer.past_key_values.22.key |
| past_key_values.22.value: initializer.past_key_values.22.value |
| past_key_values.23.key: initializer.past_key_values.23.key |
| past_key_values.23.value: initializer.past_key_values.23.value |
| past_key_values.3.key: initializer.past_key_values.3.key |
| past_key_values.3.value: initializer.past_key_values.3.value |
| past_key_values.4.key: initializer.past_key_values.4.key |
| past_key_values.4.value: initializer.past_key_values.4.value |
| past_key_values.5.key: initializer.past_key_values.5.key |
| past_key_values.5.value: initializer.past_key_values.5.value |
| past_key_values.6.key: initializer.past_key_values.6.key |
| past_key_values.6.value: initializer.past_key_values.6.value |
| past_key_values.7.key: initializer.past_key_values.7.key |
| past_key_values.7.value: initializer.past_key_values.7.value |
| past_key_values.8.key: initializer.past_key_values.8.key |
| past_key_values.8.value: initializer.past_key_values.8.value |
| past_key_values.9.key: initializer.past_key_values.9.key |
| past_key_values.9.value: initializer.past_key_values.9.value |
| - kind: invoke |
| component: model |
| inputs: |
| input_ids: request.input_ids |
| past_key_values.0.key: initializer.past_key_values.0.key |
| past_key_values.0.value: initializer.past_key_values.0.value |
| past_key_values.1.key: initializer.past_key_values.1.key |
| past_key_values.1.value: initializer.past_key_values.1.value |
| past_key_values.2.key: initializer.past_key_values.2.key |
| past_key_values.2.value: initializer.past_key_values.2.value |
| past_key_values.3.key: initializer.past_key_values.3.key |
| past_key_values.3.value: initializer.past_key_values.3.value |
| past_key_values.4.key: initializer.past_key_values.4.key |
| past_key_values.4.value: initializer.past_key_values.4.value |
| past_key_values.5.key: initializer.past_key_values.5.key |
| past_key_values.5.value: initializer.past_key_values.5.value |
| past_key_values.6.key: initializer.past_key_values.6.key |
| past_key_values.6.value: initializer.past_key_values.6.value |
| past_key_values.7.key: initializer.past_key_values.7.key |
| past_key_values.7.value: initializer.past_key_values.7.value |
| past_key_values.8.key: initializer.past_key_values.8.key |
| past_key_values.8.value: initializer.past_key_values.8.value |
| past_key_values.9.key: initializer.past_key_values.9.key |
| past_key_values.9.value: initializer.past_key_values.9.value |
| past_key_values.10.key: initializer.past_key_values.10.key |
| past_key_values.10.value: initializer.past_key_values.10.value |
| past_key_values.11.key: initializer.past_key_values.11.key |
| past_key_values.11.value: initializer.past_key_values.11.value |
| past_key_values.12.key: initializer.past_key_values.12.key |
| past_key_values.12.value: initializer.past_key_values.12.value |
| past_key_values.13.key: initializer.past_key_values.13.key |
| past_key_values.13.value: initializer.past_key_values.13.value |
| past_key_values.14.key: initializer.past_key_values.14.key |
| past_key_values.14.value: initializer.past_key_values.14.value |
| past_key_values.15.key: initializer.past_key_values.15.key |
| past_key_values.15.value: initializer.past_key_values.15.value |
| past_key_values.16.key: initializer.past_key_values.16.key |
| past_key_values.16.value: initializer.past_key_values.16.value |
| past_key_values.17.key: initializer.past_key_values.17.key |
| past_key_values.17.value: initializer.past_key_values.17.value |
| past_key_values.18.key: initializer.past_key_values.18.key |
| past_key_values.18.value: initializer.past_key_values.18.value |
| past_key_values.19.key: initializer.past_key_values.19.key |
| past_key_values.19.value: initializer.past_key_values.19.value |
| past_key_values.20.key: initializer.past_key_values.20.key |
| past_key_values.20.value: initializer.past_key_values.20.value |
| past_key_values.21.key: initializer.past_key_values.21.key |
| past_key_values.21.value: initializer.past_key_values.21.value |
| past_key_values.22.key: initializer.past_key_values.22.key |
| past_key_values.22.value: initializer.past_key_values.22.value |
| past_key_values.23.key: initializer.past_key_values.23.key |
| past_key_values.23.value: initializer.past_key_values.23.value |
| attention_mask: initializer.attention_mask |
| outputs: |
| logits: decoder.setup.logits |
| present.0.key: decoder.setup.present.0.key |
| present.0.value: decoder.setup.present.0.value |
| present.1.key: decoder.setup.present.1.key |
| present.1.value: decoder.setup.present.1.value |
| present.2.key: decoder.setup.present.2.key |
| present.2.value: decoder.setup.present.2.value |
| present.3.key: decoder.setup.present.3.key |
| present.3.value: decoder.setup.present.3.value |
| present.4.key: decoder.setup.present.4.key |
| present.4.value: decoder.setup.present.4.value |
| present.5.key: decoder.setup.present.5.key |
| present.5.value: decoder.setup.present.5.value |
| present.6.key: decoder.setup.present.6.key |
| present.6.value: decoder.setup.present.6.value |
| present.7.key: decoder.setup.present.7.key |
| present.7.value: decoder.setup.present.7.value |
| present.8.key: decoder.setup.present.8.key |
| present.8.value: decoder.setup.present.8.value |
| present.9.key: decoder.setup.present.9.key |
| present.9.value: decoder.setup.present.9.value |
| present.10.key: decoder.setup.present.10.key |
| present.10.value: decoder.setup.present.10.value |
| present.11.key: decoder.setup.present.11.key |
| present.11.value: decoder.setup.present.11.value |
| present.12.key: decoder.setup.present.12.key |
| present.12.value: decoder.setup.present.12.value |
| present.13.key: decoder.setup.present.13.key |
| present.13.value: decoder.setup.present.13.value |
| present.14.key: decoder.setup.present.14.key |
| present.14.value: decoder.setup.present.14.value |
| present.15.key: decoder.setup.present.15.key |
| present.15.value: decoder.setup.present.15.value |
| present.16.key: decoder.setup.present.16.key |
| present.16.value: decoder.setup.present.16.value |
| present.17.key: decoder.setup.present.17.key |
| present.17.value: decoder.setup.present.17.value |
| present.18.key: decoder.setup.present.18.key |
| present.18.value: decoder.setup.present.18.value |
| present.19.key: decoder.setup.present.19.key |
| present.19.value: decoder.setup.present.19.value |
| present.20.key: decoder.setup.present.20.key |
| present.20.value: decoder.setup.present.20.value |
| present.21.key: decoder.setup.present.21.key |
| present.21.value: decoder.setup.present.21.value |
| present.22.key: decoder.setup.present.22.key |
| present.22.value: decoder.setup.present.22.value |
| present.23.key: decoder.setup.present.23.key |
| present.23.value: decoder.setup.present.23.value |
| - kind: invoke |
| component: termination_batch_initializer |
| inputs: |
| input_eos_ids: request.eos_ids |
| input_eos_lengths: request.eos_lengths |
| input_max_iterations: request.row_max_iterations |
| fallback_max_iterations: request.max_iterations |
| active: package.active |
| outputs: |
| row_eos_ids: termination.eos_ids |
| eos_lengths: termination.eos_lengths |
| max_iterations: termination.max_iterations |
| - kind: invoke |
| component: last_token_logits |
| inputs: |
| logits: decoder.setup.logits |
| outputs: |
| last_logits: decoder.setup.last_logits |
| steps: |
| - kind: invoke |
| component: token_sampler |
| inputs: |
| logits: logits |
| temperature: request.temperature |
| top_k: request.top_k |
| top_p: request.top_p |
| min_p: request.min_p |
| seed: request.seed |
| counter: rng_counter |
| active: active |
| done: done |
| outputs: |
| token: sample.body |
| next_counter: sample.next_counter |
| - kind: invoke |
| component: token_to_slot |
| inputs: |
| token: sample.body |
| outputs: |
| slot: sample.slot |
| - kind: invoke |
| component: generated_length_update |
| inputs: |
| left: generated_lengths |
| right: package.one_token |
| active: active |
| done: done |
| outputs: |
| total: token.next_lengths |
| - kind: invoke |
| component: generated_length_update |
| inputs: |
| left: package.zero_batch |
| right: package.one_token |
| active: active |
| done: done |
| outputs: |
| total: token.emitted_length |
| - kind: invoke |
| component: token_state_update |
| inputs: |
| current: token |
| update: sample.slot |
| active: active |
| done: done |
| outputs: |
| next: token.body |
| - kind: invoke |
| component: termination |
| inputs: |
| tokens: sample.body |
| eos_ids: termination.eos_ids |
| eos_lengths: termination.eos_lengths |
| iteration: loop.iteration |
| max_iterations: termination.max_iterations |
| active: active |
| outputs: |
| done: loop.done |
| continue: loop.continue |
| next_active: loop.next_active |
| - kind: invoke |
| component: cache_length_update |
| inputs: |
| left: cache_lengths |
| right: package.one_token |
| active: active |
| done: done |
| outputs: |
| total: cache_lengths.next |
| - kind: invoke |
| component: cache_length_update |
| inputs: |
| left: package.zero_batch |
| right: package.one_token |
| active: active |
| done: done |
| outputs: |
| total: accepted_len.next |
| - kind: emit |
| value: token.body |
| output: tokens |
| mode: append |
| valid_length: token.emitted_length |
| when: active |
| - kind: invoke |
| component: decoder_step_update |
| inputs: |
| attention_mask: attention_mask |
| logical_length: cache_lengths |
| outputs: |
| next_attention_mask: decoder_step.body_attention_mask |
| - kind: invoke |
| component: model |
| inputs: |
| input_ids: token.body |
| past_key_values.0.key: cache_0 |
| past_key_values.0.value: cache_1 |
| past_key_values.1.key: cache_2 |
| past_key_values.1.value: cache_3 |
| past_key_values.2.key: cache_4 |
| past_key_values.2.value: cache_5 |
| past_key_values.3.key: cache_6 |
| past_key_values.3.value: cache_7 |
| past_key_values.4.key: cache_8 |
| past_key_values.4.value: cache_9 |
| past_key_values.5.key: cache_10 |
| past_key_values.5.value: cache_11 |
| past_key_values.6.key: cache_12 |
| past_key_values.6.value: cache_13 |
| past_key_values.7.key: cache_14 |
| past_key_values.7.value: cache_15 |
| past_key_values.8.key: cache_16 |
| past_key_values.8.value: cache_17 |
| past_key_values.9.key: cache_18 |
| past_key_values.9.value: cache_19 |
| past_key_values.10.key: cache_20 |
| past_key_values.10.value: cache_21 |
| past_key_values.11.key: cache_22 |
| past_key_values.11.value: cache_23 |
| past_key_values.12.key: cache_24 |
| past_key_values.12.value: cache_25 |
| past_key_values.13.key: cache_26 |
| past_key_values.13.value: cache_27 |
| past_key_values.14.key: cache_28 |
| past_key_values.14.value: cache_29 |
| past_key_values.15.key: cache_30 |
| past_key_values.15.value: cache_31 |
| past_key_values.16.key: cache_32 |
| past_key_values.16.value: cache_33 |
| past_key_values.17.key: cache_34 |
| past_key_values.17.value: cache_35 |
| past_key_values.18.key: cache_36 |
| past_key_values.18.value: cache_37 |
| past_key_values.19.key: cache_38 |
| past_key_values.19.value: cache_39 |
| past_key_values.20.key: cache_40 |
| past_key_values.20.value: cache_41 |
| past_key_values.21.key: cache_42 |
| past_key_values.21.value: cache_43 |
| past_key_values.22.key: cache_44 |
| past_key_values.22.value: cache_45 |
| past_key_values.23.key: cache_46 |
| past_key_values.23.value: cache_47 |
| attention_mask: decoder_step.body_attention_mask |
| outputs: |
| logits: decoder.body.logits |
| present.0.key: decoder.body.present.0.key |
| present.0.value: decoder.body.present.0.value |
| present.1.key: decoder.body.present.1.key |
| present.1.value: decoder.body.present.1.value |
| present.2.key: decoder.body.present.2.key |
| present.2.value: decoder.body.present.2.value |
| present.3.key: decoder.body.present.3.key |
| present.3.value: decoder.body.present.3.value |
| present.4.key: decoder.body.present.4.key |
| present.4.value: decoder.body.present.4.value |
| present.5.key: decoder.body.present.5.key |
| present.5.value: decoder.body.present.5.value |
| present.6.key: decoder.body.present.6.key |
| present.6.value: decoder.body.present.6.value |
| present.7.key: decoder.body.present.7.key |
| present.7.value: decoder.body.present.7.value |
| present.8.key: decoder.body.present.8.key |
| present.8.value: decoder.body.present.8.value |
| present.9.key: decoder.body.present.9.key |
| present.9.value: decoder.body.present.9.value |
| present.10.key: decoder.body.present.10.key |
| present.10.value: decoder.body.present.10.value |
| present.11.key: decoder.body.present.11.key |
| present.11.value: decoder.body.present.11.value |
| present.12.key: decoder.body.present.12.key |
| present.12.value: decoder.body.present.12.value |
| present.13.key: decoder.body.present.13.key |
| present.13.value: decoder.body.present.13.value |
| present.14.key: decoder.body.present.14.key |
| present.14.value: decoder.body.present.14.value |
| present.15.key: decoder.body.present.15.key |
| present.15.value: decoder.body.present.15.value |
| present.16.key: decoder.body.present.16.key |
| present.16.value: decoder.body.present.16.value |
| present.17.key: decoder.body.present.17.key |
| present.17.value: decoder.body.present.17.value |
| present.18.key: decoder.body.present.18.key |
| present.18.value: decoder.body.present.18.value |
| present.19.key: decoder.body.present.19.key |
| present.19.value: decoder.body.present.19.value |
| present.20.key: decoder.body.present.20.key |
| present.20.value: decoder.body.present.20.value |
| present.21.key: decoder.body.present.21.key |
| present.21.value: decoder.body.present.21.value |
| present.22.key: decoder.body.present.22.key |
| present.22.value: decoder.body.present.22.value |
| present.23.key: decoder.body.present.23.key |
| present.23.value: decoder.body.present.23.value |
| - kind: invoke |
| component: last_token_logits |
| inputs: |
| logits: decoder.body.logits |
| outputs: |
| last_logits: decoder.body.last_logits |
| continue_when: active |
| max_iterations: request.max_iterations |
| carried: |
| - cell: token |
| next: token.body |
| - cell: logits |
| next: decoder.body.last_logits |
| - cell: generated_lengths |
| next: token.next_lengths |
| - cell: active |
| next: loop.next_active |
| - cell: done |
| next: loop.done |
| - cell: cache_lengths |
| next: cache_lengths.next |
| - cell: accepted_len |
| next: accepted_len.next |
| - cell: rng_counter |
| next: sample.next_counter |
| - cell: attention_mask |
| next: decoder_step.body_attention_mask |
| - cell: cache_0 |
| next: decoder.body.present.0.key |
| - cell: cache_1 |
| next: decoder.body.present.0.value |
| - cell: cache_2 |
| next: decoder.body.present.1.key |
| - cell: cache_3 |
| next: decoder.body.present.1.value |
| - cell: cache_4 |
| next: decoder.body.present.2.key |
| - cell: cache_5 |
| next: decoder.body.present.2.value |
| - cell: cache_6 |
| next: decoder.body.present.3.key |
| - cell: cache_7 |
| next: decoder.body.present.3.value |
| - cell: cache_8 |
| next: decoder.body.present.4.key |
| - cell: cache_9 |
| next: decoder.body.present.4.value |
| - cell: cache_10 |
| next: decoder.body.present.5.key |
| - cell: cache_11 |
| next: decoder.body.present.5.value |
| - cell: cache_12 |
| next: decoder.body.present.6.key |
| - cell: cache_13 |
| next: decoder.body.present.6.value |
| - cell: cache_14 |
| next: decoder.body.present.7.key |
| - cell: cache_15 |
| next: decoder.body.present.7.value |
| - cell: cache_16 |
| next: decoder.body.present.8.key |
| - cell: cache_17 |
| next: decoder.body.present.8.value |
| - cell: cache_18 |
| next: decoder.body.present.9.key |
| - cell: cache_19 |
| next: decoder.body.present.9.value |
| - cell: cache_20 |
| next: decoder.body.present.10.key |
| - cell: cache_21 |
| next: decoder.body.present.10.value |
| - cell: cache_22 |
| next: decoder.body.present.11.key |
| - cell: cache_23 |
| next: decoder.body.present.11.value |
| - cell: cache_24 |
| next: decoder.body.present.12.key |
| - cell: cache_25 |
| next: decoder.body.present.12.value |
| - cell: cache_26 |
| next: decoder.body.present.13.key |
| - cell: cache_27 |
| next: decoder.body.present.13.value |
| - cell: cache_28 |
| next: decoder.body.present.14.key |
| - cell: cache_29 |
| next: decoder.body.present.14.value |
| - cell: cache_30 |
| next: decoder.body.present.15.key |
| - cell: cache_31 |
| next: decoder.body.present.15.value |
| - cell: cache_32 |
| next: decoder.body.present.16.key |
| - cell: cache_33 |
| next: decoder.body.present.16.value |
| - cell: cache_34 |
| next: decoder.body.present.17.key |
| - cell: cache_35 |
| next: decoder.body.present.17.value |
| - cell: cache_36 |
| next: decoder.body.present.18.key |
| - cell: cache_37 |
| next: decoder.body.present.18.value |
| - cell: cache_38 |
| next: decoder.body.present.19.key |
| - cell: cache_39 |
| next: decoder.body.present.19.value |
| - cell: cache_40 |
| next: decoder.body.present.20.key |
| - cell: cache_41 |
| next: decoder.body.present.20.value |
| - cell: cache_42 |
| next: decoder.body.present.21.key |
| - cell: cache_43 |
| next: decoder.body.present.21.value |
| - cell: cache_44 |
| next: decoder.body.present.22.key |
| - cell: cache_45 |
| next: decoder.body.present.22.value |
| - cell: cache_46 |
| next: decoder.body.present.23.key |
| - cell: cache_47 |
| next: decoder.body.present.23.value |
| termination: generation_eos |
| iteration: |
| value: loop.iteration |
| contract: |
| dtype: int64 |
| rank: 1 |
| shape: |
| - 1 |
| package: |
| tokenizer: |
| special_tokens: |
| eos_token_id: |
| - 151645 |
| pad_token_id: 151643 |
| bos_token_id: 151643 |
|
|
