schema_version: 'v1.2'
pipeline:
  workflow:
    manifest:
      capabilities:
      - workflow_ssa
      - linear_effects
      - nested_control_flow
      - typed_emit
      - emit_valid_length
      - loop_induction_values
      - serving_service_contract
      - bounded_state_recurrence
    inputs:
      request.input_ids:
        contract:
          dtype: int64
          rank: 2
          shape:
          - batch
          - sequence_len
          batch_layout:
            kind: request_aligned
            axis: 0
        role:
          kind: runtime
          version: '1.0'
          role: prompt_tokens
        source:
          kind: request
        required: true
      request.max_iterations:
        contract:
          dtype: int64
          rank: 1
          shape:
          - 1
        role:
          kind: runtime
          version: '1.0'
          role: max_output_tokens
        source:
          kind: request
        required: true
      package.one_token:
        contract:
          dtype: int64
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        role:
          kind: opaque
        source:
          kind: literal
        required: false
        default: 1
      package.one_step:
        contract:
          dtype: int64
          rank: 1
          shape:
          - 1
        role:
          kind: opaque
        source:
          kind: literal
        required: false
        default: 1
      package.max_context:
        contract:
          dtype: int64
          rank: 1
          shape:
          - 1
        role:
          kind: opaque
        source:
          kind: literal
        required: false
        default: 32768
      request.prompt_lengths:
        contract:
          dtype: int64
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        role:
          kind: opaque
        source:
          kind: application
          name: prompt_lengths
        required: false
        default: -1
      request.eos_ids:
        contract:
          dtype: int64
          rank: 2
          shape:
          - batch
          - num_eos
          batch_layout:
            kind: request_aligned
            axis: 0
        role:
          kind: runtime
          version: '1.0'
          role: eos_token_ids
        source:
          kind: request
        required: true
      request.eos_lengths:
        contract:
          dtype: int64
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        role:
          kind: runtime
          version: '1.0'
          role: eos_token_lengths
        source:
          kind: request
        required: true
      request.row_max_iterations:
        contract:
          dtype: int64
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        role:
          kind: opaque
        source:
          kind: application
          name: row_max_iterations
        required: false
        default: -1
      request.temperature:
        contract:
          dtype: float32
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        role:
          kind: runtime
          version: '1.0'
          role: sampling_temperature
        source:
          kind: request
        required: false
        default: 1.0
      request.top_k:
        contract:
          dtype: int64
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        role:
          kind: runtime
          version: '1.0'
          role: sampling_top_k
        source:
          kind: request
        required: false
        default: 1
      request.top_p:
        contract:
          dtype: float32
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        role:
          kind: runtime
          version: '1.0'
          role: sampling_top_p
        source:
          kind: request
        required: false
        default: 1.0
      request.min_p:
        contract:
          dtype: float32
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        role:
          kind: runtime
          version: '1.0'
          role: sampling_min_p
        source:
          kind: request
        required: false
        default: 0.0
      request.seed:
        contract:
          dtype: int64
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        role:
          kind: runtime
          version: '1.0'
          role: seed
        source:
          kind: request
        required: false
        default: 0
      request.rng_counter:
        contract:
          dtype: int64
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        role:
          kind: opaque
        source:
          kind: application
          name: rng_counter
        required: false
        default: 0
      package.active:
        contract:
          dtype: bool
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        role:
          kind: opaque
        source:
          kind: literal
        required: false
        default: true
      package.not_done:
        contract:
          dtype: bool
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        role:
          kind: opaque
        source:
          kind: literal
        required: false
        default: false
      package.cache_lengths:
        contract:
          dtype: int64
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        role:
          kind: opaque
        source:
          kind: literal
        required: false
        default: 0
      package.zero_batch:
        contract:
          dtype: int64
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        role:
          kind: opaque
        source:
          kind: literal
        required: false
        default: 0
    outputs:
      tokens:
        contract:
          dtype: int64
          rank: 2
          shape:
          - batch
          - generated_sequence
          batch_layout:
            kind: request_aligned
            axis: 0
        role: tokens
        stage: pre_adapter
    components:
      model:
        implementation:
          kind: onnx
          artifact: model.onnx
        ports:
          roles:
            input_ids: token_ids
            attention_mask: attention_mask
            logits: logits
      token_sampler:
        implementation:
          kind: onnx
          artifact: policies/token_sampler.onnx
        ports:
          inputs:
            logits:
              dtype: float32
              rank: 2
              shape:
              - batch
              - vocabulary
              batch_layout:
                kind: request_aligned
                axis: 0
            temperature:
              dtype: float32
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            top_k:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            top_p:
              dtype: float32
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            min_p:
              dtype: float32
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            seed:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            counter:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            active:
              dtype: bool
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            done:
              dtype: bool
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
          outputs:
            token:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            next_counter:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
        contract:
          id: onnx-genai.token-sampler
          version: '2'
          bindings:
            logits: logits
            token: token
            temperature: temperature
            top_k: top_k
            top_p: top_p
            min_p: min_p
            active: active
            done: done
            seed: seed
            counter: counter
            next_counter: next_counter
          parameters:
            mode: seeded_stochastic
            batching: per_row
            inactive_rows: preserve
        application_overridable: true
      termination:
        implementation:
          kind: onnx
          artifact: policies/termination.onnx
        ports:
          inputs:
            tokens:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            eos_ids:
              dtype: int64
              rank: 2
              shape:
              - batch
              - num_eos
              batch_layout:
                kind: request_aligned
                axis: 0
            eos_lengths:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            iteration:
              dtype: int64
              rank: 1
              shape:
              - 1
            max_iterations:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            active:
              dtype: bool
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
          outputs:
            done:
              dtype: bool
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            next_active:
              dtype: bool
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            continue:
              dtype: bool
              rank: 1
              shape:
              - 1
        contract:
          id: onnx-genai.termination-predicate
          version: '2'
          bindings:
            tokens: tokens
            eos_ids: eos_ids
            iteration: iteration
            max_iterations: max_iterations
            eos_lengths: eos_lengths
            active: active
            done: done
            next_active: next_active
            continue: continue
          parameters:
            batching: per_row
            inactive_rows: preserve
      token_state_update:
        implementation:
          kind: onnx
          artifact: policies/token_state_update.onnx
        ports:
          inputs:
            current:
              dtype: int64
              rank: 2
              shape:
              - batch
              - 1
              batch_layout:
                kind: request_aligned
                axis: 0
            update:
              dtype: int64
              rank: 2
              shape:
              - batch
              - 1
              batch_layout:
                kind: request_aligned
                axis: 0
            active:
              dtype: bool
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            done:
              dtype: bool
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
          outputs:
            next:
              dtype: int64
              rank: 2
              shape:
              - batch
              - 1
              batch_layout:
                kind: request_aligned
                axis: 0
        contract:
          id: onnx-genai.state-update
          version: '2'
          bindings:
            current: current
            update: update
            active: active
            done: done
            next: next
          parameters:
            batching: per_row
            inactive_rows: preserve
      last_token_logits:
        implementation:
          kind: onnx
          artifact: policies/last_token_logits.onnx
        ports:
          inputs:
            logits:
              dtype: float16
              rank: 3
              shape:
              - batch
              - sequence
              - vocabulary
              batch_layout:
                kind: request_aligned
                axis: 0
          outputs:
            last_logits:
              dtype: float32
              rank: 2
              shape:
              - batch
              - vocabulary
              batch_layout:
                kind: request_aligned
                axis: 0
      decoder_state_initializer:
        implementation:
          kind: onnx
          artifact: policies/decoder_state_initializer.onnx
        ports:
          inputs:
            prompt_tokens:
              dtype: int64
              rank: 2
              shape:
              - batch
              - prompt_sequence
              batch_layout:
                kind: request_aligned
                axis: 0
            prompt_lengths:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            max_iterations:
              dtype: int64
              rank: 1
              shape:
              - 1
          outputs:
            attention_mask:
              dtype: int64
              rank: 2
              shape:
              - batch
              - capacity
              batch_layout:
                kind: request_aligned
                axis: 0
            body_attention_mask:
              dtype: int64
              rank: 2
              shape:
              - batch
              - capacity
              batch_layout:
                kind: request_aligned
                axis: 0
            token_slot:
              dtype: int64
              rank: 2
              shape:
              - batch
              - 1
              batch_layout:
                kind: request_aligned
                axis: 0
            generated_lengths:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            cache_lengths:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.0.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.0.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.1.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.1.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.10.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.10.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.11.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.11.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.12.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.12.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.13.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.13.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.14.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.14.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.15.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.15.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.16.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.16.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.17.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.17.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.18.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.18.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.19.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.19.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.2.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.2.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.20.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.20.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.21.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.21.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.22.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.22.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.23.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.23.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.3.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.3.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.4.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.4.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.5.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.5.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.6.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.6.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.7.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.7.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.8.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.8.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.9.key:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
            past_key_values.9.value:
              dtype: float16
              rank: 4
              shape:
              - batch
              - 2
              - capacity
              - 64
              batch_layout:
                kind: request_aligned
                axis: 0
      decoder_step_update:
        implementation:
          kind: onnx
          artifact: policies/decoder_step_update.onnx
        ports:
          inputs:
            attention_mask:
              dtype: int64
              rank: 2
              shape:
              - batch
              - context
              batch_layout:
                kind: request_aligned
                axis: 0
            logical_length:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
          outputs:
            next_attention_mask:
              dtype: int64
              rank: 2
              shape:
              - batch
              - context
              batch_layout:
                kind: request_aligned
                axis: 0
      cache_length_update:
        implementation:
          kind: onnx
          artifact: policies/cache_length_update.onnx
        ports:
          inputs:
            left:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            right:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            active:
              dtype: bool
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            done:
              dtype: bool
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
          outputs:
            total:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
      termination_batch_initializer:
        implementation:
          kind: onnx
          artifact: policies/termination_batch_initializer.onnx
        ports:
          inputs:
            input_eos_ids:
              dtype: int64
              rank: 2
              shape:
              - batch
              - num_eos
              batch_layout:
                kind: request_aligned
                axis: 0
            input_eos_lengths:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            input_max_iterations:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            fallback_max_iterations:
              dtype: int64
              rank: 1
              shape:
              - 1
            active:
              dtype: bool
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
          outputs:
            row_eos_ids:
              dtype: int64
              rank: 2
              shape:
              - batch
              - num_eos
              batch_layout:
                kind: request_aligned
                axis: 0
            eos_lengths:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            max_iterations:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
      token_to_slot:
        implementation:
          kind: onnx
          artifact: policies/token_to_slot.onnx
        ports:
          inputs:
            token:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
          outputs:
            slot:
              dtype: int64
              rank: 2
              shape:
              - batch
              - 1
              batch_layout:
                kind: request_aligned
                axis: 0
      generated_length_update:
        implementation:
          kind: onnx
          artifact: policies/generated_length_update.onnx
        ports:
          inputs:
            left:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            right:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            active:
              dtype: bool
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
            done:
              dtype: bool
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
          outputs:
            total:
              dtype: int64
              rank: 1
              shape:
              - batch
              batch_layout:
                kind: request_aligned
                axis: 0
    state:
      token:
        contract:
          dtype: int64
          rank: 2
          shape:
          - batch
          - 1
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: initializer.token_slot
        recurrence:
          kind: invariant
      logits:
        contract:
          dtype: float32
          rank: 2
          shape:
          - batch
          - 151936
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.last_logits
        recurrence:
          kind: invariant
      generated_lengths:
        contract:
          dtype: int64
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        class: semantic
        scope: invocation
        initializer: initializer.generated_lengths
        recurrence:
          kind: invariant
      active:
        contract:
          dtype: bool
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        class: semantic
        scope: invocation
        initializer: package.active
        recurrence:
          kind: invariant
      done:
        contract:
          dtype: bool
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        class: semantic
        scope: invocation
        initializer: package.not_done
        recurrence:
          kind: invariant
      accepted_len:
        contract:
          dtype: int64
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        class: semantic
        scope: invocation
        initializer: package.zero_batch
        recurrence:
          kind: invariant
      cache_lengths:
        contract:
          dtype: int64
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        class: semantic
        scope: invocation
        initializer: initializer.cache_lengths
        recurrence:
          kind: invariant
      rng_counter:
        contract:
          dtype: int64
          rank: 1
          shape:
          - batch
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        class: semantic
        initializer: request.rng_counter
        recurrence:
          kind: invariant
      attention_mask:
        contract:
          dtype: int64
          rank: 2
          shape:
          - batch
          - context
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: initializer.attention_mask
        recurrence:
          kind: invariant
      cache_0:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.0.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_1:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.0.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_2:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.1.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_3:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.1.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_4:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.2.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_5:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.2.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_6:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.3.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_7:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.3.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_8:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.4.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_9:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.4.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_10:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.5.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_11:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.5.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_12:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.6.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_13:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.6.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_14:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.7.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_15:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.7.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_16:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.8.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_17:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.8.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_18:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.9.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_19:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.9.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_20:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.10.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_21:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.10.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_22:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.11.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_23:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.11.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_24:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.12.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_25:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.12.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_26:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.13.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_27:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.13.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_28:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.14.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_29:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.14.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_30:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.15.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_31:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.15.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_32:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.16.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_33:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.16.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_34:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.17.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_35:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.17.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_36:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.18.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_37:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.18.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_38:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.19.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_39:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.19.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_40:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.20.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_41:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.20.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_42:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.21.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_43:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.21.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_44:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.22.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_45:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.22.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_46:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.23.key
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
      cache_47:
        contract:
          dtype: float16
          rank: 4
          shape:
          - batch
          - 2
          - past_sequence_len
          - 64
          batch_layout:
            kind: request_aligned
            axis: 0
        scope: invocation
        initializer: decoder.setup.present.23.value
        recurrence:
          kind: bounded
          axis: 2
          max: package.max_context
        management: runtime
        release_boundary: invocation
        service_group: decoder_cache
    serving:
      active: active
      done: done
      accepted_len: accepted_len
      state_service:
        groups:
          decoder_cache:
            kind: full_attention
            sequence_axis: 2
            layout: bnsh
            logical_lengths: cache_lengths
            aliasing: permitted
            reuse:
              prefix_reusable: true
              evictable_prefix: false
            ports:
              model:
                cache_0:
                  input: past_key_values.0.key
                  output: present.0.key
                  role: key
                  layer: 0
                cache_1:
                  input: past_key_values.0.value
                  output: present.0.value
                  role: value
                  layer: 0
                cache_2:
                  input: past_key_values.1.key
                  output: present.1.key
                  role: key
                  layer: 1
                cache_3:
                  input: past_key_values.1.value
                  output: present.1.value
                  role: value
                  layer: 1
                cache_4:
                  input: past_key_values.2.key
                  output: present.2.key
                  role: key
                  layer: 2
                cache_5:
                  input: past_key_values.2.value
                  output: present.2.value
                  role: value
                  layer: 2
                cache_6:
                  input: past_key_values.3.key
                  output: present.3.key
                  role: key
                  layer: 3
                cache_7:
                  input: past_key_values.3.value
                  output: present.3.value
                  role: value
                  layer: 3
                cache_8:
                  input: past_key_values.4.key
                  output: present.4.key
                  role: key
                  layer: 4
                cache_9:
                  input: past_key_values.4.value
                  output: present.4.value
                  role: value
                  layer: 4
                cache_10:
                  input: past_key_values.5.key
                  output: present.5.key
                  role: key
                  layer: 5
                cache_11:
                  input: past_key_values.5.value
                  output: present.5.value
                  role: value
                  layer: 5
                cache_12:
                  input: past_key_values.6.key
                  output: present.6.key
                  role: key
                  layer: 6
                cache_13:
                  input: past_key_values.6.value
                  output: present.6.value
                  role: value
                  layer: 6
                cache_14:
                  input: past_key_values.7.key
                  output: present.7.key
                  role: key
                  layer: 7
                cache_15:
                  input: past_key_values.7.value
                  output: present.7.value
                  role: value
                  layer: 7
                cache_16:
                  input: past_key_values.8.key
                  output: present.8.key
                  role: key
                  layer: 8
                cache_17:
                  input: past_key_values.8.value
                  output: present.8.value
                  role: value
                  layer: 8
                cache_18:
                  input: past_key_values.9.key
                  output: present.9.key
                  role: key
                  layer: 9
                cache_19:
                  input: past_key_values.9.value
                  output: present.9.value
                  role: value
                  layer: 9
                cache_20:
                  input: past_key_values.10.key
                  output: present.10.key
                  role: key
                  layer: 10
                cache_21:
                  input: past_key_values.10.value
                  output: present.10.value
                  role: value
                  layer: 10
                cache_22:
                  input: past_key_values.11.key
                  output: present.11.key
                  role: key
                  layer: 11
                cache_23:
                  input: past_key_values.11.value
                  output: present.11.value
                  role: value
                  layer: 11
                cache_24:
                  input: past_key_values.12.key
                  output: present.12.key
                  role: key
                  layer: 12
                cache_25:
                  input: past_key_values.12.value
                  output: present.12.value
                  role: value
                  layer: 12
                cache_26:
                  input: past_key_values.13.key
                  output: present.13.key
                  role: key
                  layer: 13
                cache_27:
                  input: past_key_values.13.value
                  output: present.13.value
                  role: value
                  layer: 13
                cache_28:
                  input: past_key_values.14.key
                  output: present.14.key
                  role: key
                  layer: 14
                cache_29:
                  input: past_key_values.14.value
                  output: present.14.value
                  role: value
                  layer: 14
                cache_30:
                  input: past_key_values.15.key
                  output: present.15.key
                  role: key
                  layer: 15
                cache_31:
                  input: past_key_values.15.value
                  output: present.15.value
                  role: value
                  layer: 15
                cache_32:
                  input: past_key_values.16.key
                  output: present.16.key
                  role: key
                  layer: 16
                cache_33:
                  input: past_key_values.16.value
                  output: present.16.value
                  role: value
                  layer: 16
                cache_34:
                  input: past_key_values.17.key
                  output: present.17.key
                  role: key
                  layer: 17
                cache_35:
                  input: past_key_values.17.value
                  output: present.17.value
                  role: value
                  layer: 17
                cache_36:
                  input: past_key_values.18.key
                  output: present.18.key
                  role: key
                  layer: 18
                cache_37:
                  input: past_key_values.18.value
                  output: present.18.value
                  role: value
                  layer: 18
                cache_38:
                  input: past_key_values.19.key
                  output: present.19.key
                  role: key
                  layer: 19
                cache_39:
                  input: past_key_values.19.value
                  output: present.19.value
                  role: value
                  layer: 19
                cache_40:
                  input: past_key_values.20.key
                  output: present.20.key
                  role: key
                  layer: 20
                cache_41:
                  input: past_key_values.20.value
                  output: present.20.value
                  role: value
                  layer: 20
                cache_42:
                  input: past_key_values.21.key
                  output: present.21.key
                  role: key
                  layer: 21
                cache_43:
                  input: past_key_values.21.value
                  output: present.21.value
                  role: value
                  layer: 21
                cache_44:
                  input: past_key_values.22.key
                  output: present.22.key
                  role: key
                  layer: 22
                cache_45:
                  input: past_key_values.22.value
                  output: present.22.value
                  role: value
                  layer: 22
                cache_46:
                  input: past_key_values.23.key
                  output: present.23.key
                  role: key
                  layer: 23
                cache_47:
                  input: past_key_values.23.value
                  output: present.23.value
                  role: value
                  layer: 23
    steps:
    - kind: loop
      setup:
      - kind: invoke
        component: decoder_state_initializer
        inputs:
          prompt_tokens: request.input_ids
          prompt_lengths: request.prompt_lengths
          max_iterations: request.max_iterations
        outputs:
          attention_mask: initializer.attention_mask
          body_attention_mask: initializer.body_attention_mask
          token_slot: initializer.token_slot
          generated_lengths: initializer.generated_lengths
          cache_lengths: initializer.cache_lengths
          past_key_values.0.key: initializer.past_key_values.0.key
          past_key_values.0.value: initializer.past_key_values.0.value
          past_key_values.1.key: initializer.past_key_values.1.key
          past_key_values.1.value: initializer.past_key_values.1.value
          past_key_values.10.key: initializer.past_key_values.10.key
          past_key_values.10.value: initializer.past_key_values.10.value
          past_key_values.11.key: initializer.past_key_values.11.key
          past_key_values.11.value: initializer.past_key_values.11.value
          past_key_values.12.key: initializer.past_key_values.12.key
          past_key_values.12.value: initializer.past_key_values.12.value
          past_key_values.13.key: initializer.past_key_values.13.key
          past_key_values.13.value: initializer.past_key_values.13.value
          past_key_values.14.key: initializer.past_key_values.14.key
          past_key_values.14.value: initializer.past_key_values.14.value
          past_key_values.15.key: initializer.past_key_values.15.key
          past_key_values.15.value: initializer.past_key_values.15.value
          past_key_values.16.key: initializer.past_key_values.16.key
          past_key_values.16.value: initializer.past_key_values.16.value
          past_key_values.17.key: initializer.past_key_values.17.key
          past_key_values.17.value: initializer.past_key_values.17.value
          past_key_values.18.key: initializer.past_key_values.18.key
          past_key_values.18.value: initializer.past_key_values.18.value
          past_key_values.19.key: initializer.past_key_values.19.key
          past_key_values.19.value: initializer.past_key_values.19.value
          past_key_values.2.key: initializer.past_key_values.2.key
          past_key_values.2.value: initializer.past_key_values.2.value
          past_key_values.20.key: initializer.past_key_values.20.key
          past_key_values.20.value: initializer.past_key_values.20.value
          past_key_values.21.key: initializer.past_key_values.21.key
          past_key_values.21.value: initializer.past_key_values.21.value
          past_key_values.22.key: initializer.past_key_values.22.key
          past_key_values.22.value: initializer.past_key_values.22.value
          past_key_values.23.key: initializer.past_key_values.23.key
          past_key_values.23.value: initializer.past_key_values.23.value
          past_key_values.3.key: initializer.past_key_values.3.key
          past_key_values.3.value: initializer.past_key_values.3.value
          past_key_values.4.key: initializer.past_key_values.4.key
          past_key_values.4.value: initializer.past_key_values.4.value
          past_key_values.5.key: initializer.past_key_values.5.key
          past_key_values.5.value: initializer.past_key_values.5.value
          past_key_values.6.key: initializer.past_key_values.6.key
          past_key_values.6.value: initializer.past_key_values.6.value
          past_key_values.7.key: initializer.past_key_values.7.key
          past_key_values.7.value: initializer.past_key_values.7.value
          past_key_values.8.key: initializer.past_key_values.8.key
          past_key_values.8.value: initializer.past_key_values.8.value
          past_key_values.9.key: initializer.past_key_values.9.key
          past_key_values.9.value: initializer.past_key_values.9.value
      - kind: invoke
        component: model
        inputs:
          input_ids: request.input_ids
          past_key_values.0.key: initializer.past_key_values.0.key
          past_key_values.0.value: initializer.past_key_values.0.value
          past_key_values.1.key: initializer.past_key_values.1.key
          past_key_values.1.value: initializer.past_key_values.1.value
          past_key_values.2.key: initializer.past_key_values.2.key
          past_key_values.2.value: initializer.past_key_values.2.value
          past_key_values.3.key: initializer.past_key_values.3.key
          past_key_values.3.value: initializer.past_key_values.3.value
          past_key_values.4.key: initializer.past_key_values.4.key
          past_key_values.4.value: initializer.past_key_values.4.value
          past_key_values.5.key: initializer.past_key_values.5.key
          past_key_values.5.value: initializer.past_key_values.5.value
          past_key_values.6.key: initializer.past_key_values.6.key
          past_key_values.6.value: initializer.past_key_values.6.value
          past_key_values.7.key: initializer.past_key_values.7.key
          past_key_values.7.value: initializer.past_key_values.7.value
          past_key_values.8.key: initializer.past_key_values.8.key
          past_key_values.8.value: initializer.past_key_values.8.value
          past_key_values.9.key: initializer.past_key_values.9.key
          past_key_values.9.value: initializer.past_key_values.9.value
          past_key_values.10.key: initializer.past_key_values.10.key
          past_key_values.10.value: initializer.past_key_values.10.value
          past_key_values.11.key: initializer.past_key_values.11.key
          past_key_values.11.value: initializer.past_key_values.11.value
          past_key_values.12.key: initializer.past_key_values.12.key
          past_key_values.12.value: initializer.past_key_values.12.value
          past_key_values.13.key: initializer.past_key_values.13.key
          past_key_values.13.value: initializer.past_key_values.13.value
          past_key_values.14.key: initializer.past_key_values.14.key
          past_key_values.14.value: initializer.past_key_values.14.value
          past_key_values.15.key: initializer.past_key_values.15.key
          past_key_values.15.value: initializer.past_key_values.15.value
          past_key_values.16.key: initializer.past_key_values.16.key
          past_key_values.16.value: initializer.past_key_values.16.value
          past_key_values.17.key: initializer.past_key_values.17.key
          past_key_values.17.value: initializer.past_key_values.17.value
          past_key_values.18.key: initializer.past_key_values.18.key
          past_key_values.18.value: initializer.past_key_values.18.value
          past_key_values.19.key: initializer.past_key_values.19.key
          past_key_values.19.value: initializer.past_key_values.19.value
          past_key_values.20.key: initializer.past_key_values.20.key
          past_key_values.20.value: initializer.past_key_values.20.value
          past_key_values.21.key: initializer.past_key_values.21.key
          past_key_values.21.value: initializer.past_key_values.21.value
          past_key_values.22.key: initializer.past_key_values.22.key
          past_key_values.22.value: initializer.past_key_values.22.value
          past_key_values.23.key: initializer.past_key_values.23.key
          past_key_values.23.value: initializer.past_key_values.23.value
          attention_mask: initializer.attention_mask
        outputs:
          logits: decoder.setup.logits
          present.0.key: decoder.setup.present.0.key
          present.0.value: decoder.setup.present.0.value
          present.1.key: decoder.setup.present.1.key
          present.1.value: decoder.setup.present.1.value
          present.2.key: decoder.setup.present.2.key
          present.2.value: decoder.setup.present.2.value
          present.3.key: decoder.setup.present.3.key
          present.3.value: decoder.setup.present.3.value
          present.4.key: decoder.setup.present.4.key
          present.4.value: decoder.setup.present.4.value
          present.5.key: decoder.setup.present.5.key
          present.5.value: decoder.setup.present.5.value
          present.6.key: decoder.setup.present.6.key
          present.6.value: decoder.setup.present.6.value
          present.7.key: decoder.setup.present.7.key
          present.7.value: decoder.setup.present.7.value
          present.8.key: decoder.setup.present.8.key
          present.8.value: decoder.setup.present.8.value
          present.9.key: decoder.setup.present.9.key
          present.9.value: decoder.setup.present.9.value
          present.10.key: decoder.setup.present.10.key
          present.10.value: decoder.setup.present.10.value
          present.11.key: decoder.setup.present.11.key
          present.11.value: decoder.setup.present.11.value
          present.12.key: decoder.setup.present.12.key
          present.12.value: decoder.setup.present.12.value
          present.13.key: decoder.setup.present.13.key
          present.13.value: decoder.setup.present.13.value
          present.14.key: decoder.setup.present.14.key
          present.14.value: decoder.setup.present.14.value
          present.15.key: decoder.setup.present.15.key
          present.15.value: decoder.setup.present.15.value
          present.16.key: decoder.setup.present.16.key
          present.16.value: decoder.setup.present.16.value
          present.17.key: decoder.setup.present.17.key
          present.17.value: decoder.setup.present.17.value
          present.18.key: decoder.setup.present.18.key
          present.18.value: decoder.setup.present.18.value
          present.19.key: decoder.setup.present.19.key
          present.19.value: decoder.setup.present.19.value
          present.20.key: decoder.setup.present.20.key
          present.20.value: decoder.setup.present.20.value
          present.21.key: decoder.setup.present.21.key
          present.21.value: decoder.setup.present.21.value
          present.22.key: decoder.setup.present.22.key
          present.22.value: decoder.setup.present.22.value
          present.23.key: decoder.setup.present.23.key
          present.23.value: decoder.setup.present.23.value
      - kind: invoke
        component: termination_batch_initializer
        inputs:
          input_eos_ids: request.eos_ids
          input_eos_lengths: request.eos_lengths
          input_max_iterations: request.row_max_iterations
          fallback_max_iterations: request.max_iterations
          active: package.active
        outputs:
          row_eos_ids: termination.eos_ids
          eos_lengths: termination.eos_lengths
          max_iterations: termination.max_iterations
      - kind: invoke
        component: last_token_logits
        inputs:
          logits: decoder.setup.logits
        outputs:
          last_logits: decoder.setup.last_logits
      steps:
      - kind: invoke
        component: token_sampler
        inputs:
          logits: logits
          temperature: request.temperature
          top_k: request.top_k
          top_p: request.top_p
          min_p: request.min_p
          seed: request.seed
          counter: rng_counter
          active: active
          done: done
        outputs:
          token: sample.body
          next_counter: sample.next_counter
      - kind: invoke
        component: token_to_slot
        inputs:
          token: sample.body
        outputs:
          slot: sample.slot
      - kind: invoke
        component: generated_length_update
        inputs:
          left: generated_lengths
          right: package.one_token
          active: active
          done: done
        outputs:
          total: token.next_lengths
      - kind: invoke
        component: generated_length_update
        inputs:
          left: package.zero_batch
          right: package.one_token
          active: active
          done: done
        outputs:
          total: token.emitted_length
      - kind: invoke
        component: token_state_update
        inputs:
          current: token
          update: sample.slot
          active: active
          done: done
        outputs:
          next: token.body
      - kind: invoke
        component: termination
        inputs:
          tokens: sample.body
          eos_ids: termination.eos_ids
          eos_lengths: termination.eos_lengths
          iteration: loop.iteration
          max_iterations: termination.max_iterations
          active: active
        outputs:
          done: loop.done
          continue: loop.continue
          next_active: loop.next_active
      - kind: invoke
        component: cache_length_update
        inputs:
          left: cache_lengths
          right: package.one_token
          active: active
          done: done
        outputs:
          total: cache_lengths.next
      - kind: invoke
        component: cache_length_update
        inputs:
          left: package.zero_batch
          right: package.one_token
          active: active
          done: done
        outputs:
          total: accepted_len.next
      - kind: emit
        value: token.body
        output: tokens
        mode: append
        valid_length: token.emitted_length
        when: active
      - kind: invoke
        component: decoder_step_update
        inputs:
          attention_mask: attention_mask
          logical_length: cache_lengths
        outputs:
          next_attention_mask: decoder_step.body_attention_mask
      - kind: invoke
        component: model
        inputs:
          input_ids: token.body
          past_key_values.0.key: cache_0
          past_key_values.0.value: cache_1
          past_key_values.1.key: cache_2
          past_key_values.1.value: cache_3
          past_key_values.2.key: cache_4
          past_key_values.2.value: cache_5
          past_key_values.3.key: cache_6
          past_key_values.3.value: cache_7
          past_key_values.4.key: cache_8
          past_key_values.4.value: cache_9
          past_key_values.5.key: cache_10
          past_key_values.5.value: cache_11
          past_key_values.6.key: cache_12
          past_key_values.6.value: cache_13
          past_key_values.7.key: cache_14
          past_key_values.7.value: cache_15
          past_key_values.8.key: cache_16
          past_key_values.8.value: cache_17
          past_key_values.9.key: cache_18
          past_key_values.9.value: cache_19
          past_key_values.10.key: cache_20
          past_key_values.10.value: cache_21
          past_key_values.11.key: cache_22
          past_key_values.11.value: cache_23
          past_key_values.12.key: cache_24
          past_key_values.12.value: cache_25
          past_key_values.13.key: cache_26
          past_key_values.13.value: cache_27
          past_key_values.14.key: cache_28
          past_key_values.14.value: cache_29
          past_key_values.15.key: cache_30
          past_key_values.15.value: cache_31
          past_key_values.16.key: cache_32
          past_key_values.16.value: cache_33
          past_key_values.17.key: cache_34
          past_key_values.17.value: cache_35
          past_key_values.18.key: cache_36
          past_key_values.18.value: cache_37
          past_key_values.19.key: cache_38
          past_key_values.19.value: cache_39
          past_key_values.20.key: cache_40
          past_key_values.20.value: cache_41
          past_key_values.21.key: cache_42
          past_key_values.21.value: cache_43
          past_key_values.22.key: cache_44
          past_key_values.22.value: cache_45
          past_key_values.23.key: cache_46
          past_key_values.23.value: cache_47
          attention_mask: decoder_step.body_attention_mask
        outputs:
          logits: decoder.body.logits
          present.0.key: decoder.body.present.0.key
          present.0.value: decoder.body.present.0.value
          present.1.key: decoder.body.present.1.key
          present.1.value: decoder.body.present.1.value
          present.2.key: decoder.body.present.2.key
          present.2.value: decoder.body.present.2.value
          present.3.key: decoder.body.present.3.key
          present.3.value: decoder.body.present.3.value
          present.4.key: decoder.body.present.4.key
          present.4.value: decoder.body.present.4.value
          present.5.key: decoder.body.present.5.key
          present.5.value: decoder.body.present.5.value
          present.6.key: decoder.body.present.6.key
          present.6.value: decoder.body.present.6.value
          present.7.key: decoder.body.present.7.key
          present.7.value: decoder.body.present.7.value
          present.8.key: decoder.body.present.8.key
          present.8.value: decoder.body.present.8.value
          present.9.key: decoder.body.present.9.key
          present.9.value: decoder.body.present.9.value
          present.10.key: decoder.body.present.10.key
          present.10.value: decoder.body.present.10.value
          present.11.key: decoder.body.present.11.key
          present.11.value: decoder.body.present.11.value
          present.12.key: decoder.body.present.12.key
          present.12.value: decoder.body.present.12.value
          present.13.key: decoder.body.present.13.key
          present.13.value: decoder.body.present.13.value
          present.14.key: decoder.body.present.14.key
          present.14.value: decoder.body.present.14.value
          present.15.key: decoder.body.present.15.key
          present.15.value: decoder.body.present.15.value
          present.16.key: decoder.body.present.16.key
          present.16.value: decoder.body.present.16.value
          present.17.key: decoder.body.present.17.key
          present.17.value: decoder.body.present.17.value
          present.18.key: decoder.body.present.18.key
          present.18.value: decoder.body.present.18.value
          present.19.key: decoder.body.present.19.key
          present.19.value: decoder.body.present.19.value
          present.20.key: decoder.body.present.20.key
          present.20.value: decoder.body.present.20.value
          present.21.key: decoder.body.present.21.key
          present.21.value: decoder.body.present.21.value
          present.22.key: decoder.body.present.22.key
          present.22.value: decoder.body.present.22.value
          present.23.key: decoder.body.present.23.key
          present.23.value: decoder.body.present.23.value
      - kind: invoke
        component: last_token_logits
        inputs:
          logits: decoder.body.logits
        outputs:
          last_logits: decoder.body.last_logits
      continue_when: active
      max_iterations: request.max_iterations
      carried:
      - cell: token
        next: token.body
      - cell: logits
        next: decoder.body.last_logits
      - cell: generated_lengths
        next: token.next_lengths
      - cell: active
        next: loop.next_active
      - cell: done
        next: loop.done
      - cell: cache_lengths
        next: cache_lengths.next
      - cell: accepted_len
        next: accepted_len.next
      - cell: rng_counter
        next: sample.next_counter
      - cell: attention_mask
        next: decoder_step.body_attention_mask
      - cell: cache_0
        next: decoder.body.present.0.key
      - cell: cache_1
        next: decoder.body.present.0.value
      - cell: cache_2
        next: decoder.body.present.1.key
      - cell: cache_3
        next: decoder.body.present.1.value
      - cell: cache_4
        next: decoder.body.present.2.key
      - cell: cache_5
        next: decoder.body.present.2.value
      - cell: cache_6
        next: decoder.body.present.3.key
      - cell: cache_7
        next: decoder.body.present.3.value
      - cell: cache_8
        next: decoder.body.present.4.key
      - cell: cache_9
        next: decoder.body.present.4.value
      - cell: cache_10
        next: decoder.body.present.5.key
      - cell: cache_11
        next: decoder.body.present.5.value
      - cell: cache_12
        next: decoder.body.present.6.key
      - cell: cache_13
        next: decoder.body.present.6.value
      - cell: cache_14
        next: decoder.body.present.7.key
      - cell: cache_15
        next: decoder.body.present.7.value
      - cell: cache_16
        next: decoder.body.present.8.key
      - cell: cache_17
        next: decoder.body.present.8.value
      - cell: cache_18
        next: decoder.body.present.9.key
      - cell: cache_19
        next: decoder.body.present.9.value
      - cell: cache_20
        next: decoder.body.present.10.key
      - cell: cache_21
        next: decoder.body.present.10.value
      - cell: cache_22
        next: decoder.body.present.11.key
      - cell: cache_23
        next: decoder.body.present.11.value
      - cell: cache_24
        next: decoder.body.present.12.key
      - cell: cache_25
        next: decoder.body.present.12.value
      - cell: cache_26
        next: decoder.body.present.13.key
      - cell: cache_27
        next: decoder.body.present.13.value
      - cell: cache_28
        next: decoder.body.present.14.key
      - cell: cache_29
        next: decoder.body.present.14.value
      - cell: cache_30
        next: decoder.body.present.15.key
      - cell: cache_31
        next: decoder.body.present.15.value
      - cell: cache_32
        next: decoder.body.present.16.key
      - cell: cache_33
        next: decoder.body.present.16.value
      - cell: cache_34
        next: decoder.body.present.17.key
      - cell: cache_35
        next: decoder.body.present.17.value
      - cell: cache_36
        next: decoder.body.present.18.key
      - cell: cache_37
        next: decoder.body.present.18.value
      - cell: cache_38
        next: decoder.body.present.19.key
      - cell: cache_39
        next: decoder.body.present.19.value
      - cell: cache_40
        next: decoder.body.present.20.key
      - cell: cache_41
        next: decoder.body.present.20.value
      - cell: cache_42
        next: decoder.body.present.21.key
      - cell: cache_43
        next: decoder.body.present.21.value
      - cell: cache_44
        next: decoder.body.present.22.key
      - cell: cache_45
        next: decoder.body.present.22.value
      - cell: cache_46
        next: decoder.body.present.23.key
      - cell: cache_47
        next: decoder.body.present.23.value
      termination: generation_eos
      iteration:
        value: loop.iteration
        contract:
          dtype: int64
          rank: 1
          shape:
          - 1
package:
  tokenizer:
    special_tokens:
      eos_token_id:
      - 151645
      pad_token_id: 151643
      bos_token_id: 151643