examples/family_examples.json

{
  "format": "tiny_receipt_vqa_family_examples_v1",
  "scope": "bpe1536_explicit_and_auto_answer_exact_smoke_test",
  "model": {
    "tokenizer": "byte_fallback_bpe",
    "vocab_size": 1536,
    "target_schema": "direct_answer_no_value_v1"
  },
  "source": {
    "dataset": "synthetic validation split",
    "heldout_used": false,
    "selection": "visually clear English and Korean receipts that pass all seven listed family checks plus one phone-position check under explicit and AUTO routing in both deployment variants"
  },
  "families": {
    "tested": [
      "phone",
      "address",
      "store",
      "item_row",
      "item_math",
      "item_lookup",
      "math"
    ],
    "trained_structured_families": [
      "phone",
      "address",
      "store",
      "item_row",
      "item_math",
      "item_lookup"
    ],
    "execution_only_untrained_slots": [
      "math"
    ],
    "excluded_untrained_slots": [
      "other"
    ],
    "note": "The training distribution contains records for the six structured families only. The explicit math case verifies shared-backbone and adapter-slot execution; it is not evidence of separately trained math-family accuracy."
  },
  "validation": {
    "case_count": 16,
    "claim": "answer exact; selected-family checks are reported separately",
    "explicit_family_override": {
      "fp32": {
        "passed": 16,
        "total": 16,
        "selected_family_matches": 16
      },
      "int8_w8a8": {
        "passed": 16,
        "total": 16,
        "selected_family_matches": 16
      }
    },
    "learned_auto_router": {
      "fp32": {
        "passed": 16,
        "total": 16,
        "listed_family_matches": 14
      },
      "int8_w8a8": {
        "passed": 16,
        "total": 16,
        "listed_family_matches": 14
      },
      "selected_family_note": "AUTO selects trained item_math for both listed math questions and still produces the exact answers."
    },
    "math_note": "The selected explicit math smoke cases produce the exact structured outputs, but math still has zero family-specific training records and is not a family-level accuracy claim."
  },
  "examples": [
    {
      "id": "receipt_en",
      "language": "en",
      "image": "receipt_en.jpg",
      "image_sha256": "24d7dde7d28bec844fee9134e6cc8d3180ee85315d6d493cc39c7cf0e1ebf328",
      "source_split": "synth/data/val",
      "source_id": "000868",
      "receipt": {
        "store": "Best price Mall",
        "address": "Seoul Hwasun Jodonan St. 837",
        "phone": "653-3831",
        "items": [
          {
            "name": "Packed Yogurt",
            "unit_price": 600,
            "quantity": 3,
            "total": 1800
          },
          {
            "name": "HACCP Salt",
            "unit_price": 100,
            "quantity": 4,
            "total": 400
          }
        ]
      },
      "cases": [
        {
          "family": "phone",
          "question": "What is the store phone number?",
          "expected_answer": "6533831",
          "validated_answers": {
            "fp32": "6533831",
            "int8_w8a8": "6533831"
          },
          "validated_auto": {
            "fp32": {
              "answer": "6533831",
              "selected_family": "phone"
            },
            "int8_w8a8": {
              "answer": "6533831",
              "selected_family": "phone"
            }
          }
        },
        {
          "family": "address",
          "question": "What is the street number in the store address?",
          "expected_answer": "837",
          "validated_answers": {
            "fp32": "837",
            "int8_w8a8": "837"
          },
          "validated_auto": {
            "fp32": {
              "answer": "837",
              "selected_family": "address"
            },
            "int8_w8a8": {
              "answer": "837",
              "selected_family": "address"
            }
          }
        },
        {
          "family": "store",
          "question": "What is the store name?",
          "expected_answer": "Best price Mall",
          "validated_answers": {
            "fp32": "Best price Mall",
            "int8_w8a8": "Best price Mall"
          },
          "validated_auto": {
            "fp32": {
              "answer": "Best price Mall",
              "selected_family": "store"
            },
            "int8_w8a8": {
              "answer": "Best price Mall",
              "selected_family": "store"
            }
          }
        },
        {
          "family": "item_row",
          "question": "What is the name of item 1?",
          "expected_answer": "Packed Yogurt",
          "validated_answers": {
            "fp32": "Packed Yogurt",
            "int8_w8a8": "Packed Yogurt"
          },
          "validated_auto": {
            "fp32": {
              "answer": "Packed Yogurt",
              "selected_family": "item_row"
            },
            "int8_w8a8": {
              "answer": "Packed Yogurt",
              "selected_family": "item_row"
            }
          }
        },
        {
          "family": "item_math",
          "question": "What is the sum of item totals?",
          "expected_answer": "2200",
          "validated_answers": {
            "fp32": "2200",
            "int8_w8a8": "2200"
          },
          "validated_auto": {
            "fp32": {
              "answer": "2200",
              "selected_family": "item_math"
            },
            "int8_w8a8": {
              "answer": "2200",
              "selected_family": "item_math"
            }
          }
        },
        {
          "family": "item_lookup",
          "question": "What is the quantity of Packed Yogurt?",
          "expected_answer": "3",
          "validated_answers": {
            "fp32": "3",
            "int8_w8a8": "3"
          },
          "validated_auto": {
            "fp32": {
              "answer": "3",
              "selected_family": "item_lookup"
            },
            "int8_w8a8": {
              "answer": "3",
              "selected_family": "item_lookup"
            }
          }
        },
        {
          "family": "math",
          "question": "Calculate item 1 unit price multiplied by quantity.",
          "expected_answer": "1800",
          "validated_answers": {
            "fp32": "1800",
            "int8_w8a8": "1800"
          },
          "validated_auto": {
            "fp32": {
              "answer": "1800",
              "selected_family": "item_math"
            },
            "int8_w8a8": {
              "answer": "1800",
              "selected_family": "item_math"
            }
          }
        }
      ],
      "additional_cases": [
        {
          "case_type": "phone_digit_position",
          "family": "phone",
          "question": "What is the second number of the store's phone number?",
          "expected_answer": "5",
          "validated_answers": {
            "fp32": "5",
            "int8_w8a8": "5"
          },
          "validated_auto": {
            "fp32": {
              "answer": "5",
              "selected_family": "phone"
            },
            "int8_w8a8": {
              "answer": "5",
              "selected_family": "phone"
            }
          }
        }
      ]
    },
    {
      "id": "receipt_ko",
      "language": "ko",
      "image": "receipt_ko.jpg",
      "image_sha256": "0c9e7b52a5205eddd7944046a6db35eac2b2034535bfef34edc23b666e41d5ff",
      "source_split": "synth/data/val",
      "source_id": "002267",
      "receipt": {
        "store": "착한 도매슈퍼마켓",
        "address": "의정부시 도봉구 엄치길 875",
        "phone": "911-3125",
        "items": [
          {
            "name": "과일류",
            "unit_price": 100,
            "quantity": 4,
            "total": 400
          },
          {
            "name": "더블샷 녹차",
            "unit_price": 400,
            "quantity": 8,
            "total": 3200
          }
        ]
      },
      "cases": [
        {
          "family": "phone",
          "question": "가게 전화번호는 무엇입니까?",
          "expected_answer": "9113125",
          "validated_answers": {
            "fp32": "9113125",
            "int8_w8a8": "9113125"
          },
          "validated_auto": {
            "fp32": {
              "answer": "9113125",
              "selected_family": "phone"
            },
            "int8_w8a8": {
              "answer": "9113125",
              "selected_family": "phone"
            }
          }
        },
        {
          "family": "address",
          "question": "영수증의 가게 위치에서 도로명 뒤 숫자는 무엇입니까?",
          "expected_answer": "875",
          "validated_answers": {
            "fp32": "875",
            "int8_w8a8": "875"
          },
          "validated_auto": {
            "fp32": {
              "answer": "875",
              "selected_family": "address"
            },
            "int8_w8a8": {
              "answer": "875",
              "selected_family": "address"
            }
          }
        },
        {
          "family": "store",
          "question": "가게 이름은 무엇입니까?",
          "expected_answer": "착한 도매슈퍼마켓",
          "validated_answers": {
            "fp32": "착한 도매슈퍼마켓",
            "int8_w8a8": "착한 도매슈퍼마켓"
          },
          "validated_auto": {
            "fp32": {
              "answer": "착한 도매슈퍼마켓",
              "selected_family": "store"
            },
            "int8_w8a8": {
              "answer": "착한 도매슈퍼마켓",
              "selected_family": "store"
            }
          }
        },
        {
          "family": "item_row",
          "question": "1번째 품목명은 무엇입니까?",
          "expected_answer": "과일류",
          "validated_answers": {
            "fp32": "과일류",
            "int8_w8a8": "과일류"
          },
          "validated_auto": {
            "fp32": {
              "answer": "과일류",
              "selected_family": "item_row"
            },
            "int8_w8a8": {
              "answer": "과일류",
              "selected_family": "item_row"
            }
          }
        },
        {
          "family": "item_math",
          "question": "품목 총합을 모두 더하면 얼마입니까?",
          "expected_answer": "3600",
          "validated_answers": {
            "fp32": "3600",
            "int8_w8a8": "3600"
          },
          "validated_auto": {
            "fp32": {
              "answer": "3600",
              "selected_family": "item_math"
            },
            "int8_w8a8": {
              "answer": "3600",
              "selected_family": "item_math"
            }
          }
        },
        {
          "family": "item_lookup",
          "question": "과일류 개수는 몇 개입니까?",
          "expected_answer": "4",
          "validated_answers": {
            "fp32": "4",
            "int8_w8a8": "4"
          },
          "validated_auto": {
            "fp32": {
              "answer": "4",
              "selected_family": "item_lookup"
            },
            "int8_w8a8": {
              "answer": "4",
              "selected_family": "item_lookup"
            }
          }
        },
        {
          "family": "math",
          "question": "1번째 품목의 가격 곱하기 개수는 얼마입니까?",
          "expected_answer": "400",
          "validated_answers": {
            "fp32": "400",
            "int8_w8a8": "400"
          },
          "validated_auto": {
            "fp32": {
              "answer": "400",
              "selected_family": "item_math"
            },
            "int8_w8a8": {
              "answer": "400",
              "selected_family": "item_math"
            }
          }
        }
      ],
      "additional_cases": [
        {
          "case_type": "phone_digit_position",
          "family": "phone",
          "question": "가게 전화번호의 뒤에서 3번째 숫자는 무엇입니까?",
          "expected_answer": "1",
          "validated_answers": {
            "fp32": "1",
            "int8_w8a8": "1"
          },
          "validated_auto": {
            "fp32": {
              "answer": "1",
              "selected_family": "phone"
            },
            "int8_w8a8": {
              "answer": "1",
              "selected_family": "phone"
            }
          }
        }
      ]
    }
  ]
}