{
  "cases": [
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 0.4045,
      "fact_detail": "privacy/locality concepts present",
      "facts_correct": true,
      "fixture": null,
      "generation_tokens": 25,
      "id": "text_privacy",
      "passed": true,
      "peak_memory_gb": 5.07432437,
      "prompt": "In one concise sentence, explain how running a vision-language model entirely on-device can improve privacy for clinical documents.",
      "prompt_tokens": 30,
      "response": "Running a vision-language model entirely on-device reduces the need for cloud storage and transmission of sensitive clinical data, thereby enhancing privacy."
    },
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 0.0535,
      "fact_detail": "all expected facts present",
      "facts_correct": true,
      "fixture": null,
      "generation_tokens": 1,
      "id": "text_fact_extraction",
      "passed": true,
      "peak_memory_gb": 5.083239598,
      "prompt": "A synthetic note states: \"The follow-up appointment is scheduled for Tuesday at 10:30 AM.\" What day is the follow-up? Answer with only the day.",
      "prompt_tokens": 44,
      "response": "Tuesday"
    },
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 1.3907,
      "fact_detail": "all expected facts present",
      "facts_correct": true,
      "fixture": "synthetic_clinical_document.png",
      "generation_tokens": 31,
      "id": "image_clinical_document",
      "passed": true,
      "peak_memory_gb": 7.668086766,
      "prompt": "This is synthetic test data. In one concise sentence, report the exact patient name, record ID, medication with dose and frequency, and allergy shown in the image.",
      "prompt_tokens": 1161,
      "response": "The synthetic test data includes patient Alex Rivera, record ID SYN-2048, medication Metformin 500 mg twice daily, and allergy to Penicillin."
    },
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 0.8995,
      "fact_detail": "all expected facts present",
      "facts_correct": true,
      "fixture": "synthetic_clinic_chart.png",
      "generation_tokens": 4,
      "id": "image_chart",
      "passed": true,
      "peak_memory_gb": 7.668086766,
      "prompt": "Which category has the tallest bar, and what exact value is printed above it? Answer concisely.",
      "prompt_tokens": 1053,
      "response": "Screening, 42"
    }
  ],
  "device": {
    "architecture": "applegpu_g15d",
    "device_name": "Apple M3 Ultra",
    "max_buffer_length": 373662154752,
    "max_recommended_working_set_size": 498216206336,
    "memory_size": 549755813888,
    "resource_limit": 499000
  },
  "load_seconds": 3.0775,
  "model_path": "OpenMed/North-Micro-Vision-Instruct-bf16-mlx",
  "passed": true,
  "runtime": "openmed.mlx.OpenMedMLXVisionLanguageModel",
  "runtime_versions": {
    "huggingface-hub": "1.27.0",
    "mlx": "0.32.0",
    "mlx-lm": "0.31.3",
    "mlx-metal": "0.32.0",
    "openmed": "1.0.0",
    "transformers": "5.15.0"
  },
  "schema_version": 1,
  "source_model": "CohereLabs/North-Micro-Vision-Instruct",
  "source_revision": "373bda96ac70bf89f99f7048f420cf00dc07c149",
  "variant": "bf16",
  "weights": {
    "actual": {
      "sha256": "83b1212694fe6bdb7013c81899a237933a68bfde21aacd6974f9944cb1239360",
      "size": 4969764749
    },
    "embedding_health": null,
    "payload_valid": true,
    "reference": {
      "sha256": "83b1212694fe6bdb7013c81899a237933a68bfde21aacd6974f9944cb1239360",
      "size": 4969764749
    },
    "reference_hash_match": true,
    "reference_size_match": true
  }
}