{
  "cases": [
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 0.423,
      "fact_detail": "privacy/locality concepts present",
      "facts_correct": true,
      "fixture": null,
      "generation_tokens": 37,
      "id": "text_privacy",
      "passed": true,
      "peak_memory_gb": 3.23761089,
      "prompt": "In one concise sentence, explain how running a vision-language model entirely on-device can improve privacy for clinical documents.",
      "prompt_tokens": 30,
      "response": "Running a vision-language model entirely on-device can improve privacy for clinical documents by reducing the need to transmit sensitive data over the internet, thus minimizing the risk of data breaches and unauthorized access."
    },
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 0.0375,
      "fact_detail": "all expected facts present",
      "facts_correct": true,
      "fixture": null,
      "generation_tokens": 1,
      "id": "text_fact_extraction",
      "passed": true,
      "peak_memory_gb": 3.281032518,
      "prompt": "A synthetic note states: \"The follow-up appointment is scheduled for Tuesday at 10:30 AM.\" What day is the follow-up? Answer with only the day.",
      "prompt_tokens": 44,
      "response": "Tuesday"
    },
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 1.2672,
      "fact_detail": "all expected facts present",
      "facts_correct": true,
      "fixture": "synthetic_clinical_document.png",
      "generation_tokens": 31,
      "id": "image_clinical_document",
      "passed": true,
      "peak_memory_gb": 5.843571694,
      "prompt": "This is synthetic test data. In one concise sentence, report the exact patient name, record ID, medication with dose and frequency, and allergy shown in the image.",
      "prompt_tokens": 1161,
      "response": "The synthetic test data includes patient Alex Rivera (Record ID: SYN-2048), prescribed Metformin 500 mg twice daily, with a penicillin allergy."
    },
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 0.8766,
      "fact_detail": "all expected facts present",
      "facts_correct": true,
      "fixture": "synthetic_clinic_chart.png",
      "generation_tokens": 4,
      "id": "image_chart",
      "passed": true,
      "peak_memory_gb": 5.843571694,
      "prompt": "Which category has the tallest bar, and what exact value is printed above it? Answer concisely.",
      "prompt_tokens": 1053,
      "response": "Screening, 42"
    }
  ],
  "device": {
    "architecture": "applegpu_g15d",
    "device_name": "Apple M3 Ultra",
    "max_buffer_length": 373662154752,
    "max_recommended_working_set_size": 498216206336,
    "memory_size": 549755813888,
    "resource_limit": 499000
  },
  "load_seconds": 2.9559,
  "model_path": "OpenMed/North-Micro-Vision-Instruct-8bit-mlx",
  "passed": true,
  "runtime": "openmed.mlx.OpenMedMLXVisionLanguageModel",
  "runtime_versions": {
    "huggingface-hub": "1.27.0",
    "mlx": "0.32.0",
    "mlx-lm": "0.31.3",
    "mlx-metal": "0.32.0",
    "openmed": "1.0.0",
    "transformers": "5.15.0"
  },
  "schema_version": 1,
  "source_model": "CohereLabs/North-Micro-Vision-Instruct",
  "source_revision": "373bda96ac70bf89f99f7048f420cf00dc07c149",
  "variant": "8bit",
  "weights": {
    "actual": {
      "sha256": "912eb8d8463c14539790711b4516beee01510963b34e1d892ecfaa4838906018",
      "size": 3145292807
    },
    "embedding_health": {
      "healthy": true,
      "tensors": {
        "language_model.model.embed_tokens.biases": {
          "dtype": "mlx.core.bfloat16",
          "healthy": true,
          "nonzero_count": 8388608,
          "shape": [
            262144,
            32
          ],
          "size": 8388608
        },
        "language_model.model.embed_tokens.scales": {
          "dtype": "mlx.core.bfloat16",
          "healthy": true,
          "nonzero_count": 8388608,
          "shape": [
            262144,
            32
          ],
          "size": 8388608
        },
        "language_model.model.embed_tokens.weight": {
          "dtype": "mlx.core.uint32",
          "healthy": true,
          "nonzero_count": 134217728,
          "shape": [
            262144,
            512
          ],
          "size": 134217728
        }
      }
    },
    "payload_valid": true,
    "reference": {
      "sha256": "82c8ba545f0c9e533f8c08cfaa8420d27ca3c460470e236c28164ef9afc025e4",
      "size": 3145292807
    },
    "reference_hash_match": false,
    "reference_size_match": true
  }
}