summary.json

{
  "Qwen/Qwen2.5-7B-Instruct": {
    "model": "Qwen/Qwen2.5-7B-Instruct",
    "fmt": "json",
    "n_items": 120,
    "overall_acc": 0.1917,
    "elapsed_sec": 27.8,
    "by_cat": {
      "Topology": {
        "acc": 0.3333,
        "n": 60
      },
      "Metric": {
        "acc": 0.075,
        "n": 40
      },
      "Dynamic": {
        "acc": 0.0,
        "n": 20
      }
    },
    "by_rt": {
      "kitchen": {
        "acc": 0.2,
        "n": 30
      },
      "living_room": {
        "acc": 0.2667,
        "n": 30
      },
      "bedroom": {
        "acc": 0.2,
        "n": 30
      },
      "hssd": {
        "acc": 0.1,
        "n": 30
      }
    }
  },
  "Qwen/Qwen3-32B": {
    "model": "Qwen/Qwen3-32B",
    "fmt": "json",
    "n_items": 120,
    "overall_acc": 0.3417,
    "elapsed_sec": 405.9,
    "by_cat": {
      "Metric": {
        "acc": 0.025,
        "n": 40
      },
      "Topology": {
        "acc": 0.3667,
        "n": 60
      },
      "Dynamic": {
        "acc": 0.9,
        "n": 20
      }
    },
    "by_rt": {
      "kitchen": {
        "acc": 0.2667,
        "n": 30
      },
      "living_room": {
        "acc": 0.3667,
        "n": 30
      },
      "bedroom": {
        "acc": 0.3667,
        "n": 30
      },
      "hssd": {
        "acc": 0.3667,
        "n": 30
      }
    }
  },
  "Qwen/Qwen2.5-72B-Instruct": {
    "model": "Qwen/Qwen2.5-72B-Instruct",
    "fmt": "json",
    "n_items": 120,
    "overall_acc": 0.1917,
    "elapsed_sec": 122.7,
    "by_cat": {
      "Topology": {
        "acc": 0.3333,
        "n": 60
      },
      "Metric": {
        "acc": 0.075,
        "n": 40
      },
      "Dynamic": {
        "acc": 0.0,
        "n": 20
      }
    },
    "by_rt": {
      "kitchen": {
        "acc": 0.2,
        "n": 30
      },
      "living_room": {
        "acc": 0.2,
        "n": 30
      },
      "bedroom": {
        "acc": 0.2,
        "n": 30
      },
      "hssd": {
        "acc": 0.1667,
        "n": 30
      }
    }
  },
  "xml_7b": {
    "model": "Qwen/Qwen2.5-7B-Instruct",
    "fmt": "xml",
    "n_items": 120,
    "overall_acc": 0.1917,
    "elapsed_sec": 25.7,
    "by_cat": {
      "Metric": {
        "acc": 0.1,
        "n": 40
      },
      "Topology": {
        "acc": 0.3167,
        "n": 60
      },
      "Dynamic": {
        "acc": 0.0,
        "n": 20
      }
    },
    "by_rt": {
      "kitchen": {
        "acc": 0.2333,
        "n": 30
      },
      "living_room": {
        "acc": 0.2333,
        "n": 30
      },
      "bedroom": {
        "acc": 0.1667,
        "n": 30
      },
      "hssd": {
        "acc": 0.1333,
        "n": 30
      }
    }
  },
  "perturb_7b": {
    "model": "Qwen/Qwen2.5-7B-Instruct",
    "fmt": "json",
    "n_items": 69,
    "overall_acc": 0.2174,
    "elapsed_sec": 15.6,
    "by_cat": {
      "Metric": {
        "acc": 0.0455,
        "n": 22
      },
      "Topology": {
        "acc": 0.4,
        "n": 35
      },
      "Dynamic": {
        "acc": 0.0,
        "n": 12
      }
    },
    "by_rt": {
      "kitchen": {
        "acc": 0.2222,
        "n": 18
      },
      "living_room": {
        "acc": 0.2778,
        "n": 18
      },
      "bedroom": {
        "acc": 0.1667,
        "n": 18
      },
      "hssd": {
        "acc": 0.2,
        "n": 15
      }
    }
  }
}