{
  "checkpoint": "checkpoints/model.pt",
  "device": "cpu",
  "held_out_test_seeds": [
    202656,
    202657,
    202658,
    202659,
    202660,
    202661,
    202662,
    202663
  ],
  "results": [
    {
      "explorer": "random",
      "budget": 0,
      "architectures": 8,
      "queries": 320,
      "exact": 0.159375,
      "register_accuracy": 0.8380617559523833,
      "flag_accuracy": 0.8390625,
      "operation_accuracy": 0.14375,
      "semantic_exact": 0.09375,
      "query_opcode_observed_fraction": 0.0,
      "exact_when_opcode_observed": null,
      "exact_when_opcode_unobserved": 0.159375
    },
    {
      "explorer": "random",
      "budget": 1,
      "architectures": 8,
      "queries": 320,
      "exact": 0.125,
      "register_accuracy": 0.8348958333333354,
      "flag_accuracy": 0.8140625,
      "operation_accuracy": 0.15625,
      "semantic_exact": 0.084375,
      "query_opcode_observed_fraction": 0.109375,
      "exact_when_opcode_observed": 0.3142857142857143,
      "exact_when_opcode_unobserved": 0.10175438596491228
    },
    {
      "explorer": "random",
      "budget": 2,
      "architectures": 8,
      "queries": 320,
      "exact": 0.209375,
      "register_accuracy": 0.8486644345238116,
      "flag_accuracy": 0.8375,
      "operation_accuracy": 0.20625,
      "semantic_exact": 0.1,
      "query_opcode_observed_fraction": 0.190625,
      "exact_when_opcode_observed": 0.3114754098360656,
      "exact_when_opcode_unobserved": 0.18532818532818532
    },
    {
      "explorer": "random",
      "budget": 5,
      "architectures": 8,
      "queries": 320,
      "exact": 0.196875,
      "register_accuracy": 0.8642261904761931,
      "flag_accuracy": 0.825,
      "operation_accuracy": 0.2375,
      "semantic_exact": 0.128125,
      "query_opcode_observed_fraction": 0.39375,
      "exact_when_opcode_observed": 0.3333333333333333,
      "exact_when_opcode_unobserved": 0.10824742268041238
    },
    {
      "explorer": "random",
      "budget": 10,
      "architectures": 8,
      "queries": 320,
      "exact": 0.275,
      "register_accuracy": 0.8688355654761933,
      "flag_accuracy": 0.840625,
      "operation_accuracy": 0.3,
      "semantic_exact": 0.140625,
      "query_opcode_observed_fraction": 0.596875,
      "exact_when_opcode_observed": 0.4293193717277487,
      "exact_when_opcode_unobserved": 0.046511627906976744
    },
    {
      "explorer": "random",
      "budget": 20,
      "architectures": 8,
      "queries": 320,
      "exact": 0.296875,
      "register_accuracy": 0.8548474702380974,
      "flag_accuracy": 0.86875,
      "operation_accuracy": 0.3625,
      "semantic_exact": 0.20625,
      "query_opcode_observed_fraction": 0.840625,
      "exact_when_opcode_observed": 0.3420074349442379,
      "exact_when_opcode_unobserved": 0.058823529411764705
    },
    {
      "explorer": "random",
      "budget": 30,
      "architectures": 8,
      "queries": 320,
      "exact": 0.3125,
      "register_accuracy": 0.8718043154761931,
      "flag_accuracy": 0.8515625,
      "operation_accuracy": 0.390625,
      "semantic_exact": 0.23125,
      "query_opcode_observed_fraction": 0.940625,
      "exact_when_opcode_observed": 0.33222591362126247,
      "exact_when_opcode_unobserved": 0.0
    },
    {
      "explorer": "coverage",
      "budget": 0,
      "architectures": 8,
      "queries": 320,
      "exact": 0.159375,
      "register_accuracy": 0.8380617559523833,
      "flag_accuracy": 0.8390625,
      "operation_accuracy": 0.14375,
      "semantic_exact": 0.09375,
      "query_opcode_observed_fraction": 0.0,
      "exact_when_opcode_observed": null,
      "exact_when_opcode_unobserved": 0.159375
    },
    {
      "explorer": "coverage",
      "budget": 1,
      "architectures": 8,
      "queries": 320,
      "exact": 0.171875,
      "register_accuracy": 0.8447172619047638,
      "flag_accuracy": 0.8203125,
      "operation_accuracy": 0.209375,
      "semantic_exact": 0.13125,
      "query_opcode_observed_fraction": 0.08125,
      "exact_when_opcode_observed": 0.8076923076923077,
      "exact_when_opcode_unobserved": 0.11564625850340136
    },
    {
      "explorer": "coverage",
      "budget": 2,
      "architectures": 8,
      "queries": 320,
      "exact": 0.275,
      "register_accuracy": 0.8673549107142874,
      "flag_accuracy": 0.8328125,
      "operation_accuracy": 0.33125,
      "semantic_exact": 0.15625,
      "query_opcode_observed_fraction": 0.196875,
      "exact_when_opcode_observed": 0.6507936507936508,
      "exact_when_opcode_unobserved": 0.1828793774319066
    },
    {
      "explorer": "coverage",
      "budget": 5,
      "architectures": 8,
      "queries": 320,
      "exact": 0.30625,
      "register_accuracy": 0.880252976190478,
      "flag_accuracy": 0.84375,
      "operation_accuracy": 0.4,
      "semantic_exact": 0.234375,
      "query_opcode_observed_fraction": 0.4375,
      "exact_when_opcode_observed": 0.6642857142857143,
      "exact_when_opcode_unobserved": 0.027777777777777776
    },
    {
      "explorer": "coverage",
      "budget": 10,
      "architectures": 8,
      "queries": 320,
      "exact": 0.59375,
      "register_accuracy": 0.9609077380952386,
      "flag_accuracy": 0.865625,
      "operation_accuracy": 0.69375,
      "semantic_exact": 0.3375,
      "query_opcode_observed_fraction": 0.89375,
      "exact_when_opcode_observed": 0.6538461538461539,
      "exact_when_opcode_unobserved": 0.08823529411764706
    },
    {
      "explorer": "coverage",
      "budget": 20,
      "architectures": 8,
      "queries": 320,
      "exact": 0.665625,
      "register_accuracy": 0.9750446428571431,
      "flag_accuracy": 0.88125,
      "operation_accuracy": 0.815625,
      "semantic_exact": 0.465625,
      "query_opcode_observed_fraction": 1.0,
      "exact_when_opcode_observed": 0.665625,
      "exact_when_opcode_unobserved": null
    },
    {
      "explorer": "coverage",
      "budget": 30,
      "architectures": 8,
      "queries": 320,
      "exact": 0.65,
      "register_accuracy": 0.9707403273809525,
      "flag_accuracy": 0.8734375,
      "operation_accuracy": 0.75,
      "semantic_exact": 0.425,
      "query_opcode_observed_fraction": 1.0,
      "exact_when_opcode_observed": 0.65,
      "exact_when_opcode_unobserved": null
    }
  ]
}