{
  "model_family": "nllb",
  "recipe_id": "nllb200-spm8k-directional-v3",
  "mt_standardization": {
    "id": "formosan-mt-standard-v3",
    "sha256": "4bbded87eb4833b2d1cd5a88a1f2b059560416e0e69ecfa55433a77f418c903e",
    "namespace": "formosan-mt"
  },
  "step": 210000,
  "best_metric": "chrF2",
  "best_value": 29.69675669457501,
  "direction": "f2en",
  "run_contract_sha256": "aecd06637deb76f5e92f1f74099539741859863bfdb1ee63bdc2bde875205c09",
  "validation": {
    "mean_token_loss": 3.3026913455802602,
    "ppl": 27.18570668278096,
    "by_language": {
      "ami": 2.7741046558310103,
      "bnn": 2.4203313634080708,
      "ckv": 3.8672077235665663,
      "dru": 2.6078648727627933,
      "pwn": 2.528691228550362,
      "pyu": 2.6529325262628793,
      "ssf": 3.959471394623352,
      "sxr": 3.549196793319494,
      "szy": 3.1354670804153746,
      "tao": 5.619721773664961,
      "tay": 2.977861526899009,
      "trv": 3.2536238391231334,
      "tsu": 2.991666843242918,
      "xnb": 4.993570713988536,
      "xsy": 3.4992934907904347
    },
    "generation": {
      "samples": 1920,
      "bleu_tokenize": "13a",
      "global": {
        "BLEU": 9.527384853249911,
        "chrF2": 29.69675669457501,
        "TER": 92.68359895283149,
        "exact_match_rate": 0.00625,
        "empty_output_rate": 0.0,
        "character_length_ratio": 1.0418471274597054,
        "signatures": {
          "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
          "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
          "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
        }
      },
      "by_language": {
        "ami": {
          "samples": 128,
          "BLEU": 9.238455573040238,
          "chrF2": 31.95266793741873,
          "TER": 87.55020080321285,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9711417816813049,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          }
        },
        "bnn": {
          "samples": 128,
          "BLEU": 15.111661288601976,
          "chrF2": 37.447343289996,
          "TER": 94.64285714285714,
          "exact_match_rate": 0.015625,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.1645185746777862,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          }
        },
        "ckv": {
          "samples": 128,
          "BLEU": 8.31432371756987,
          "chrF2": 28.393994850834748,
          "TER": 90.07751937984496,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.0116120218579234,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          }
        },
        "dru": {
          "samples": 128,
          "BLEU": 7.778519779287307,
          "chrF2": 25.795375902173102,
          "TER": 88.82226980728052,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.918150488225158,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          }
        },
        "pwn": {
          "samples": 128,
          "BLEU": 8.658812231685097,
          "chrF2": 32.22194063034251,
          "TER": 89.4127732533219,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.0827989963758016,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          }
        },
        "pyu": {
          "samples": 128,
          "BLEU": 16.483653914447636,
          "chrF2": 37.10931345333428,
          "TER": 80.96856414613424,
          "exact_match_rate": 0.0078125,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.1018188636789799,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          }
        },
        "ssf": {
          "samples": 128,
          "BLEU": 5.545373975940428,
          "chrF2": 28.047744106886313,
          "TER": 109.63855421686748,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.2119829235978212,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          }
        },
        "sxr": {
          "samples": 128,
          "BLEU": 7.707769211359447,
          "chrF2": 29.688050193906157,
          "TER": 97.93991416309012,
          "exact_match_rate": 0.0078125,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.0473008616142079,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          }
        },
        "szy": {
          "samples": 128,
          "BLEU": 10.566286844226358,
          "chrF2": 31.689864087527926,
          "TER": 89.02517753389283,
          "exact_match_rate": 0.0078125,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.0423351759471484,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          }
        },
        "tao": {
          "samples": 128,
          "BLEU": 1.5669790931362446,
          "chrF2": 25.267381629423703,
          "TER": 122.75600505689002,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.2265066394279878,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          }
        },
        "tay": {
          "samples": 128,
          "BLEU": 7.238946215102692,
          "chrF2": 24.056495084442492,
          "TER": 87.7221324717286,
          "exact_match_rate": 0.015625,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.827789393651397,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          }
        },
        "trv": {
          "samples": 128,
          "BLEU": 9.010710983806707,
          "chrF2": 29.99822177170911,
          "TER": 95.42947202521671,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.1590753424657534,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          }
        },
        "tsu": {
          "samples": 128,
          "BLEU": 14.586628714061831,
          "chrF2": 30.754865405687898,
          "TER": 77.6869893148963,
          "exact_match_rate": 0.0234375,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9626507775032699,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          }
        },
        "xnb": {
          "samples": 128,
          "BLEU": 5.352878317943197,
          "chrF2": 23.01127510854248,
          "TER": 109.5948827292111,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.0608658982288446,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          }
        },
        "xsy": {
          "samples": 128,
          "BLEU": 12.065210750445159,
          "chrF2": 30.269374379711817,
          "TER": 89.06942392909897,
          "exact_match_rate": 0.015625,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9753761969904241,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:13a|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:no|version:2.5.1"
          }
        }
      }
    },
    "step": 210000,
    "selection": {
      "metric": "chrF2",
      "value": 29.69675669457501,
      "improved": true,
      "best_value_before_eval": 29.56158400843854
    }
  }
}