{
  "model_family": "nllb",
  "recipe_id": "nllb200-spm8k-directional-v3",
  "mt_standardization": {
    "id": "formosan-mt-standard-v3",
    "sha256": "4bbded87eb4833b2d1cd5a88a1f2b059560416e0e69ecfa55433a77f418c903e",
    "namespace": "formosan-mt"
  },
  "step": 220000,
  "best_metric": "chrF2",
  "best_value": 11.256583041753892,
  "direction": "f2zh",
  "run_contract_sha256": "4cfe683caf4d25810d668f761921cb154a2d41e4e83822352faacc2d5b14293f",
  "validation": {
    "mean_token_loss": 3.481121237640053,
    "ppl": 32.49613754942568,
    "by_language": {
      "ami": 4.034636270919837,
      "bnn": 3.5769809744689085,
      "ckv": 2.955491952107692,
      "dru": 3.4944524112298745,
      "pwn": 3.7311584976757053,
      "pyu": 2.6635534406284203,
      "ssf": 3.8926185333821532,
      "sxr": 2.5630961256169162,
      "szy": 3.0292117092905326,
      "tao": 3.5835890884774324,
      "tay": 3.1955901890426928,
      "trv": 3.134190788581884,
      "tsu": 4.479148868581087,
      "xnb": 1.7955884545588958,
      "xsy": 4.265404218997709
    },
    "generation": {
      "samples": 1920,
      "bleu_tokenize": "zh",
      "global": {
        "BLEU": 9.412108007110993,
        "chrF2": 11.256583041753892,
        "TER": 90.38131436978911,
        "exact_match_rate": 0.010416666666666666,
        "empty_output_rate": 0.0,
        "character_length_ratio": 0.9647018635897825,
        "signatures": {
          "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
          "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
          "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
        }
      },
      "by_language": {
        "ami": {
          "samples": 128,
          "BLEU": 4.367336716716484,
          "chrF2": 7.79001854487594,
          "TER": 94.51874175927671,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.8488700564971752,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          }
        },
        "bnn": {
          "samples": 128,
          "BLEU": 7.43278344662541,
          "chrF2": 8.30436731110693,
          "TER": 86.1777627576884,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.8770355599867066,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          }
        },
        "ckv": {
          "samples": 128,
          "BLEU": 13.913159689280391,
          "chrF2": 14.232097621479383,
          "TER": 76.30648330058939,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9460856720827179,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          }
        },
        "dru": {
          "samples": 128,
          "BLEU": 7.257745876888657,
          "chrF2": 11.379785118423582,
          "TER": 100.21342186388429,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.0559664201479113,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          }
        },
        "pwn": {
          "samples": 128,
          "BLEU": 6.1170651870234245,
          "chrF2": 7.937279691190215,
          "TER": 94.11955815464587,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.921300516560316,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          }
        },
        "pyu": {
          "samples": 128,
          "BLEU": 10.846317334976424,
          "chrF2": 15.366836715237858,
          "TER": 78.68571428571428,
          "exact_match_rate": 0.0078125,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9635897435897436,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          }
        },
        "ssf": {
          "samples": 128,
          "BLEU": 6.739578125938128,
          "chrF2": 9.015721660009403,
          "TER": 89.45256520492978,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9204152249134948,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          }
        },
        "sxr": {
          "samples": 128,
          "BLEU": 13.977859965034813,
          "chrF2": 15.815619449135317,
          "TER": 70.94994087504926,
          "exact_match_rate": 0.015625,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.8838782412626832,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          }
        },
        "szy": {
          "samples": 128,
          "BLEU": 14.160740671900028,
          "chrF2": 16.97155381389867,
          "TER": 88.17733990147784,
          "exact_match_rate": 0.046875,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.0388719512195121,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          }
        },
        "tao": {
          "samples": 128,
          "BLEU": 6.998578356091424,
          "chrF2": 16.04089062942359,
          "TER": 90.4168922577152,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9255461435577351,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          }
        },
        "tay": {
          "samples": 128,
          "BLEU": 11.001312009737747,
          "chrF2": 10.895381601226708,
          "TER": 84.46982055464926,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.940184537066497,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          }
        },
        "trv": {
          "samples": 128,
          "BLEU": 11.434671909068165,
          "chrF2": 12.101809517498904,
          "TER": 84.21642837118026,
          "exact_match_rate": 0.0078125,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.00375234521576,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          }
        },
        "tsu": {
          "samples": 128,
          "BLEU": 4.998406609073161,
          "chrF2": 5.827140351620994,
          "TER": 109.9028227672374,
          "exact_match_rate": 0.0,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.0106544901065448,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          }
        },
        "xnb": {
          "samples": 128,
          "BLEU": 25.945659871800135,
          "chrF2": 24.697976362629582,
          "TER": 55.62326869806095,
          "exact_match_rate": 0.046875,
          "empty_output_rate": 0.0,
          "character_length_ratio": 0.9163598106259864,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          }
        },
        "xsy": {
          "samples": 128,
          "BLEU": 6.906682364790633,
          "chrF2": 8.045205239872994,
          "TER": 111.64319248826291,
          "exact_match_rate": 0.03125,
          "empty_output_rate": 0.0,
          "character_length_ratio": 1.1322869955156951,
          "signatures": {
            "BLEU": "nrefs:1|case:mixed|eff:yes|tok:zh|smooth:exp|version:2.5.1",
            "chrF2": "nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.5.1",
            "TER": "nrefs:1|case:mixed|tok:tercom|norm:yes|punct:yes|asian:yes|version:2.5.1"
          }
        }
      }
    },
    "step": 220000,
    "selection": {
      "metric": "chrF2",
      "value": 11.256583041753892,
      "improved": true,
      "best_value_before_eval": 11.132391003032263
    }
  }
}