| { | |
| "known_overflow_failure_record": "train:10076:f94b98732ab74f1f9b59a5162fdc1f1a", | |
| "untruncated_tokens": 553, | |
| "manual_window_lengths": [ | |
| 512, | |
| 87 | |
| ], | |
| "token_ids_and_offsets_reconstruction_exact": true, | |
| "cache_train_stats": { | |
| "documents": 89307, | |
| "chunks": 99258, | |
| "tokens": 24285133, | |
| "ignored_boundary_tokens": 16, | |
| "partial_window_entity_tokens": 2741 | |
| }, | |
| "train_content_tokens": 23668675, | |
| "special_tokens": 198516, | |
| "repeated_overlap_tokens": 417942, | |
| "complete_token_accounting_verified": true | |
| } |