Hugging Face's logo

miguelcsx
/
factorized-strict-small-bpe16k-tokenizer

Model card Files
xet
 Community
factorized-strict-small-bpe16k-tokenizer
File size: 3,001 Bytes
e771fea
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
{
  "corpus_null_audit": {
    "brown": {
      "pairs": 11125,
      "retained_by_prefix": [
        0,
        0,
        0
      ],
      "retained_fraction_by_prefix": [
        0.0,
        0.0,
        0.0
      ]
    },
    "ppmi": {
      "pairs": 11125,
      "retained_direct": 0,
      "retained_two_hop": 0
    }
  },
  "document_disjoint": true,
  "files": {
    "special_tokens_map.json": "2be97b602cd0e4a6c2874cbf03c0d1025b3af5474666da931bc04f6c23a9a39d",
    "tokenizer.json": "2b8d1b3f51c0d8f276a64ea6a69efa50b4d9899780f1b282b4d0dcb7be5bbb0f",
    "tokenizer_config.json": "8f4a998564d87fc85b27652f0b295677a1e03744bf400bfacc342b700cc3653a",
    "word_start_mask.npy": "80ca7ca13174695ac5f3b287377c52711103a9d2839c019ed5d45c110cee8880"
  },
  "microdocuments": {
    "null_assignment": "deterministic-repaired-permutation",
    "occurrence_cap_per_anchor": 256,
    "pair_words": 10,
    "scheduler": "round-robin-drop-exhausted"
  },
  "partition_proof": {
    "holdout": {
      "document_ids_sha256": "3f1565055bcf177a3d721f27b980409e2e88093f9b94db4451e5ec2a89a8012f",
      "documents": 9282,
      "text_sha256": "6659bdddb99e32a57b55a76fdd5841a4c3242040b1b54db63efaf3e560a9c169",
      "words": 100000
    },
    "inducer": {
      "document_ids_sha256": "1ef8aeecb319086b66c3da8c9a84c1856067c3bd4248300e71e6ca52d175295c",
      "documents": 93866,
      "text_sha256": "a3555fec3a53982285698c9c5902292006196226b7c5dff5f1c0be524358a160",
      "words": 1000000
    },
    "lm": {
      "document_ids_sha256": "38053791854cf2802344d873437b646824fa7ef1c52b23f4b70fdfd571d2eaf3",
      "documents": 835041,
      "text_sha256": "4c2dc3f0f93e3cbc2025e398e25468686f30d7a39c79aa95491d41d978926787",
      "words": 8900000
    }
  },
  "partitions": {
    "holdout": 100000,
    "inducer": 1000000,
    "lm": 8900000
  },
  "prior_audit": {
    "brown_top_5k_coverage": 1.0,
    "matched_null": {
      "degree_exact": true,
      "null_edge_overlap": 0.7830319888734353,
      "null_swaps": 6801,
      "strength_exact": true
    },
    "ppmi": {
      "mean_degree": 0.719,
      "stability_threshold": 0.8,
      "stable_edges": 719
    },
    "ppmi_top_2k_anchor_coverage": 0.411
  },
  "probe": {
    "items": 20000,
    "match_rate": 1.0,
    "matches": 20000
  },
  "provenance": {
    "brown_binary_sha256": "44851b50c73851a05e2bb7808f1491d7050e37e0e9745e5cbf0f9ebc9b20b3fd",
    "brown_commit": "fe90f87f9cd10d76c834166b4a8560eae3bc38c9",
    "builder_sha256": "621a511ad680f6e40b676eb41eb6a9493412b3d5bd8d07c036a0830997256b69",
    "graph_builder_sha256": "d5c9891df65fb37c43f3a8b5edf6672be4f382f9963a3ab9c13583f588c0eec2",
    "source_data_sha256": "3a4ada4f0c6526d4e678a0bc0d5651e86ee06b9c1245046dcb02ebdf295bf654"
  },
  "relational_words": {
    "brown": 111250,
    "ppmi": 111250
  },
  "schema_version": 2,
  "source_repo": "miguelcsx/babylm-orderlab-causal-affordance-v1",
  "source_revision": "40fda5368a4b1179020d80c1a3ebdc0a40c70dd5",
  "training_words": 8900000
}