Upload folder using huggingface_hub

manifest.json

@@ -0,0 +1,95 @@
+{
+  "corpus_null_audit": {
+    "brown": {
+      "pairs": 11125,
+      "retained_by_prefix": [
+        0,
+        0,
+        0
+      ],
+      "retained_fraction_by_prefix": [
+        0.0,
+        0.0,
+        0.0
+      ]
+    },
+    "ppmi": {
+      "pairs": 11125,
+      "retained_direct": 0,
+      "retained_two_hop": 0
+    }
+  },
+  "document_disjoint": true,
+  "files": {
+    "special_tokens_map.json": "2be97b602cd0e4a6c2874cbf03c0d1025b3af5474666da931bc04f6c23a9a39d",
+    "tokenizer.json": "2b8d1b3f51c0d8f276a64ea6a69efa50b4d9899780f1b282b4d0dcb7be5bbb0f",
+    "tokenizer_config.json": "8f4a998564d87fc85b27652f0b295677a1e03744bf400bfacc342b700cc3653a",
+    "word_start_mask.npy": "80ca7ca13174695ac5f3b287377c52711103a9d2839c019ed5d45c110cee8880"
+  },
+  "microdocuments": {
+    "null_assignment": "deterministic-repaired-permutation",
+    "occurrence_cap_per_anchor": 256,
+    "pair_words": 10,
+    "scheduler": "round-robin-drop-exhausted"
+  },
+  "partition_proof": {
+    "holdout": {
+      "document_ids_sha256": "3f1565055bcf177a3d721f27b980409e2e88093f9b94db4451e5ec2a89a8012f",
+      "documents": 9282,
+      "text_sha256": "6659bdddb99e32a57b55a76fdd5841a4c3242040b1b54db63efaf3e560a9c169",
+      "words": 100000
+    },
+    "inducer": {
+      "document_ids_sha256": "1ef8aeecb319086b66c3da8c9a84c1856067c3bd4248300e71e6ca52d175295c",
+      "documents": 93866,
+      "text_sha256": "a3555fec3a53982285698c9c5902292006196226b7c5dff5f1c0be524358a160",
+      "words": 1000000
+    },
+    "lm": {
+      "document_ids_sha256": "38053791854cf2802344d873437b646824fa7ef1c52b23f4b70fdfd571d2eaf3",
+      "documents": 835041,
+      "text_sha256": "4c2dc3f0f93e3cbc2025e398e25468686f30d7a39c79aa95491d41d978926787",
+      "words": 8900000
+    }
+  },
+  "partitions": {
+    "holdout": 100000,
+    "inducer": 1000000,
+    "lm": 8900000
+  },
+  "prior_audit": {
+    "brown_top_5k_coverage": 1.0,
+    "matched_null": {
+      "degree_exact": true,
+      "null_edge_overlap": 0.7830319888734353,
+      "null_swaps": 6801,
+      "strength_exact": true
+    },
+    "ppmi": {
+      "mean_degree": 0.719,
+      "stability_threshold": 0.8,
+      "stable_edges": 719
+    },
+    "ppmi_top_2k_anchor_coverage": 0.411
+  },
+  "probe": {
+    "items": 20000,
+    "match_rate": 1.0,
+    "matches": 20000
+  },
+  "provenance": {
+    "brown_binary_sha256": "44851b50c73851a05e2bb7808f1491d7050e37e0e9745e5cbf0f9ebc9b20b3fd",
+    "brown_commit": "fe90f87f9cd10d76c834166b4a8560eae3bc38c9",
+    "builder_sha256": "621a511ad680f6e40b676eb41eb6a9493412b3d5bd8d07c036a0830997256b69",
+    "graph_builder_sha256": "d5c9891df65fb37c43f3a8b5edf6672be4f382f9963a3ab9c13583f588c0eec2",
+    "source_data_sha256": "3a4ada4f0c6526d4e678a0bc0d5651e86ee06b9c1245046dcb02ebdf295bf654"
+  },
+  "relational_words": {
+    "brown": 111250,
+    "ppmi": 111250
+  },
+  "schema_version": 2,
+  "source_repo": "miguelcsx/babylm-orderlab-causal-affordance-v1",
+  "source_revision": "40fda5368a4b1179020d80c1a3ebdc0a40c70dd5",
+  "training_words": 8900000
+}

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<unk>"
+}

word_start_mask.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80ca7ca13174695ac5f3b287377c52711103a9d2839c019ed5d45c110cee8880
+size 16512