{ "status": "complete", "config": { "model_name_or_path": "jhu-clsp/ettin-encoder-400m", "model_revision": "7662476d60abb071a5bd319c9f3074f3072c062d", "max_length": 1024, "max_candidates": 16, "seed": 42, "per_device_train_batch_size": 1, "per_device_eval_batch_size": 1, "gradient_accumulation_steps": 32, "num_train_epochs": 1, "learning_rate": 2e-05, "weight_decay": 0.01, "warmup_ratio": 0.03, "bf16": true, "gradient_checkpointing": true, "eval_strategy": "steps", "save_strategy": "steps", "eval_steps": 500, "save_steps": 500, "logging_steps": 10, "save_total_limit": 2, "report_to": "none", "max_train_per_source": 50000, "max_train_per_task": 5000, "max_eval_per_source": 2000, "scan_limit_per_split": null }, "sources": [ { "dataset": "tasksource/zero-shot-label-nli", "revision": "ee693dba923b5d5484aa9232b7357c5e45dd39b8", "adapter": "zero_shot", "aggregate": true, "configs": [ "default" ], "splits": { "train": "train", "validation": "validation", "test": "test" }, "upstream_url": "https://huggingface.co/datasets/tasksource/zero-shot-label-nli", "license_terms_reference": "https://huggingface.co/datasets/tasksource/zero-shot-label-nli" }, { "dataset": "tasksource/tasksource-instruct-v0", "revision": "1dee7ed51b87880e37882c2cbed2d50bd114e0df", "adapter": "instruct", "aggregate": true, "configs": [ "default" ], "splits": { "train": "train", "validation": "validation", "test": "test" }, "upstream_url": "https://huggingface.co/datasets/tasksource/tasksource-instruct-v0", "license_terms_reference": "https://huggingface.co/datasets/tasksource/tasksource-instruct-v0" }, { "dataset": "nyu-mll/multi_nli", "revision": "da70db2af9d09693783c3320c4249840212ee221", "adapter": "nli", "configs": [ "default" ], "splits": { "train": "train", "validation_matched": "validation", "validation_mismatched": "validation" }, "upstream_url": "https://huggingface.co/datasets/nyu-mll/multi_nli", "license_terms_reference": "https://huggingface.co/datasets/nyu-mll/multi_nli" }, { "dataset": "facebook/anli", "revision": "8e4813d81f46d313dac7892e1c28076917cfcdf9", "adapter": "nli", "configs": [ "plain_text" ], "splits": { "train_r1": "train", "train_r2": "train", "train_r3": "train", "dev_r1": "validation", "dev_r2": "validation", "dev_r3": "validation", "test_r1": "test", "test_r2": "test", "test_r3": "test" }, "upstream_url": "https://huggingface.co/datasets/facebook/anli", "license_terms_reference": "https://huggingface.co/datasets/facebook/anli" }, { "dataset": "tasksource/defeasible-nli", "revision": "7c4a57df9d8de5c36d4e9caa977907b5e8469c4f", "adapter": "defeasible", "configs": [ "atomic", "snli", "social" ], "splits": { "train": "train", "validation": "validation", "test": "test" }, "upstream_url": "https://huggingface.co/datasets/tasksource/defeasible-nli", "license_terms_reference": "https://huggingface.co/datasets/tasksource/defeasible-nli" }, { "dataset": "tasksource/FOL-nli", "revision": "6b8a2ec01b226ed871fbde4677a31146ac5f370c", "adapter": "nli", "configs": [ "default" ], "splits": { "train": "train", "validation": "validation", "test": "test" }, "upstream_url": "https://huggingface.co/datasets/tasksource/FOL-nli", "license_terms_reference": "https://huggingface.co/datasets/tasksource/FOL-nli" }, { "dataset": "tasksource/doc-nli", "revision": "9b32389b6e21e232b8114f259f762f91fb486aff", "adapter": "doc_nli", "configs": [ "default" ], "splits": { "train": "train", "validation": "validation", "test": "test" }, "upstream_url": "https://huggingface.co/datasets/tasksource/doc-nli", "license_terms_reference": "https://huggingface.co/datasets/tasksource/doc-nli" }, { "dataset": "tasksource/bigbench", "revision": "210c156767d2f4f05d2f4fd0bb275017a67040fd", "adapter": "bigbench", "aggregate": true, "configs": [ "abstract_narrative_understanding", "anachronisms", "analogical_similarity", "analytic_entailment", "arithmetic", "ascii_word_recognition", "authorship_verification", "auto_categorization", "auto_debugging", "bbq_lite_json", "bridging_anaphora_resolution_barqa", "causal_judgment", "cause_and_effect", "checkmate_in_one", "chess_state_tracking", "chinese_remainder_theorem", "cifar10_classification", "code_line_description", "codenames", "color", "common_morpheme", "conceptual_combinations", "conlang_translation", "contextual_parametric_knowledge_conflicts", "crash_blossom", "crass_ai", "cryobiology_spanish", "cryptonite", "cs_algorithms", "dark_humor_detection", "date_understanding", "disambiguation_qa", "discourse_marker_prediction", "disfl_qa", "dyck_languages", "elementary_math_qa", "emoji_movie", "emojis_emotion_prediction", "empirical_judgments", "english_proverbs", "english_russian_proverbs", "entailed_polarity", "entailed_polarity_hindi", "epistemic_reasoning", "evaluating_information_essentiality", "fact_checker", "fantasy_reasoning", "few_shot_nlg", "figure_of_speech_detection", "formal_fallacies_syllogisms_negation", "gem", "gender_inclusive_sentences_german", "general_knowledge", "geometric_shapes", "goal_step_wikihow", "gre_reading_comprehension", "hhh_alignment", "hindi_question_answering", "hindu_knowledge", "hinglish_toxicity", "human_organs_senses", "hyperbaton", "identify_math_theorems", "identify_odd_metaphor", "implicatures", "implicit_relations", "indic_cause_and_effect", "intent_recognition", "international_phonetic_alphabet_nli", "international_phonetic_alphabet_transliterate", "intersect_geometry", "irony_identification", "kanji_ascii", "kannada", "key_value_maps", "known_unknowns", "language_games", "language_identification", "linguistic_mappings", "linguistics_puzzles", "list_functions", "logic_grid_puzzle", "logical_args", "logical_deduction", "logical_fallacy_detection", "logical_sequence", "mathematical_induction", "matrixshapes", "medical_questions_russian", "metaphor_boolean", "metaphor_understanding", "minute_mysteries_qa", "misconceptions", "misconceptions_russian", "mnist_ascii", "modified_arithmetic", "moral_permissibility", "movie_dialog_same_or_different", "movie_recommendation", "mult_data_wrangling", "navigate", "nonsense_words_grammar", "novel_concepts", "object_counting", "odd_one_out", "operators", "paragraph_segmentation", "parsinlu_qa", "parsinlu_reading_comprehension", "penguins_in_a_table", "periodic_elements", "persian_idioms", "phrase_relatedness", "physical_intuition", "physics", "physics_questions", "play_dialog_same_or_different", "polish_sequence_labeling", "presuppositions_as_nli", "qa_wikidata", "question_selection", "real_or_fake_text", "reasoning_about_colored_objects", "repeat_copy_logic", "rephrase", "rhyming", "riddle_sense", "ruin_names", "salient_translation_error_detection", "scientific_press_release", "semantic_parsing_in_context_sparc", "semantic_parsing_spider", "sentence_ambiguity", "similarities_abstraction", "simp_turing_concept", "simple_arithmetic_json", "simple_arithmetic_json_subtasks", "simple_ethical_questions", "simple_text_editing", "snarks", "social_iqa", "social_support", "sports_understanding", "strange_stories", "strategyqa", "sufficient_information", "suicide_risk", "swahili_english_proverbs", "swedish_to_german_proverbs", "symbol_interpretation", "tellmewhy", "temporal_sequences", "tense", "timedial", "topical_chat", "tracking_shuffled_objects", "understanding_fables", "undo_permutation", "unit_conversion", "unit_interpretation", "unnatural_in_context_learning", "vitaminc_fact_verification", "what_is_the_tao", "which_wiki_edit", "winowhy", "word_sorting", "word_unscrambling" ], "splits": { "train": "train", "validation": "validation" }, "upstream_url": "https://huggingface.co/datasets/tasksource/bigbench", "license_terms_reference": "https://huggingface.co/datasets/tasksource/bigbench" }, { "dataset": "Rowan/hellaswag", "revision": "218ec52e09a7e7462a5400043bb9a69a41d06b76", "adapter": "hellaswag", "configs": [ "default" ], "splits": { "train": "train", "validation": "validation" }, "excluded_splits": { "test": "Labels are hidden (empty strings)." }, "upstream_url": "https://huggingface.co/datasets/Rowan/hellaswag", "license_terms_reference": "https://huggingface.co/datasets/Rowan/hellaswag" }, { "dataset": "google/boolq", "revision": "35b264d03638db9f4ce671b711558bf7ff0f80d5", "adapter": "boolq", "configs": [ "default" ], "splits": { "train": "train", "validation": "validation" }, "upstream_url": "https://huggingface.co/datasets/google/boolq", "license_terms_reference": "https://huggingface.co/datasets/google/boolq" } ], "format_version": 1, "seed": 42, "scan_limit_per_split": null, "smoke_prefix_scan": false, "partition_policy": "Native test retained; native validation: 50/50 validation/calibration with test, otherwise 50/25/25 validation/calibration/test.", "dedup_policy": "Normalized state across tasks/sources; related content groups remain in one partition. Holdouts win, with test > calibration > validation > train.", "sampling_policy": "Lowest seeded row hashes per source/output partition, then aggregate task caps, then tokenization. Caps are maxima and are not backfilled after later filtering.", "limitations": "Smoke scans use only a prefix per native split; no claim of full-corpus leakage checking. Exact/group dedup does not detect paraphrases.", "max_eval_per_source_applies_to": "each output partition", "instruct_allowlist": [ "yelp_review_full/yelp_review_full", "tweet_eval/sentiment" ], "counts": { "test": 17476, "calibration": 16267, "validation": 16947, "train": 393549 }, "tokenizer": { "name": "jhu-clsp/ettin-encoder-400m", "revision": "7662476d60abb071a5bd319c9f3074f3072c062d" }, "source_counts": { "tasksource/zero-shot-label-nli": { "raw_scanned": 1119432, "eligible:calibration": 6857, "eligible:validation": 6911, "dropped:overlapping_task_family": 68811, "dropped:blank_or_invalid_premise": 21, "eligible:test": 14030, "eligible:train": 1011905, "dropped:holdout_overlap": 10897, "sampled:test": 2000, "dropped:sampling_or_repetition:test": 12030, "written:test": 1979, "dropped:overlength:test": 20, "dropped:duplicate_content": 15, "sampled:calibration": 2000, "dropped:sampling_or_repetition:calibration": 4857, "written:calibration": 1959, "dropped:overlength:calibration": 26, "dropped:group_partition_conflict": 23, "sampled:validation": 2000, "dropped:sampling_or_repetition:validation": 4911, "written:validation": 1970, "dropped:overlength:validation": 20, "sampled:train": 50000, "dropped:sampling_or_repetition:train": 961905, "written:train": 49597, "dropped:overlength:train": 391 }, "tasksource/tasksource-instruct-v0": { "raw_scanned": 5607620, "dropped:outside_instruct_allowlist": 5545620, "eligible:calibration": 504, "eligible:validation": 496, "eligible:test": 1000, "eligible:train": 59998, "dropped:holdout_overlap": 2, "sampled:test": 1000, "dropped:sampling_or_repetition:test": 0, "dropped:overlength:test": 3, "written:test": 997, "sampled:calibration": 504, "dropped:sampling_or_repetition:calibration": 0, "written:calibration": 503, "dropped:overlength:calibration": 1, "sampled:validation": 496, "dropped:sampling_or_repetition:validation": 0, "written:validation": 495, "dropped:overlength:validation": 1, "sampled:train": 50000, "dropped:sampling_or_repetition:train": 9998, "written:train": 10000, "dropped:overlength:train": 20, "dropped:task_cap": 39980 }, "nyu-mll/multi_nli": { "raw_scanned": 412349, "eligible:test": 4945, "eligible:validation": 9666, "eligible:calibration": 5036, "eligible:train": 392301, "dropped:holdout_overlap": 401, "sampled:test": 2000, "dropped:sampling_or_repetition:test": 2945, "written:test": 2000, "sampled:calibration": 2000, "dropped:sampling_or_repetition:calibration": 3036, "written:calibration": 2000, "sampled:validation": 2000, "dropped:sampling_or_repetition:validation": 7666, "written:validation": 2000, "sampled:train": 50000, "dropped:sampling_or_repetition:train": 342301, "written:train": 49999, "dropped:duplicate_content": 1 }, "facebook/anli": { "raw_scanned": 169265, "eligible:calibration": 1677, "eligible:validation": 1523, "eligible:test": 3200, "dropped:holdout_overlap": 69400, "eligible:train": 93465, "sampled:test": 2000, "dropped:sampling_or_repetition:test": 1200, "written:test": 2000, "sampled:calibration": 1677, "dropped:sampling_or_repetition:calibration": 0, "written:calibration": 1271, "dropped:group_partition_conflict": 711, "sampled:validation": 1523, "dropped:sampling_or_repetition:validation": 0, "written:validation": 1218, "sampled:train": 50000, "dropped:sampling_or_repetition:train": 43465, "written:train": 49998, "dropped:duplicate_content": 2 }, "tasksource/defeasible-nli": { "raw_scanned": 231074, "eligible:calibration": 7461, "eligible:validation": 7507, "eligible:test": 15414, "eligible:train": 199984, "dropped:holdout_overlap": 708, "sampled:test": 2000, "dropped:sampling_or_repetition:test": 13414, "written:test": 2000, "sampled:calibration": 2000, "dropped:sampling_or_repetition:calibration": 5461, "written:calibration": 1989, "dropped:group_partition_conflict": 19, "sampled:validation": 2000, "dropped:sampling_or_repetition:validation": 5507, "written:validation": 1991, "dropped:duplicate_content": 7, "sampled:train": 50000, "dropped:sampling_or_repetition:train": 149984, "written:train": 49994 }, "tasksource/FOL-nli": { "raw_scanned": 102774, "eligible:validation": 5123, "eligible:calibration": 5155, "eligible:test": 10277, "eligible:train": 82189, "dropped:holdout_overlap": 30, "sampled:test": 2000, "dropped:sampling_or_repetition:test": 8277, "written:test": 2000, "sampled:calibration": 2000, "dropped:sampling_or_repetition:calibration": 3155, "written:calibration": 1999, "dropped:group_partition_conflict": 1, "sampled:validation": 2000, "dropped:sampling_or_repetition:validation": 3123, "written:validation": 2000, "sampled:train": 50000, "dropped:sampling_or_repetition:train": 32189, "written:train": 50000 }, "tasksource/doc-nli": { "raw_scanned": 1315851, "eligible:calibration": 105110, "eligible:validation": 105573, "eligible:test": 243460, "dropped:holdout_overlap": 87030, "eligible:train": 774678, "sampled:test": 2000, "dropped:sampling_or_repetition:test": 241460, "written:test": 1776, "dropped:overlength:test": 205, "dropped:duplicate_content": 3330, "sampled:calibration": 2000, "dropped:sampling_or_repetition:calibration": 103110, "written:calibration": 1800, "dropped:overlength:calibration": 168, "dropped:group_partition_conflict": 15, "sampled:validation": 2000, "dropped:sampling_or_repetition:validation": 103573, "written:validation": 1786, "dropped:overlength:validation": 186, "sampled:train": 50000, "dropped:sampling_or_repetition:train": 724678, "written:train": 43368, "dropped:overlength:train": 3366 }, "tasksource/bigbench": { "raw_scanned": 841339, "eligible:validation": 45873, "eligible:calibration": 22618, "eligible:test": 22828, "dropped:over_candidate_limit": 107048, "dropped:open_ended_bigbench": 283772, "dropped:blank_or_invalid_inputs": 295, "dropped:invalid_candidates": 26, "dropped:uninformative_bigbench_scores": 131, "dropped:duplicate_candidates": 24, "eligible:train": 357516, "dropped:holdout_overlap": 1207, "dropped:invalid_bigbench_scores": 1, "sampled:test": 2000, "dropped:sampling_or_repetition:test": 20828, "written:test": 1896, "dropped:overlength:test": 104, "sampled:calibration": 2000, "dropped:sampling_or_repetition:calibration": 20618, "dropped:overlength:calibration": 82, "written:calibration": 1917, "dropped:duplicate_content": 30, "sampled:validation": 2000, "dropped:sampling_or_repetition:validation": 43873, "written:validation": 1880, "dropped:overlength:validation": 120, "sampled:train": 50000, "dropped:sampling_or_repetition:train": 307516, "written:train": 42835, "dropped:overlength:train": 2827, "dropped:task_cap": 4309 }, "Rowan/hellaswag": { "raw_scanned": 49947, "eligible:validation": 5071, "eligible:test": 2502, "eligible:calibration": 2469, "eligible:train": 39880, "dropped:holdout_overlap": 25, "sampled:test": 2000, "dropped:sampling_or_repetition:test": 502, "written:test": 2000, "sampled:calibration": 2000, "dropped:sampling_or_repetition:calibration": 469, "written:calibration": 1997, "dropped:duplicate_content": 109, "sampled:validation": 2000, "dropped:sampling_or_repetition:validation": 3071, "written:validation": 1998, "sampled:train": 39880, "dropped:sampling_or_repetition:train": 0, "written:train": 39776 }, "google/boolq": { "raw_scanned": 12697, "eligible:calibration": 832, "eligible:validation": 1610, "eligible:test": 828, "eligible:train": 7982, "dropped:holdout_overlap": 1445, "sampled:test": 828, "dropped:sampling_or_repetition:test": 0, "written:test": 828, "sampled:calibration": 832, "dropped:sampling_or_repetition:calibration": 0, "written:calibration": 832, "sampled:validation": 1610, "dropped:sampling_or_repetition:validation": 0, "written:validation": 1609, "dropped:overlength:validation": 1, "sampled:train": 7982, "dropped:sampling_or_repetition:train": 0, "written:train": 7982 } }, "scans": [ { "source": "tasksource/zero-shot-label-nli", "config": "default", "split": "validation", "scanned": 14419, "scan_limit_reached": false }, { "source": "tasksource/zero-shot-label-nli", "config": "default", "split": "test", "scanned": 14680, "scan_limit_reached": false }, { "source": "tasksource/tasksource-instruct-v0", "config": "default", "split": "validation", "scanned": 142950, "scan_limit_reached": false }, { "source": "tasksource/tasksource-instruct-v0", "config": "default", "split": "test", "scanned": 150287, "scan_limit_reached": false }, { "source": "nyu-mll/multi_nli", "config": "default", "split": "validation_matched", "scanned": 9815, "scan_limit_reached": false }, { "source": "nyu-mll/multi_nli", "config": "default", "split": "validation_mismatched", "scanned": 9832, "scan_limit_reached": false }, { "source": "facebook/anli", "config": "plain_text", "split": "dev_r1", "scanned": 1000, "scan_limit_reached": false }, { "source": "facebook/anli", "config": "plain_text", "split": "dev_r2", "scanned": 1000, "scan_limit_reached": false }, { "source": "facebook/anli", "config": "plain_text", "split": "dev_r3", "scanned": 1200, "scan_limit_reached": false }, { "source": "facebook/anli", "config": "plain_text", "split": "test_r1", "scanned": 1000, "scan_limit_reached": false }, { "source": "facebook/anli", "config": "plain_text", "split": "test_r2", "scanned": 1000, "scan_limit_reached": false }, { "source": "facebook/anli", "config": "plain_text", "split": "test_r3", "scanned": 1200, "scan_limit_reached": false }, { "source": "tasksource/defeasible-nli", "config": "atomic", "split": "validation", "scanned": 3840, "scan_limit_reached": false }, { "source": "tasksource/defeasible-nli", "config": "atomic", "split": "test", "scanned": 4138, "scan_limit_reached": false }, { "source": "tasksource/defeasible-nli", "config": "snli", "split": "validation", "scanned": 1785, "scan_limit_reached": false }, { "source": "tasksource/defeasible-nli", "config": "snli", "split": "test", "scanned": 1837, "scan_limit_reached": false }, { "source": "tasksource/defeasible-nli", "config": "social", "split": "validation", "scanned": 9343, "scan_limit_reached": false }, { "source": "tasksource/defeasible-nli", "config": "social", "split": "test", "scanned": 9439, "scan_limit_reached": false }, { "source": "tasksource/FOL-nli", "config": "default", "split": "validation", "scanned": 10278, "scan_limit_reached": false }, { "source": "tasksource/FOL-nli", "config": "default", "split": "test", "scanned": 10277, "scan_limit_reached": false }, { "source": "tasksource/doc-nli", "config": "default", "split": "validation", "scanned": 210683, "scan_limit_reached": false }, { "source": "tasksource/doc-nli", "config": "default", "split": "test", "scanned": 243460, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "abstract_narrative_understanding", "split": "validation", "scanned": 600, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "anachronisms", "split": "validation", "scanned": 46, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "analogical_similarity", "split": "validation", "scanned": 64, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "analytic_entailment", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "arithmetic", "split": "validation", "scanned": 3004, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "ascii_word_recognition", "split": "validation", "scanned": 1000, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "authorship_verification", "split": "validation", "scanned": 176, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "auto_categorization", "split": "validation", "scanned": 65, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "auto_debugging", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "bbq_lite_json", "split": "validation", "scanned": 3210, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "bridging_anaphora_resolution_barqa", "split": "validation", "scanned": 129, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "causal_judgment", "split": "validation", "scanned": 38, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "cause_and_effect", "split": "validation", "scanned": 30, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "checkmate_in_one", "split": "validation", "scanned": 699, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "chess_state_tracking", "split": "validation", "scanned": 1200, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "chinese_remainder_theorem", "split": "validation", "scanned": 100, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "cifar10_classification", "split": "validation", "scanned": 4000, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "code_line_description", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "codenames", "split": "validation", "scanned": 17, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "color", "split": "validation", "scanned": 800, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "common_morpheme", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "conceptual_combinations", "split": "validation", "scanned": 19, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "conlang_translation", "split": "validation", "scanned": 32, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "contextual_parametric_knowledge_conflicts", "split": "validation", "scanned": 3505, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "crash_blossom", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "crass_ai", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "cryobiology_spanish", "split": "validation", "scanned": 29, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "cryptonite", "split": "validation", "scanned": 5231, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "cs_algorithms", "split": "validation", "scanned": 264, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "dark_humor_detection", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "date_understanding", "split": "validation", "scanned": 73, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "disambiguation_qa", "split": "validation", "scanned": 51, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "discourse_marker_prediction", "split": "validation", "scanned": 171, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "disfl_qa", "split": "validation", "scanned": 1600, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "dyck_languages", "split": "validation", "scanned": 200, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "elementary_math_qa", "split": "validation", "scanned": 7629, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "emoji_movie", "split": "validation", "scanned": 20, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "emojis_emotion_prediction", "split": "validation", "scanned": 26, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "empirical_judgments", "split": "validation", "scanned": 19, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "english_proverbs", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "english_russian_proverbs", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "entailed_polarity", "split": "validation", "scanned": 29, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "entailed_polarity_hindi", "split": "validation", "scanned": 27, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "epistemic_reasoning", "split": "validation", "scanned": 400, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "evaluating_information_essentiality", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "fact_checker", "split": "validation", "scanned": 1430, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "fantasy_reasoning", "split": "validation", "scanned": 40, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "few_shot_nlg", "split": "validation", "scanned": 30, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "figure_of_speech_detection", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "formal_fallacies_syllogisms_negation", "split": "validation", "scanned": 2840, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "gem", "split": "validation", "scanned": 2957, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "gender_inclusive_sentences_german", "split": "validation", "scanned": 40, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "general_knowledge", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "geometric_shapes", "split": "validation", "scanned": 71, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "goal_step_wikihow", "split": "validation", "scanned": 1410, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "gre_reading_comprehension", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "hhh_alignment", "split": "validation", "scanned": 42, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "hindi_question_answering", "split": "validation", "scanned": 1322, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "hindu_knowledge", "split": "validation", "scanned": 35, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "hinglish_toxicity", "split": "validation", "scanned": 40, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "human_organs_senses", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "hyperbaton", "split": "validation", "scanned": 10000, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "identify_math_theorems", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "identify_odd_metaphor", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "implicatures", "split": "validation", "scanned": 98, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "implicit_relations", "split": "validation", "scanned": 17, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "indic_cause_and_effect", "split": "validation", "scanned": 117, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "intent_recognition", "split": "validation", "scanned": 138, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "international_phonetic_alphabet_nli", "split": "validation", "scanned": 25, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "international_phonetic_alphabet_transliterate", "split": "validation", "scanned": 200, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "intersect_geometry", "split": "validation", "scanned": 49999, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "irony_identification", "split": "validation", "scanned": 19, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "kanji_ascii", "split": "validation", "scanned": 217, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "kannada", "split": "validation", "scanned": 63, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "key_value_maps", "split": "validation", "scanned": 21, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "known_unknowns", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "language_games", "split": "validation", "scanned": 424, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "language_identification", "split": "validation", "scanned": 2000, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "linguistic_mappings", "split": "validation", "scanned": 3101, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "linguistics_puzzles", "split": "validation", "scanned": 400, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "list_functions", "split": "validation", "scanned": 2050, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "logic_grid_puzzle", "split": "validation", "scanned": 200, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "logical_args", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "logical_deduction", "split": "validation", "scanned": 300, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "logical_fallacy_detection", "split": "validation", "scanned": 560, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "logical_sequence", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "mathematical_induction", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "matrixshapes", "split": "validation", "scanned": 892, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "medical_questions_russian", "split": "validation", "scanned": 51, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "metaphor_boolean", "split": "validation", "scanned": 136, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "metaphor_understanding", "split": "validation", "scanned": 46, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "minute_mysteries_qa", "split": "validation", "scanned": 94, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "misconceptions", "split": "validation", "scanned": 43, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "misconceptions_russian", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "mnist_ascii", "split": "validation", "scanned": 13996, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "modified_arithmetic", "split": "validation", "scanned": 1200, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "moral_permissibility", "split": "validation", "scanned": 68, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "movie_dialog_same_or_different", "split": "validation", "scanned": 10000, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "movie_recommendation", "split": "validation", "scanned": 100, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "mult_data_wrangling", "split": "validation", "scanned": 1474, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "navigate", "split": "validation", "scanned": 200, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "nonsense_words_grammar", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "novel_concepts", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "object_counting", "split": "validation", "scanned": 200, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "odd_one_out", "split": "validation", "scanned": 17, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "operators", "split": "validation", "scanned": 42, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "paragraph_segmentation", "split": "validation", "scanned": 1800, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "parsinlu_qa", "split": "validation", "scanned": 210, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "parsinlu_reading_comprehension", "split": "validation", "scanned": 103, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "penguins_in_a_table", "split": "validation", "scanned": 29, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "periodic_elements", "split": "validation", "scanned": 130, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "persian_idioms", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "phrase_relatedness", "split": "validation", "scanned": 20, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "physical_intuition", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "physics", "split": "validation", "scanned": 45, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "physics_questions", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "play_dialog_same_or_different", "split": "validation", "scanned": 652, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "polish_sequence_labeling", "split": "validation", "scanned": 2562, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "presuppositions_as_nli", "split": "validation", "scanned": 147, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "qa_wikidata", "split": "validation", "scanned": 4064, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "question_selection", "split": "validation", "scanned": 316, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "real_or_fake_text", "split": "validation", "scanned": 3016, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "reasoning_about_colored_objects", "split": "validation", "scanned": 400, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "repeat_copy_logic", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "rephrase", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "rhyming", "split": "validation", "scanned": 190, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "riddle_sense", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "ruin_names", "split": "validation", "scanned": 89, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "salient_translation_error_detection", "split": "validation", "scanned": 199, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "scientific_press_release", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "semantic_parsing_in_context_sparc", "split": "validation", "scanned": 231, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "semantic_parsing_spider", "split": "validation", "scanned": 206, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "sentence_ambiguity", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "similarities_abstraction", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "simp_turing_concept", "split": "validation", "scanned": 1278, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "simple_arithmetic_json", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "simple_arithmetic_json_subtasks", "split": "validation", "scanned": 15, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "simple_ethical_questions", "split": "validation", "scanned": 23, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "simple_text_editing", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "snarks", "split": "validation", "scanned": 36, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "social_iqa", "split": "validation", "scanned": 387, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "social_support", "split": "validation", "scanned": 179, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "sports_understanding", "split": "validation", "scanned": 197, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "strange_stories", "split": "validation", "scanned": 34, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "strategyqa", "split": "validation", "scanned": 457, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "sufficient_information", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "suicide_risk", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "swahili_english_proverbs", "split": "validation", "scanned": 30, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "swedish_to_german_proverbs", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "symbol_interpretation", "split": "validation", "scanned": 195, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "tellmewhy", "split": "validation", "scanned": 712, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "temporal_sequences", "split": "validation", "scanned": 200, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "tense", "split": "validation", "scanned": 57, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "timedial", "split": "validation", "scanned": 510, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "topical_chat", "split": "validation", "scanned": 4459, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "tracking_shuffled_objects", "split": "validation", "scanned": 750, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "understanding_fables", "split": "validation", "scanned": 37, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "undo_permutation", "split": "validation", "scanned": 60, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "unit_conversion", "split": "validation", "scanned": 4785, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "unit_interpretation", "split": "validation", "scanned": 20, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "unnatural_in_context_learning", "split": "validation", "scanned": 14684, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "vitaminc_fact_verification", "split": "validation", "scanned": 10933, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "what_is_the_tao", "split": "validation", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "which_wiki_edit", "split": "validation", "scanned": 114, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "winowhy", "split": "validation", "scanned": 572, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "word_sorting", "split": "validation", "scanned": 380, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "word_unscrambling", "split": "validation", "scanned": 1783, "scan_limit_reached": false }, { "source": "Rowan/hellaswag", "config": "default", "split": "validation", "scanned": 10042, "scan_limit_reached": false }, { "source": "google/boolq", "config": "default", "split": "validation", "scanned": 3270, "scan_limit_reached": false }, { "source": "tasksource/zero-shot-label-nli", "config": "default", "split": "train", "scanned": 1090333, "scan_limit_reached": false }, { "source": "tasksource/tasksource-instruct-v0", "config": "default", "split": "train", "scanned": 5314383, "scan_limit_reached": false }, { "source": "nyu-mll/multi_nli", "config": "default", "split": "train", "scanned": 392702, "scan_limit_reached": false }, { "source": "facebook/anli", "config": "plain_text", "split": "train_r1", "scanned": 16946, "scan_limit_reached": false }, { "source": "facebook/anli", "config": "plain_text", "split": "train_r2", "scanned": 45460, "scan_limit_reached": false }, { "source": "facebook/anli", "config": "plain_text", "split": "train_r3", "scanned": 100459, "scan_limit_reached": false }, { "source": "tasksource/defeasible-nli", "config": "atomic", "split": "train", "scanned": 35002, "scan_limit_reached": false }, { "source": "tasksource/defeasible-nli", "config": "snli", "split": "train", "scanned": 88674, "scan_limit_reached": false }, { "source": "tasksource/defeasible-nli", "config": "social", "split": "train", "scanned": 77016, "scan_limit_reached": false }, { "source": "tasksource/FOL-nli", "config": "default", "split": "train", "scanned": 82219, "scan_limit_reached": false }, { "source": "tasksource/doc-nli", "config": "default", "split": "train", "scanned": 861708, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "abstract_narrative_understanding", "split": "train", "scanned": 2400, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "anachronisms", "split": "train", "scanned": 184, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "analogical_similarity", "split": "train", "scanned": 259, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "analytic_entailment", "split": "train", "scanned": 54, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "arithmetic", "split": "train", "scanned": 12019, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "ascii_word_recognition", "split": "train", "scanned": 4000, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "authorship_verification", "split": "train", "scanned": 704, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "auto_categorization", "split": "train", "scanned": 263, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "auto_debugging", "split": "train", "scanned": 18, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "bbq_lite_json", "split": "train", "scanned": 12866, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "bridging_anaphora_resolution_barqa", "split": "train", "scanned": 519, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "causal_judgment", "split": "train", "scanned": 152, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "cause_and_effect", "split": "train", "scanned": 123, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "checkmate_in_one", "split": "train", "scanned": 2799, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "chess_state_tracking", "split": "train", "scanned": 4800, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "chinese_remainder_theorem", "split": "train", "scanned": 400, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "cifar10_classification", "split": "train", "scanned": 16000, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "code_line_description", "split": "train", "scanned": 44, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "codenames", "split": "train", "scanned": 68, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "color", "split": "train", "scanned": 3200, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "common_morpheme", "split": "train", "scanned": 34, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "conceptual_combinations", "split": "train", "scanned": 84, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "conlang_translation", "split": "train", "scanned": 132, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "contextual_parametric_knowledge_conflicts", "split": "train", "scanned": 14023, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "crash_blossom", "split": "train", "scanned": 22, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "crass_ai", "split": "train", "scanned": 28, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "cryobiology_spanish", "split": "train", "scanned": 117, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "cryptonite", "split": "train", "scanned": 20926, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "cs_algorithms", "split": "train", "scanned": 1056, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "dark_humor_detection", "split": "train", "scanned": 64, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "date_understanding", "split": "train", "scanned": 296, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "disambiguation_qa", "split": "train", "scanned": 207, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "discourse_marker_prediction", "split": "train", "scanned": 686, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "disfl_qa", "split": "train", "scanned": 6400, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "dyck_languages", "split": "train", "scanned": 800, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "elementary_math_qa", "split": "train", "scanned": 30531, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "emoji_movie", "split": "train", "scanned": 80, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "emojis_emotion_prediction", "split": "train", "scanned": 105, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "empirical_judgments", "split": "train", "scanned": 80, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "english_proverbs", "split": "train", "scanned": 18, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "english_russian_proverbs", "split": "train", "scanned": 64, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "entailed_polarity", "split": "train", "scanned": 119, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "entailed_polarity_hindi", "split": "train", "scanned": 111, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "epistemic_reasoning", "split": "train", "scanned": 1600, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "evaluating_information_essentiality", "split": "train", "scanned": 52, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "fact_checker", "split": "train", "scanned": 5724, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "fantasy_reasoning", "split": "train", "scanned": 161, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "few_shot_nlg", "split": "train", "scanned": 123, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "figure_of_speech_detection", "split": "train", "scanned": 43, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "formal_fallacies_syllogisms_negation", "split": "train", "scanned": 11360, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "gem", "split": "train", "scanned": 11845, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "gender_inclusive_sentences_german", "split": "train", "scanned": 160, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "general_knowledge", "split": "train", "scanned": 54, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "geometric_shapes", "split": "train", "scanned": 288, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "goal_step_wikihow", "split": "train", "scanned": 5643, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "gre_reading_comprehension", "split": "train", "scanned": 15, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "hhh_alignment", "split": "train", "scanned": 179, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "hindi_question_answering", "split": "train", "scanned": 5288, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "hindu_knowledge", "split": "train", "scanned": 140, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "hinglish_toxicity", "split": "train", "scanned": 160, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "human_organs_senses", "split": "train", "scanned": 26, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "hyperbaton", "split": "train", "scanned": 40000, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "identify_math_theorems", "split": "train", "scanned": 37, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "identify_odd_metaphor", "split": "train", "scanned": 31, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "implicatures", "split": "train", "scanned": 394, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "implicit_relations", "split": "train", "scanned": 68, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "indic_cause_and_effect", "split": "train", "scanned": 468, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "intent_recognition", "split": "train", "scanned": 555, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "international_phonetic_alphabet_nli", "split": "train", "scanned": 101, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "international_phonetic_alphabet_transliterate", "split": "train", "scanned": 803, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "intersect_geometry", "split": "train", "scanned": 50000, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "irony_identification", "split": "train", "scanned": 80, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "kanji_ascii", "split": "train", "scanned": 875, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "kannada", "split": "train", "scanned": 253, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "key_value_maps", "split": "train", "scanned": 80, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "known_unknowns", "split": "train", "scanned": 30, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "language_games", "split": "train", "scanned": 1704, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "language_identification", "split": "train", "scanned": 8000, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "linguistic_mappings", "split": "train", "scanned": 12426, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "linguistics_puzzles", "split": "train", "scanned": 1600, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "list_functions", "split": "train", "scanned": 8700, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "logic_grid_puzzle", "split": "train", "scanned": 800, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "logical_args", "split": "train", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "logical_deduction", "split": "train", "scanned": 1200, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "logical_fallacy_detection", "split": "train", "scanned": 2240, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "logical_sequence", "split": "train", "scanned": 23, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "mathematical_induction", "split": "train", "scanned": 53, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "matrixshapes", "split": "train", "scanned": 3570, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "medical_questions_russian", "split": "train", "scanned": 205, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "metaphor_boolean", "split": "train", "scanned": 544, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "metaphor_understanding", "split": "train", "scanned": 188, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "minute_mysteries_qa", "split": "train", "scanned": 383, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "misconceptions", "split": "train", "scanned": 176, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "misconceptions_russian", "split": "train", "scanned": 33, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "mnist_ascii", "split": "train", "scanned": 50000, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "modified_arithmetic", "split": "train", "scanned": 4800, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "moral_permissibility", "split": "train", "scanned": 274, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "movie_dialog_same_or_different", "split": "train", "scanned": 40000, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "movie_recommendation", "split": "train", "scanned": 400, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "mult_data_wrangling", "split": "train", "scanned": 6380, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "navigate", "split": "train", "scanned": 800, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "nonsense_words_grammar", "split": "train", "scanned": 34, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "novel_concepts", "split": "train", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "object_counting", "split": "train", "scanned": 800, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "odd_one_out", "split": "train", "scanned": 69, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "operators", "split": "train", "scanned": 168, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "paragraph_segmentation", "split": "train", "scanned": 7200, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "parsinlu_qa", "split": "train", "scanned": 840, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "parsinlu_reading_comprehension", "split": "train", "scanned": 415, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "penguins_in_a_table", "split": "train", "scanned": 120, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "periodic_elements", "split": "train", "scanned": 524, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "persian_idioms", "split": "train", "scanned": 50, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "phrase_relatedness", "split": "train", "scanned": 80, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "physical_intuition", "split": "train", "scanned": 65, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "physics", "split": "train", "scanned": 184, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "physics_questions", "split": "train", "scanned": 38, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "play_dialog_same_or_different", "split": "train", "scanned": 2612, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "polish_sequence_labeling", "split": "train", "scanned": 10250, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "presuppositions_as_nli", "split": "train", "scanned": 588, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "qa_wikidata", "split": "train", "scanned": 16257, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "question_selection", "split": "train", "scanned": 1266, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "real_or_fake_text", "split": "train", "scanned": 12072, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "reasoning_about_colored_objects", "split": "train", "scanned": 1600, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "repeat_copy_logic", "split": "train", "scanned": 16, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "rephrase", "split": "train", "scanned": 62, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "rhyming", "split": "train", "scanned": 763, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "riddle_sense", "split": "train", "scanned": 33, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "ruin_names", "split": "train", "scanned": 359, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "salient_translation_error_detection", "split": "train", "scanned": 799, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "scientific_press_release", "split": "train", "scanned": 34, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "semantic_parsing_in_context_sparc", "split": "train", "scanned": 924, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "semantic_parsing_spider", "split": "train", "scanned": 828, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "sentence_ambiguity", "split": "train", "scanned": 44, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "similarities_abstraction", "split": "train", "scanned": 60, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "simp_turing_concept", "split": "train", "scanned": 5112, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "simple_arithmetic_json", "split": "train", "scanned": 14, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "simple_arithmetic_json_subtasks", "split": "train", "scanned": 15, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "simple_ethical_questions", "split": "train", "scanned": 92, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "simple_text_editing", "split": "train", "scanned": 31, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "snarks", "split": "train", "scanned": 145, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "social_iqa", "split": "train", "scanned": 1548, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "social_support", "split": "train", "scanned": 718, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "sports_understanding", "split": "train", "scanned": 789, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "strange_stories", "split": "train", "scanned": 140, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "strategyqa", "split": "train", "scanned": 1832, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "sufficient_information", "split": "train", "scanned": 23, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "suicide_risk", "split": "train", "scanned": 24, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "swahili_english_proverbs", "split": "train", "scanned": 123, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "swedish_to_german_proverbs", "split": "train", "scanned": 56, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "symbol_interpretation", "split": "train", "scanned": 795, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "tellmewhy", "split": "train", "scanned": 2850, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "temporal_sequences", "split": "train", "scanned": 800, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "tense", "split": "train", "scanned": 229, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "timedial", "split": "train", "scanned": 2040, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "topical_chat", "split": "train", "scanned": 17836, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "tracking_shuffled_objects", "split": "train", "scanned": 3000, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "understanding_fables", "split": "train", "scanned": 152, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "undo_permutation", "split": "train", "scanned": 240, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "unit_conversion", "split": "train", "scanned": 19151, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "unit_interpretation", "split": "train", "scanned": 80, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "unnatural_in_context_learning", "split": "train", "scanned": 50000, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "vitaminc_fact_verification", "split": "train", "scanned": 43735, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "what_is_the_tao", "split": "train", "scanned": 20, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "which_wiki_edit", "split": "train", "scanned": 457, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "winowhy", "split": "train", "scanned": 2290, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "word_sorting", "split": "train", "scanned": 1520, "scan_limit_reached": false }, { "source": "tasksource/bigbench", "config": "word_unscrambling", "split": "train", "scanned": 7134, "scan_limit_reached": false }, { "source": "Rowan/hellaswag", "config": "default", "split": "train", "scanned": 39905, "scan_limit_reached": false }, { "source": "google/boolq", "config": "default", "split": "train", "scanned": 9427, "scan_limit_reached": false } ], "retained_tasks": { "tasksource/zero-shot-label-nli:test": { "rotten_tomatoes": 9, "imdb": 14, "crowdflower/political-media-bias": 10, "Touche23-ValueEval": 20, "ethics/virtue": 18, "humicroedit/subtask-2": 17, "silicone/meld_e": 15, "citation_intent": 13, "lex_glue/ledgar": 21, "dynasent/dynabench.dynasent.r2.all/r2": 18, "lexical_relation_classification/CogALexV": 11, "tweet_eval/emotion": 16, "lexical_relation_classification/K&H+N": 10, "scientific-exaggeration-detection": 15, "yahoo_answers_topics": 21, "poem_sentiment": 17, "tweet_eval/stance_climate": 18, "fever-evidence-related/mwong--fever-related": 19, "crowdflower/political-media-message": 16, "dynasent/dynabench.dynasent.r1.all/r1": 16, "silicone/dyda_e": 17, "CREAK": 21, "pragmeval/sarcasm": 18, "medical_questions_pairs": 8, "tweet_eval/stance_abortion": 16, "ade_corpus_v2/Ade_corpus_v2_classification": 18, "commonsense_qa_2.0": 9, "health_fact": 17, "crowdflower/political-media-audience": 23, "circa": 21, "silicone/iemocap": 16, "tweet_eval/hate": 16, "pragmeval/pdtb": 11, "winowhy": 20, "hyperpartisan_news": 11, "sms_spam": 12, "lexical_relation_classification/ROOT09": 21, "clcd-english": 20, "FLUTE": 14, "starcon": 19, "lexical_relation_classification/BLESS": 19, "lexical_relation_classification/EVALution": 14, "dbpedia_14/dbpedia_14": 14, "strategy-qa": 17, "cycic_classification": 7, "blog_authorship_corpus/horoscope": 16, "hlgd": 18, "paws/labeled_swap": 14, "numer_sense": 15, "implicit-hate-stg1": 16, "EffectiveFeedbackStudentWriting": 10, "phrase_similarity": 8, "tweet_eval/stance_atheism": 10, "crowdflower/airline-sentiment": 11, "MOH": 13, "sciie": 8, "hate_speech18": 17, "mbib-base/racial-bias": 13, "glue/sst2": 9, "mbib-base/gender-bias": 15, "tweet_eval/offensive": 11, "hate_speech_offensive": 18, "twentyquestions": 13, "sem_eval_2010_task_8": 16, "financial_phrasebank/sentences_allagree": 8, "mc_taco": 9, "super_glue/multirc": 8, "TuringBench": 15, "probability_words_nli/reasoning_1hop": 8, "crowdflower/tweet_global_warming": 15, "blog_authorship_corpus/job": 13, "parade": 13, "probability_words_nli/usnli": 16, "pragmeval/stac": 13, "quarel": 22, "silicone/sem": 11, "mbib-base/hate-speech": 13, "crowdflower/economic-news": 20, "liar": 10, "scruples": 13, "open_question_type": 14, "tweet_eval/emoji": 13, "wiki_qa": 16, "natural-language-satisfiability": 11, "CONDAQA": 17, "language-identification": 14, "vitaminc/tals--vitaminc": 18, "tweet_eval/irony": 13, "blog_authorship_corpus/gender": 11, "pragmeval/mrda": 17, "VUAC": 18, "amazon_polarity/amazon_polarity": 15, "logical-fallacy": 20, "mbib-base/cognitive-bias": 19, "ambient": 13, "silicone/oasis": 15, "mbib-base/linguistic-bias": 17, "PARARULE-Plus": 14, "pragmeval/gum": 13, "pragmeval/switchboard": 12, "pragmeval/verifiability": 17, "spartqa-yn": 9, "pragmeval/emergent": 12, "emo/emo2019": 9, "glue/mrpc": 13, "I2D2": 14, "tweet_eval/stance_hillary": 9, "mbib-base/political-bias": 22, "insincere-questions": 12, "discovery/discovery": 18, "ethics/justice": 12, "glue/cola": 5, "banking77": 17, "cladder": 12, "ethics/commonsense": 15, "scicite": 11, "silicone/dyda_da": 10, "ag_news": 15, "silicone/maptask": 14, "dynahate": 12, "probability_words_nli/reasoning_2hop": 13, "counterfactually-augmented-imdb": 10, "tweets_hate_speech_detection": 18, "ethics/deontology": 11, "hope_edi/english": 10, "mbib-base/text-level-bias": 12, "paws/labeled_final": 15, "avicenna": 8, "subjectivity": 15, "v1/gen_train234_test2to10": 16, "folio": 8, "crowdflower/text_emotion": 9, "crowdflower/sentiment_nuclear_power": 1, "crowdflower/corporate-messaging": 8, "silicone/meld_s": 9, "args_me": 13, "super_glue/wic": 9, "glue/qqp": 9, "rumoureval_2019/RumourEval2019": 6, "mbib-base/fake-news": 13, "ethos/binary": 10, "tweet_eval/stance_feminist": 13, "trec": 10, "snips_built_in_intents": 4 }, "tasksource/tasksource-instruct-v0:test": { "yelp_review_full/yelp_review_full": 497, "tweet_eval/sentiment": 500 }, "nyu-mll/multi_nli:test": { "nyu-mll/multi_nli/default": 2000 }, "facebook/anli:test": { "facebook/anli/plain_text": 2000 }, "tasksource/defeasible-nli:test": { "tasksource/defeasible-nli/social": 1217, "tasksource/defeasible-nli/atomic": 542, "tasksource/defeasible-nli/snli": 241 }, "tasksource/FOL-nli:test": { "tasksource/FOL-nli/default": 2000 }, "tasksource/doc-nli:test": { "tasksource/doc-nli/default": 1776 }, "tasksource/bigbench:test": { "tasksource/bigbench/vitaminc_fact_verification": 262, "tasksource/bigbench/mnist_ascii": 349, "tasksource/bigbench/cs_algorithms": 6, "tasksource/bigbench/language_identification": 39, "tasksource/bigbench/unit_conversion": 73, "tasksource/bigbench/bbq_lite_json": 61, "tasksource/bigbench/movie_dialog_same_or_different": 197, "tasksource/bigbench/fact_checker": 34, "tasksource/bigbench/social_support": 2, "tasksource/bigbench/hyperbaton": 179, "tasksource/bigbench/formal_fallacies_syllogisms_negation": 70, "tasksource/bigbench/contextual_parametric_knowledge_conflicts": 72, "tasksource/bigbench/abstract_narrative_understanding": 5, "tasksource/bigbench/goal_step_wikihow": 34, "tasksource/bigbench/real_or_fake_text": 47, "tasksource/bigbench/symbol_interpretation": 5, "tasksource/bigbench/arithmetic": 74, "tasksource/bigbench/timedial": 18, "tasksource/bigbench/discourse_marker_prediction": 4, "tasksource/bigbench/strategyqa": 10, "tasksource/bigbench/parsinlu_qa": 9, "tasksource/bigbench/logical_deduction": 9, "tasksource/bigbench/elementary_math_qa": 159, "tasksource/bigbench/tracking_shuffled_objects": 20, "tasksource/bigbench/snarks": 1, "tasksource/bigbench/color": 18, "tasksource/bigbench/play_dialog_same_or_different": 13, "tasksource/bigbench/indic_cause_and_effect": 2, "tasksource/bigbench/logic_grid_puzzle": 10, "tasksource/bigbench/sports_understanding": 4, "tasksource/bigbench/fantasy_reasoning": 2, "tasksource/bigbench/question_selection": 7, "tasksource/bigbench/code_line_description": 1, "tasksource/bigbench/kanji_ascii": 2, "tasksource/bigbench/analogical_similarity": 2, "tasksource/bigbench/winowhy": 11, "tasksource/bigbench/salient_translation_error_detection": 3, "tasksource/bigbench/rhyming": 2, "tasksource/bigbench/movie_recommendation": 3, "tasksource/bigbench/analytic_entailment": 2, "tasksource/bigbench/logical_fallacy_detection": 7, "tasksource/bigbench/temporal_sequences": 3, "tasksource/bigbench/kannada": 3, "tasksource/bigbench/navigate": 2, "tasksource/bigbench/social_iqa": 6, "tasksource/bigbench/implicatures": 2, "tasksource/bigbench/penguins_in_a_table": 2, "tasksource/bigbench/epistemic_reasoning": 8, "tasksource/bigbench/reasoning_about_colored_objects": 5, "tasksource/bigbench/strange_stories": 1, "tasksource/bigbench/what_is_the_tao": 1, "tasksource/bigbench/international_phonetic_alphabet_nli": 2, "tasksource/bigbench/hindu_knowledge": 1, "tasksource/bigbench/cryobiology_spanish": 1, "tasksource/bigbench/intent_recognition": 4, "tasksource/bigbench/metaphor_boolean": 2, "tasksource/bigbench/suicide_risk": 1, "tasksource/bigbench/logical_args": 1, "tasksource/bigbench/ruin_names": 1, "tasksource/bigbench/causal_judgment": 1, "tasksource/bigbench/english_proverbs": 1, "tasksource/bigbench/empirical_judgments": 1, "tasksource/bigbench/physical_intuition": 2, "tasksource/bigbench/similarities_abstraction": 1, "tasksource/bigbench/swahili_english_proverbs": 1, "tasksource/bigbench/date_understanding": 1, "tasksource/bigbench/medical_questions_russian": 1, "tasksource/bigbench/anachronisms": 1, "tasksource/bigbench/simple_ethical_questions": 2, "tasksource/bigbench/dark_humor_detection": 1, "tasksource/bigbench/irony_identification": 2, "tasksource/bigbench/geometric_shapes": 2, "tasksource/bigbench/cause_and_effect": 1, "tasksource/bigbench/moral_permissibility": 1, "tasksource/bigbench/phrase_relatedness": 1, "tasksource/bigbench/crass_ai": 1, "tasksource/bigbench/figure_of_speech_detection": 1 }, "Rowan/hellaswag:test": { "Rowan/hellaswag/default": 2000 }, "google/boolq:test": { "google/boolq/default": 828 }, "tasksource/zero-shot-label-nli:calibration": { "dynasent/dynabench.dynasent.r1.all/r1": 9, "crowdflower/text_emotion": 16, "pragmeval/gum": 10, "probability_words_nli/usnli": 25, "clcd-english": 9, "ethics/justice": 14, "ade_corpus_v2/Ade_corpus_v2_classification": 10, "rotten_tomatoes": 12, "spartqa-yn": 21, "counterfactually-augmented-imdb": 18, "dynasent/dynabench.dynasent.r2.all/r2": 13, "pragmeval/sarcasm": 18, "strategy-qa": 11, "ethos/binary": 20, "sms_spam": 17, "avicenna": 17, "ag_news": 18, "tweet_eval/emotion": 24, "silicone/dyda_da": 13, "pragmeval/verifiability": 12, "implicit-hate-stg1": 11, "silicone/oasis": 14, "silicone/sem": 12, "pragmeval/emergent": 20, "mc_taco": 19, "lexical_relation_classification/BLESS": 19, "hate_speech_offensive": 19, "crowdflower/economic-news": 14, "mbib-base/racial-bias": 10, "crowdflower/sentiment_nuclear_power": 3, "glue/sst2": 13, "scicite": 15, "silicone/iemocap": 8, "crowdflower/political-media-audience": 17, "banking77": 9, "health_fact": 12, "glue/qqp": 8, "tweets_hate_speech_detection": 12, "ethics/deontology": 18, "mbib-base/linguistic-bias": 18, "ethics/virtue": 17, "logical-fallacy": 12, "CREAK": 11, "tweet_eval/stance_atheism": 3, "pragmeval/stac": 19, "silicone/meld_s": 19, "parade": 14, "crowdflower/airline-sentiment": 21, "crowdflower/corporate-messaging": 14, "paws/labeled_final": 10, "natural-language-satisfiability": 14, "paws/labeled_swap": 16, "silicone/meld_e": 17, "MOH": 12, "mbib-base/text-level-bias": 19, "hope_edi/english": 18, "crowdflower/political-media-bias": 16, "I2D2": 20, "lexical_relation_classification/EVALution": 15, "mbib-base/gender-bias": 18, "sem_eval_2010_task_8": 16, "lex_glue/ledgar": 12, "lexical_relation_classification/ROOT09": 12, "pragmeval/mrda": 15, "mbib-base/hate-speech": 12, "trec": 17, "hate_speech18": 18, "lexical_relation_classification/K&H+N": 11, "EffectiveFeedbackStudentWriting": 19, "winowhy": 17, "liar": 16, "silicone/dyda_e": 13, "glue/mrpc": 10, "hyperpartisan_news": 6, "circa": 10, "lexical_relation_classification/CogALexV": 17, "commonsense_qa_2.0": 14, "imdb": 13, "scientific-exaggeration-detection": 8, "tweet_eval/irony": 15, "tweet_eval/stance_feminist": 9, "cladder": 14, "subjectivity": 13, "rumoureval_2019/RumourEval2019": 12, "v1/gen_train234_test2to10": 20, "blog_authorship_corpus/horoscope": 9, "probability_words_nli/reasoning_2hop": 17, "CONDAQA": 15, "hlgd": 14, "emo/emo2019": 16, "open_question_type": 14, "tweet_eval/offensive": 16, "amazon_polarity/amazon_polarity": 14, "tweet_eval/hate": 9, "phrase_similarity": 21, "VUAC": 16, "sciie": 12, "poem_sentiment": 18, "fever-evidence-related/mwong--fever-related": 15, "crowdflower/tweet_global_warming": 10, "vitaminc/tals--vitaminc": 17, "mbib-base/cognitive-bias": 14, "humicroedit/subtask-2": 16, "FLUTE": 12, "folio": 10, "super_glue/multirc": 4, "ambient": 10, "Touche23-ValueEval": 15, "PARARULE-Plus": 12, "glue/cola": 10, "yahoo_answers_topics": 13, "blog_authorship_corpus/gender": 13, "probability_words_nli/reasoning_1hop": 15, "financial_phrasebank/sentences_allagree": 11, "quarel": 9, "numer_sense": 14, "tweet_eval/stance_climate": 4, "citation_intent": 13, "dbpedia_14/dbpedia_14": 10, "ethics/commonsense": 15, "silicone/maptask": 10, "medical_questions_pairs": 14, "pragmeval/switchboard": 15, "wiki_qa": 20, "mbib-base/fake-news": 16, "cycic_classification": 16, "super_glue/wic": 12, "tweet_eval/emoji": 15, "scruples": 15, "dynahate": 13, "blog_authorship_corpus/job": 12, "pragmeval/pdtb": 10, "crowdflower/political-media-message": 16, "tweet_eval/stance_abortion": 12, "mbib-base/political-bias": 11, "starcon": 12, "tweet_eval/stance_hillary": 12, "discovery/discovery": 7, "TuringBench": 6, "language-identification": 11, "twentyquestions": 16, "insincere-questions": 5, "args_me": 7, "snips_built_in_intents": 1 }, "tasksource/tasksource-instruct-v0:calibration": { "yelp_review_full/yelp_review_full": 234, "tweet_eval/sentiment": 269 }, "nyu-mll/multi_nli:calibration": { "nyu-mll/multi_nli/default": 2000 }, "facebook/anli:calibration": { "facebook/anli/plain_text": 1271 }, "tasksource/defeasible-nli:calibration": { "tasksource/defeasible-nli/snli": 259, "tasksource/defeasible-nli/social": 1211, "tasksource/defeasible-nli/atomic": 519 }, "tasksource/FOL-nli:calibration": { "tasksource/FOL-nli/default": 1999 }, "tasksource/doc-nli:calibration": { "tasksource/doc-nli/default": 1800 }, "tasksource/bigbench:calibration": { "tasksource/bigbench/mnist_ascii": 322, "tasksource/bigbench/unit_conversion": 67, "tasksource/bigbench/contextual_parametric_knowledge_conflicts": 69, "tasksource/bigbench/formal_fallacies_syllogisms_negation": 63, "tasksource/bigbench/elementary_math_qa": 173, "tasksource/bigbench/movie_dialog_same_or_different": 227, "tasksource/bigbench/vitaminc_fact_verification": 234, "tasksource/bigbench/tracking_shuffled_objects": 21, "tasksource/bigbench/color": 18, "tasksource/bigbench/abstract_narrative_understanding": 11, "tasksource/bigbench/logical_fallacy_detection": 10, "tasksource/bigbench/language_identification": 35, "tasksource/bigbench/logic_grid_puzzle": 4, "tasksource/bigbench/play_dialog_same_or_different": 17, "tasksource/bigbench/hyperbaton": 225, "tasksource/bigbench/real_or_fake_text": 50, "tasksource/bigbench/bbq_lite_json": 66, "tasksource/bigbench/navigate": 8, "tasksource/bigbench/fact_checker": 31, "tasksource/bigbench/social_iqa": 7, "tasksource/bigbench/strategyqa": 20, "tasksource/bigbench/analytic_entailment": 1, "tasksource/bigbench/implicatures": 5, "tasksource/bigbench/goal_step_wikihow": 20, "tasksource/bigbench/crass_ai": 1, "tasksource/bigbench/timedial": 10, "tasksource/bigbench/arithmetic": 79, "tasksource/bigbench/moral_permissibility": 4, "tasksource/bigbench/metaphor_boolean": 3, "tasksource/bigbench/winowhy": 18, "tasksource/bigbench/temporal_sequences": 3, "tasksource/bigbench/sports_understanding": 2, "tasksource/bigbench/presuppositions_as_nli": 6, "tasksource/bigbench/logical_deduction": 6, "tasksource/bigbench/strange_stories": 1, "tasksource/bigbench/hinglish_toxicity": 1, "tasksource/bigbench/date_understanding": 5, "tasksource/bigbench/question_selection": 6, "tasksource/bigbench/what_is_the_tao": 1, "tasksource/bigbench/symbol_interpretation": 3, "tasksource/bigbench/irony_identification": 1, "tasksource/bigbench/movie_recommendation": 3, "tasksource/bigbench/epistemic_reasoning": 7, "tasksource/bigbench/cryobiology_spanish": 2, "tasksource/bigbench/evaluating_information_essentiality": 1, "tasksource/bigbench/cs_algorithms": 7, "tasksource/bigbench/penguins_in_a_table": 1, "tasksource/bigbench/medical_questions_russian": 3, "tasksource/bigbench/rhyming": 5, "tasksource/bigbench/odd_one_out": 1, "tasksource/bigbench/kannada": 1, "tasksource/bigbench/discourse_marker_prediction": 4, "tasksource/bigbench/parsinlu_qa": 5, "tasksource/bigbench/entailed_polarity_hindi": 1, "tasksource/bigbench/disambiguation_qa": 2, "tasksource/bigbench/suicide_risk": 1, "tasksource/bigbench/fantasy_reasoning": 1, "tasksource/bigbench/social_support": 2, "tasksource/bigbench/misconceptions": 2, "tasksource/bigbench/physical_intuition": 1, "tasksource/bigbench/similarities_abstraction": 1, "tasksource/bigbench/swahili_english_proverbs": 1, "tasksource/bigbench/hhh_alignment": 1, "tasksource/bigbench/salient_translation_error_detection": 4, "tasksource/bigbench/dark_humor_detection": 1, "tasksource/bigbench/geometric_shapes": 1, "tasksource/bigbench/which_wiki_edit": 1, "tasksource/bigbench/ruin_names": 1, "tasksource/bigbench/reasoning_about_colored_objects": 1, "tasksource/bigbench/understanding_fables": 1 }, "Rowan/hellaswag:calibration": { "Rowan/hellaswag/default": 1997 }, "google/boolq:calibration": { "google/boolq/default": 832 }, "tasksource/zero-shot-label-nli:validation": { "banking77": 21, "health_fact": 19, "ethics/deontology": 8, "TuringBench": 25, "crowdflower/political-media-bias": 11, "silicone/oasis": 11, "quarel": 20, "pragmeval/pdtb": 20, "poem_sentiment": 18, "numer_sense": 18, "mbib-base/hate-speech": 16, "paws/labeled_swap": 9, "probability_words_nli/usnli": 10, "tweet_eval/stance_climate": 9, "subjectivity": 13, "snips_built_in_intents": 8, "tweet_eval/stance_abortion": 9, "MOH": 16, "pragmeval/mrda": 15, "scruples": 17, "insincere-questions": 16, "avicenna": 11, "tweet_eval/emotion": 14, "yahoo_answers_topics": 19, "phrase_similarity": 15, "silicone/sem": 12, "mbib-base/linguistic-bias": 12, "hate_speech18": 13, "commonsense_qa_2.0": 18, "dynahate": 13, "lexical_relation_classification/K&H+N": 13, "ade_corpus_v2/Ade_corpus_v2_classification": 15, "ag_news": 9, "counterfactually-augmented-imdb": 19, "pragmeval/gum": 17, "lexical_relation_classification/ROOT09": 16, "tweet_eval/irony": 15, "tweet_eval/emoji": 15, "dbpedia_14/dbpedia_14": 19, "trec": 11, "VUAC": 15, "pragmeval/verifiability": 14, "dynasent/dynabench.dynasent.r2.all/r2": 17, "crowdflower/political-media-audience": 12, "humicroedit/subtask-2": 13, "CREAK": 10, "twentyquestions": 11, "Touche23-ValueEval": 14, "sciie": 17, "vitaminc/tals--vitaminc": 19, "tweet_eval/stance_feminist": 9, "liar": 14, "mbib-base/text-level-bias": 17, "probability_words_nli/reasoning_2hop": 14, "mbib-base/fake-news": 15, "crowdflower/economic-news": 15, "pragmeval/switchboard": 12, "silicone/dyda_da": 19, "lexical_relation_classification/EVALution": 18, "imdb": 19, "wiki_qa": 13, "cycic_classification": 14, "silicone/maptask": 9, "ethos/binary": 7, "scicite": 15, "spartqa-yn": 9, "CONDAQA": 11, "crowdflower/tweet_global_warming": 15, "hlgd": 14, "crowdflower/text_emotion": 13, "circa": 15, "ethics/commonsense": 16, "mbib-base/racial-bias": 16, "PARARULE-Plus": 13, "folio": 12, "lexical_relation_classification/CogALexV": 20, "mc_taco": 12, "parade": 15, "EffectiveFeedbackStudentWriting": 10, "silicone/iemocap": 16, "rumoureval_2019/RumourEval2019": 15, "starcon": 18, "ethics/virtue": 13, "cladder": 9, "fever-evidence-related/mwong--fever-related": 14, "v1/gen_train234_test2to10": 15, "I2D2": 17, "discovery/discovery": 11, "natural-language-satisfiability": 15, "emo/emo2019": 11, "implicit-hate-stg1": 19, "blog_authorship_corpus/horoscope": 10, "crowdflower/corporate-messaging": 14, "pragmeval/stac": 21, "glue/sst2": 4, "glue/mrpc": 19, "language-identification": 15, "citation_intent": 15, "silicone/meld_s": 11, "silicone/meld_e": 14, "super_glue/wic": 8, "crowdflower/political-media-message": 14, "medical_questions_pairs": 20, "blog_authorship_corpus/gender": 18, "clcd-english": 16, "dynasent/dynabench.dynasent.r1.all/r1": 13, "glue/qqp": 6, "winowhy": 16, "tweet_eval/hate": 18, "paws/labeled_final": 17, "mbib-base/cognitive-bias": 11, "args_me": 15, "lexical_relation_classification/BLESS": 14, "silicone/dyda_e": 19, "open_question_type": 13, "mbib-base/gender-bias": 18, "sms_spam": 16, "pragmeval/emergent": 10, "lex_glue/ledgar": 11, "hyperpartisan_news": 9, "probability_words_nli/reasoning_1hop": 12, "tweet_eval/stance_atheism": 9, "tweets_hate_speech_detection": 9, "blog_authorship_corpus/job": 13, "rotten_tomatoes": 16, "tweet_eval/stance_hillary": 6, "strategy-qa": 12, "financial_phrasebank/sentences_allagree": 17, "hate_speech_offensive": 21, "mbib-base/political-bias": 11, "glue/cola": 4, "hope_edi/english": 13, "ethics/justice": 17, "sem_eval_2010_task_8": 8, "logical-fallacy": 12, "scientific-exaggeration-detection": 15, "tweet_eval/offensive": 18, "crowdflower/airline-sentiment": 11, "FLUTE": 13, "super_glue/multirc": 6, "amazon_polarity/amazon_polarity": 9, "crowdflower/sentiment_nuclear_power": 1, "ambient": 4, "pragmeval/sarcasm": 7 }, "tasksource/tasksource-instruct-v0:validation": { "yelp_review_full/yelp_review_full": 264, "tweet_eval/sentiment": 231 }, "nyu-mll/multi_nli:validation": { "nyu-mll/multi_nli/default": 2000 }, "facebook/anli:validation": { "facebook/anli/plain_text": 1218 }, "tasksource/defeasible-nli:validation": { "tasksource/defeasible-nli/snli": 240, "tasksource/defeasible-nli/social": 1251, "tasksource/defeasible-nli/atomic": 500 }, "tasksource/FOL-nli:validation": { "tasksource/FOL-nli/default": 2000 }, "tasksource/doc-nli:validation": { "tasksource/doc-nli/default": 1786 }, "tasksource/bigbench:validation": { "tasksource/bigbench/elementary_math_qa": 180, "tasksource/bigbench/formal_fallacies_syllogisms_negation": 57, "tasksource/bigbench/analogical_similarity": 2, "tasksource/bigbench/mnist_ascii": 302, "tasksource/bigbench/movie_dialog_same_or_different": 217, "tasksource/bigbench/logical_fallacy_detection": 9, "tasksource/bigbench/bbq_lite_json": 54, "tasksource/bigbench/arithmetic": 68, "tasksource/bigbench/hyperbaton": 227, "tasksource/bigbench/color": 17, "tasksource/bigbench/unit_conversion": 50, "tasksource/bigbench/contextual_parametric_knowledge_conflicts": 90, "tasksource/bigbench/vitaminc_fact_verification": 240, "tasksource/bigbench/physical_intuition": 2, "tasksource/bigbench/real_or_fake_text": 54, "tasksource/bigbench/play_dialog_same_or_different": 19, "tasksource/bigbench/language_identification": 47, "tasksource/bigbench/navigate": 6, "tasksource/bigbench/timedial": 14, "tasksource/bigbench/goal_step_wikihow": 41, "tasksource/bigbench/winowhy": 11, "tasksource/bigbench/temporal_sequences": 4, "tasksource/bigbench/logic_grid_puzzle": 10, "tasksource/bigbench/discourse_marker_prediction": 4, "tasksource/bigbench/fact_checker": 25, "tasksource/bigbench/social_support": 2, "tasksource/bigbench/causal_judgment": 1, "tasksource/bigbench/social_iqa": 13, "tasksource/bigbench/cs_algorithms": 5, "tasksource/bigbench/fantasy_reasoning": 1, "tasksource/bigbench/movie_recommendation": 2, "tasksource/bigbench/penguins_in_a_table": 1, "tasksource/bigbench/intent_recognition": 3, "tasksource/bigbench/logical_deduction": 3, "tasksource/bigbench/strategyqa": 10, "tasksource/bigbench/ruin_names": 2, "tasksource/bigbench/sports_understanding": 4, "tasksource/bigbench/epistemic_reasoning": 7, "tasksource/bigbench/salient_translation_error_detection": 3, "tasksource/bigbench/irony_identification": 1, "tasksource/bigbench/abstract_narrative_understanding": 15, "tasksource/bigbench/reasoning_about_colored_objects": 4, "tasksource/bigbench/tracking_shuffled_objects": 14, "tasksource/bigbench/metaphor_boolean": 2, "tasksource/bigbench/cryobiology_spanish": 2, "tasksource/bigbench/human_organs_senses": 1, "tasksource/bigbench/question_selection": 4, "tasksource/bigbench/mathematical_induction": 1, "tasksource/bigbench/key_value_maps": 1, "tasksource/bigbench/misconceptions": 2, "tasksource/bigbench/gre_reading_comprehension": 1, "tasksource/bigbench/conceptual_combinations": 1, "tasksource/bigbench/understanding_fables": 1, "tasksource/bigbench/symbol_interpretation": 1, "tasksource/bigbench/indic_cause_and_effect": 1, "tasksource/bigbench/evaluating_information_essentiality": 1, "tasksource/bigbench/implicatures": 2, "tasksource/bigbench/swahili_english_proverbs": 2, "tasksource/bigbench/snarks": 1, "tasksource/bigbench/metaphor_understanding": 2, "tasksource/bigbench/figure_of_speech_detection": 1, "tasksource/bigbench/strange_stories": 1, "tasksource/bigbench/disambiguation_qa": 2, "tasksource/bigbench/kannada": 1, "tasksource/bigbench/rhyming": 1, "tasksource/bigbench/logical_args": 1, "tasksource/bigbench/unit_interpretation": 1, "tasksource/bigbench/parsinlu_qa": 1, "tasksource/bigbench/presuppositions_as_nli": 2, "tasksource/bigbench/emoji_movie": 1, "tasksource/bigbench/riddle_sense": 1 }, "Rowan/hellaswag:validation": { "Rowan/hellaswag/default": 1998 }, "google/boolq:validation": { "google/boolq/default": 1609 }, "tasksource/zero-shot-label-nli:train": { "mbib-base/fake-news": 347, "mbib-base/political-bias": 492, "ethics/deontology": 424, "quarel": 94, "pragmeval/sarcasm": 174, "Touche23-ValueEval": 267, "medical_questions_pairs": 125, "blog_authorship_corpus/job": 501, "hate_speech_offensive": 509, "clcd-english": 486, "CREAK": 503, "lexical_relation_classification/BLESS": 503, "tweet_eval/emoji": 501, "scruples": 483, "VUAC": 508, "ethics/virtue": 508, "blog_authorship_corpus/gender": 471, "tweet_eval/offensive": 500, "hate_speech18": 474, "ethics/justice": 473, "mbib-base/linguistic-bias": 532, "hlgd": 500, "CONDAQA": 294, "lexical_relation_classification/EVALution": 244, "ag_news": 518, "blog_authorship_corpus/horoscope": 480, "imdb": 438, "fever-evidence-related/mwong--fever-related": 471, "paws/labeled_swap": 508, "banking77": 460, "insincere-questions": 462, "EffectiveFeedbackStudentWriting": 504, "silicone/meld_s": 438, "dbpedia_14/dbpedia_14": 503, "pragmeval/stac": 523, "crowdflower/text_emotion": 500, "vitaminc/tals--vitaminc": 536, "glue/qqp": 494, "scicite": 433, "lexical_relation_classification/K&H+N": 471, "mbib-base/text-level-bias": 366, "mbib-base/racial-bias": 424, "cladder": 374, "lex_glue/ledgar": 490, "ade_corpus_v2/Ade_corpus_v2_classification": 456, "trec": 281, "twentyquestions": 508, "args_me": 480, "silicone/dyda_da": 460, "probability_words_nli/reasoning_1hop": 209, "I2D2": 472, "implicit-hate-stg1": 518, "humicroedit/subtask-2": 463, "tweets_hate_speech_detection": 452, "hope_edi/english": 504, "pragmeval/mrda": 496, "dynasent/dynabench.dynasent.r2.all/r2": 477, "wiki_qa": 504, "FLUTE": 303, "TuringBench": 464, "silicone/oasis": 351, "rotten_tomatoes": 356, "circa": 458, "glue/cola": 427, "mbib-base/gender-bias": 483, "discovery/discovery": 511, "crowdflower/airline-sentiment": 485, "glue/sst2": 505, "tweet_eval/emotion": 166, "crowdflower/economic-news": 316, "emo/emo2019": 456, "lexical_relation_classification/ROOT09": 480, "open_question_type": 197, "yahoo_answers_topics": 486, "commonsense_qa_2.0": 461, "dynasent/dynabench.dynasent.r1.all/r1": 516, "silicone/meld_e": 434, "financial_phrasebank/sentences_allagree": 102, "paws/labeled_final": 512, "liar": 509, "tweet_eval/hate": 445, "silicone/maptask": 309, "sciie": 179, "ethics/commonsense": 479, "crowdflower/political-media-message": 225, "pragmeval/pdtb": 496, "dynahate": 489, "pragmeval/verifiability": 292, "parade": 369, "spartqa-yn": 512, "super_glue/multirc": 503, "phrase_similarity": 343, "sem_eval_2010_task_8": 384, "pragmeval/switchboard": 462, "sms_spam": 209, "super_glue/wic": 272, "v1/gen_train234_test2to10": 530, "natural-language-satisfiability": 373, "silicone/dyda_e": 481, "language-identification": 499, "pragmeval/gum": 88, "amazon_polarity/amazon_polarity": 490, "glue/mrpc": 179, "probability_words_nli/usnli": 505, "crowdflower/tweet_global_warming": 156, "crowdflower/political-media-bias": 209, "mbib-base/cognitive-bias": 305, "mbib-base/hate-speech": 510, "health_fact": 474, "PARARULE-Plus": 491, "winowhy": 107, "counterfactually-augmented-imdb": 86, "citation_intent": 80, "silicone/sem": 188, "silicone/iemocap": 361, "probability_words_nli/reasoning_2hop": 192, "cycic_classification": 230, "MOH": 60, "ethos/binary": 48, "tweet_eval/irony": 141, "avicenna": 231, "numer_sense": 436, "crowdflower/political-media-audience": 208, "pragmeval/emergent": 101, "lexical_relation_classification/CogALexV": 148, "subjectivity": 45, "starcon": 76, "rumoureval_2019/RumourEval2019": 215, "folio": 49, "mc_taco": 183, "tweet_eval/stance_atheism": 17, "logical-fallacy": 140, "poem_sentiment": 41, "strategy-qa": 96, "crowdflower/corporate-messaging": 121, "tweet_eval/stance_climate": 17, "tweet_eval/stance_abortion": 40, "lex_glue/scotus": 28, "tweet_eval/stance_hillary": 25, "ambient": 3, "snips_built_in_intents": 15, "hyperpartisan_news": 12, "tweet_eval/stance_feminist": 29, "crowdflower/sentiment_nuclear_power": 5, "scientific-exaggeration-detection": 1 }, "tasksource/tasksource-instruct-v0:train": { "tweet_eval/sentiment": 5000, "yelp_review_full/yelp_review_full": 5000 }, "nyu-mll/multi_nli:train": { "nyu-mll/multi_nli/default": 49999 }, "facebook/anli:train": { "facebook/anli/plain_text": 49998 }, "tasksource/defeasible-nli:train": { "tasksource/defeasible-nli/atomic": 8695, "tasksource/defeasible-nli/social": 19097, "tasksource/defeasible-nli/snli": 22202 }, "tasksource/FOL-nli:train": { "tasksource/FOL-nli/default": 50000 }, "tasksource/doc-nli:train": { "tasksource/doc-nli/default": 43368 }, "tasksource/bigbench:train": { "tasksource/bigbench/vitaminc_fact_verification": 5000, "tasksource/bigbench/hyperbaton": 5000, "tasksource/bigbench/mnist_ascii": 5000, "tasksource/bigbench/unit_conversion": 1657, "tasksource/bigbench/goal_step_wikihow": 814, "tasksource/bigbench/bbq_lite_json": 1782, "tasksource/bigbench/movie_dialog_same_or_different": 5000, "tasksource/bigbench/contextual_parametric_knowledge_conflicts": 1945, "tasksource/bigbench/tracking_shuffled_objects": 442, "tasksource/bigbench/abstract_narrative_understanding": 170, "tasksource/bigbench/elementary_math_qa": 4362, "tasksource/bigbench/question_selection": 161, "tasksource/bigbench/timedial": 262, "tasksource/bigbench/formal_fallacies_syllogisms_negation": 1593, "tasksource/bigbench/logical_fallacy_detection": 302, "tasksource/bigbench/arithmetic": 1648, "tasksource/bigbench/language_identification": 1106, "tasksource/bigbench/discourse_marker_prediction": 86, "tasksource/bigbench/real_or_fake_text": 1274, "tasksource/bigbench/temporal_sequences": 124, "tasksource/bigbench/geometric_shapes": 35, "tasksource/bigbench/intent_recognition": 72, "tasksource/bigbench/play_dialog_same_or_different": 387, "tasksource/bigbench/parsinlu_qa": 118, "tasksource/bigbench/fact_checker": 777, "tasksource/bigbench/implicatures": 55, "tasksource/bigbench/cause_and_effect": 13, "tasksource/bigbench/cs_algorithms": 146, "tasksource/bigbench/social_iqa": 229, "tasksource/bigbench/winowhy": 343, "tasksource/bigbench/color": 441, "tasksource/bigbench/snarks": 18, "tasksource/bigbench/epistemic_reasoning": 233, "tasksource/bigbench/strategyqa": 258, "tasksource/bigbench/reasoning_about_colored_objects": 69, "tasksource/bigbench/metaphor_boolean": 78, "tasksource/bigbench/movie_recommendation": 63, "tasksource/bigbench/ruin_names": 46, "tasksource/bigbench/social_support": 105, "tasksource/bigbench/penguins_in_a_table": 21, "tasksource/bigbench/navigate": 110, "tasksource/bigbench/undo_permutation": 25, "tasksource/bigbench/odd_one_out": 9, "tasksource/bigbench/cifar10_classification": 2, "tasksource/bigbench/anachronisms": 31, "tasksource/bigbench/date_understanding": 36, "tasksource/bigbench/causal_judgment": 24, "tasksource/bigbench/logic_grid_puzzle": 113, "tasksource/bigbench/sports_understanding": 107, "tasksource/bigbench/rhyming": 77, "tasksource/bigbench/which_wiki_edit": 20, "tasksource/bigbench/medical_questions_russian": 23, "tasksource/bigbench/key_value_maps": 14, "tasksource/bigbench/symbol_interpretation": 109, "tasksource/bigbench/salient_translation_error_detection": 104, "tasksource/bigbench/suicide_risk": 4, "tasksource/bigbench/analogical_similarity": 35, "tasksource/bigbench/english_russian_proverbs": 7, "tasksource/bigbench/sentence_ambiguity": 8, "tasksource/bigbench/emoji_movie": 13, "tasksource/bigbench/kannada": 47, "tasksource/bigbench/cryobiology_spanish": 14, "tasksource/bigbench/hhh_alignment": 19, "tasksource/bigbench/swahili_english_proverbs": 19, "tasksource/bigbench/phrase_relatedness": 10, "tasksource/bigbench/presuppositions_as_nli": 64, "tasksource/bigbench/kanji_ascii": 16, "tasksource/bigbench/metaphor_understanding": 22, "tasksource/bigbench/logical_deduction": 41, "tasksource/bigbench/unit_interpretation": 13, "tasksource/bigbench/dark_humor_detection": 4, "tasksource/bigbench/physics": 26, "tasksource/bigbench/hindu_knowledge": 14, "tasksource/bigbench/empirical_judgments": 13, "tasksource/bigbench/conceptual_combinations": 11, "tasksource/bigbench/understanding_fables": 23, "tasksource/bigbench/moral_permissibility": 33, "tasksource/bigbench/figure_of_speech_detection": 8, "tasksource/bigbench/indic_cause_and_effect": 38, "tasksource/bigbench/disambiguation_qa": 20, "tasksource/bigbench/evaluating_information_essentiality": 8, "tasksource/bigbench/entailed_polarity_hindi": 12, "tasksource/bigbench/irony_identification": 11, "tasksource/bigbench/hinglish_toxicity": 20, "tasksource/bigbench/analytic_entailment": 3, "tasksource/bigbench/strange_stories": 19, "tasksource/bigbench/common_morpheme": 6, "tasksource/bigbench/human_organs_senses": 5, "tasksource/bigbench/identify_odd_metaphor": 8, "tasksource/bigbench/entailed_polarity": 19, "tasksource/bigbench/persian_idioms": 12, "tasksource/bigbench/misconceptions": 31, "tasksource/bigbench/what_is_the_tao": 2, "tasksource/bigbench/checkmate_in_one": 5, "tasksource/bigbench/logical_sequence": 6, "tasksource/bigbench/fantasy_reasoning": 18, "tasksource/bigbench/english_proverbs": 1, "tasksource/bigbench/simple_ethical_questions": 11, "tasksource/bigbench/physical_intuition": 5, "tasksource/bigbench/swedish_to_german_proverbs": 7, "tasksource/bigbench/crass_ai": 3, "tasksource/bigbench/international_phonetic_alphabet_nli": 6, "tasksource/bigbench/similarities_abstraction": 2, "tasksource/bigbench/nonsense_words_grammar": 5, "tasksource/bigbench/code_line_description": 6, "tasksource/bigbench/mathematical_induction": 6, "tasksource/bigbench/general_knowledge": 8, "tasksource/bigbench/riddle_sense": 7, "tasksource/bigbench/emojis_emotion_prediction": 2, "tasksource/bigbench/logical_args": 3, "tasksource/bigbench/gre_reading_comprehension": 1, "tasksource/bigbench/known_unknowns": 7, "tasksource/bigbench/identify_math_theorems": 3, "tasksource/bigbench/crash_blossom": 4, "tasksource/bigbench/novel_concepts": 1, "tasksource/bigbench/minute_mysteries_qa": 1 }, "Rowan/hellaswag:train": { "Rowan/hellaswag/default": 39776 }, "google/boolq:train": { "google/boolq/default": 7982 } }, "prepared_dataset_hub": { "repo_id": "skundu42/kev-prepared", "revision": "5cea4539b6676edfb4fd1fb60878e9aee602b0cd" } }