{
    "compilation_info": {
        "project": "CreateLLM_JP",
        "description": "Training dataset compiled from various sources including CC-BY 4.0 creative/encyclopedic datasets.",
        "sources": [
            {
                "name": "Aozora Bunko (via Globis University)",
                "domain": "Creative Writing / Novels",
                "url": "https://huggingface.co/datasets/globis-university/aozorabunko-clean",
                "original_url": "https://www.aozora.gr.jp/",
                "license": "Public Domain (Text)",
                "file_key": "aozora"
            },
            {
                "name": "Wikipedia (Japanese)",
                "domain": "Encyclopedic / General Knowledge",
                "url": "https://huggingface.co/datasets/wikimedia/wikipedia",
                "original_url": "https://ja.wikipedia.org/",
                "license": "CC-BY-SA 3.0 / CC-BY-SA 4.0",
                "file_key": "wiki"
            },
            {
                "name": "Nemotron-Personas-Japan",
                "domain": "Persona / Dialog Generation",
                "url": "https://huggingface.co/datasets/nvidia/Nemotron-Personas-Japan",
                "original_url": "https://huggingface.co/datasets/nvidia/Nemotron-Personas-Japan",
                "license": "CC-BY 4.0",
                "file_key": "nemotron"
            },
            {
                "name": "xwinograd-ja",
                "domain": "Reasoning / Coreference Resolution",
                "url": "https://huggingface.co/datasets/polm-stability/xwinograd-ja",
                "original_url": "https://github.com/aida-muc/xwinograd",
                "license": "CC-BY 4.0",
                "file_key": "xwin"
            },
            {
                "name": "Law QA (Digital Agency)",
                "domain": "Legal / Reasoning",
                "url": "https://github.com/digital-go-jp/lawqa_jp",
                "original_url": "https://github.com/digital-go-jp/lawqa_jp",
                "license": "Public Data License (CC-BY 4.0 compatible)",
                "file_key": "law"
            },
            {
                "name": "Tanaka Corpus (ja-en)",
                "domain": "Translation / General",
                "url": "https://huggingface.co/datasets/hpprc/tanaka-corpus",
                "original_url": "http://www.edrdg.org/wiki/index.php/Tanaka_Corpus",
                "license": "Public Domain",
                "file_key": "snow"
            },
            {
                "name": "Weather Forecast Japan (JMA)",
                "domain": "Weather / Description",
                "url": "https://huggingface.co/datasets/jniimi/weather_forecast_japan",
                "original_url": "https://www.jma.go.jp/",
                "license": "CC-BY 4.0",
                "file_key": "weather"
            },
            {
                "name": "Open-Platypus-Japanese-masked",
                "domain": "Reasoning / Math",
                "url": "https://huggingface.co/datasets/weblab-GENIAC/Open-Platypus-Japanese-masked",
                "original_url": "https://huggingface.co/datasets/weblab-GENIAC/Open-Platypus-Japanese-masked",
                "license": "MIT / CC-BY-4.0",
                "file_key": "platypus"
            },
            {
                "name": "Japanese Honorifics",
                "domain": "Linguistics / Keigo",
                "url": "https://huggingface.co/datasets/ronantakizawa/japanese-honorifics",
                "original_url": "https://huggingface.co/datasets/ronantakizawa/japanese-honorifics",
                "license": "CC-BY 4.0",
                "file_key": "honorifics"
            },
            {
                "name": "Japanese Trending Words",
                "domain": "Culture / Neologisms",
                "url": "https://huggingface.co/datasets/ronantakizawa/japanese-trending-words",
                "original_url": "https://huggingface.co/datasets/ronantakizawa/japanese-trending-words",
                "license": "CC-BY 4.0",
                "file_key": "trending"
            }
        ]
    },
    "statistics": {
        "aozora.jsonl": 1008756,
        "wiki.jsonl": 2073483,
        "nemotron.jsonl": 1000000,
        "xwinograd.jsonl": 959,
        "lawqa.jsonl": 140,
        "weather.jsonl": 15000,
        "honorifics.jsonl": 345,
        "trending.jsonl": 593,
        "snow.jsonl": 147876,
        "platypus.jsonl": 13881
    },
    "total_records": 4261033,
    "output_file": "train_v2.jsonl"
}