Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Hugging Face Dataset Upload Decision Guide","local":"hugging-face-dataset-upload-decision-guide","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Working Without File Access","local":"working-without-file-access","sections":[],"depth":2},{"title":"Critical Constraints","local":"critical-constraints","sections":[],"depth":2},{"title":"Quick Reference by Data Type","local":"quick-reference-by-data-type","sections":[],"depth":2},{"title":"Upload Workflow","local":"upload-workflow","sections":[],"depth":2},{"title":"Common Conversion Patterns","local":"common-conversion-patterns","sections":[{"title":"From DataFrames","local":"from-dataframes","sections":[],"depth":3}],"depth":2},{"title":"Custom Format Conversion","local":"custom-format-conversion","sections":[{"title":"Design Principles","local":"design-principles","sections":[],"depth":3},{"title":"Conversion Methods","local":"conversion-methods","sections":[],"depth":3},{"title":"Using Configs for Dataset Variations","local":"using-configs-for-dataset-variations","sections":[],"depth":3},{"title":"Multi-modal Examples","local":"multi-modal-examples","sections":[],"depth":3}],"depth":2},{"title":"Essential Features","local":"essential-features","sections":[{"title":"Feature Types Overview","local":"feature-types-overview","sections":[],"depth":3}],"depth":2},{"title":"Upload Methods","local":"upload-methods","sections":[],"depth":2},{"title":"Validation","local":"validation","sections":[],"depth":2},{"title":"Common Issues → Solutions","local":"common-issues--solutions","sections":[],"depth":2},{"title":"Dataset Viewer Configuration","local":"dataset-viewer-configuration","sections":[{"title":"When automatic detection works","local":"when-automatic-detection-works","sections":[],"depth":3},{"title":"Manual configuration","local":"manual-configuration","sections":[],"depth":3},{"title":"Common viewer issues","local":"common-viewer-issues","sections":[],"depth":3}],"depth":2},{"title":"Quick Templates","local":"quick-templates","sections":[],"depth":2},{"title":"Documentation","local":"documentation","sections":[],"depth":2},{"title":"Dataset Cards","local":"dataset-cards","sections":[],"depth":2},{"title":"Appendix: Special Cases","local":"appendix-special-cases","sections":[{"title":"WebDataset Structure","local":"webdataset-structure","sections":[],"depth":3},{"title":"Scientific Data","local":"scientific-data","sections":[],"depth":3},{"title":"Community Resources","local":"community-resources","sections":[],"depth":3}],"depth":2}],"depth":1}"> | |
| <link href="/docs/hub/pr_2549/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> | |
| <link rel="modulepreload" href="/docs/hub/pr_2549/en/_app/immutable/entry/start.e1a19059.js"> | |
| <link rel="modulepreload" href="/docs/hub/pr_2549/en/_app/immutable/chunks/scheduler.409792a1.js"> | |
| <link rel="modulepreload" href="/docs/hub/pr_2549/en/_app/immutable/chunks/singletons.1870f91e.js"> | |
| <link rel="modulepreload" href="/docs/hub/pr_2549/en/_app/immutable/chunks/index.0f0d9f26.js"> | |
| <link rel="modulepreload" href="/docs/hub/pr_2549/en/_app/immutable/chunks/paths.c8bba6e0.js"> | |
| <link rel="modulepreload" href="/docs/hub/pr_2549/en/_app/immutable/entry/app.d45f53b9.js"> | |
| <link rel="modulepreload" href="/docs/hub/pr_2549/en/_app/immutable/chunks/preload-helper.0bfc46c3.js"> | |
| <link rel="modulepreload" href="/docs/hub/pr_2549/en/_app/immutable/chunks/index.92d389ff.js"> | |
| <link rel="modulepreload" href="/docs/hub/pr_2549/en/_app/immutable/nodes/0.6cd7bceb.js"> | |
| <link rel="modulepreload" href="/docs/hub/pr_2549/en/_app/immutable/chunks/each.e59479a4.js"> | |
| <link rel="modulepreload" href="/docs/hub/pr_2549/en/_app/immutable/nodes/63.38736e9f.js"> | |
| <link rel="modulepreload" href="/docs/hub/pr_2549/en/_app/immutable/chunks/CopyLLMTxtMenu.a4554287.js"> | |
| <link rel="modulepreload" href="/docs/hub/pr_2549/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.e0ffed5d.js"> | |
| <link rel="modulepreload" href="/docs/hub/pr_2549/en/_app/immutable/chunks/CodeBlock.24a6533e.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Hugging Face Dataset Upload Decision Guide","local":"hugging-face-dataset-upload-decision-guide","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Working Without File Access","local":"working-without-file-access","sections":[],"depth":2},{"title":"Critical Constraints","local":"critical-constraints","sections":[],"depth":2},{"title":"Quick Reference by Data Type","local":"quick-reference-by-data-type","sections":[],"depth":2},{"title":"Upload Workflow","local":"upload-workflow","sections":[],"depth":2},{"title":"Common Conversion Patterns","local":"common-conversion-patterns","sections":[{"title":"From DataFrames","local":"from-dataframes","sections":[],"depth":3}],"depth":2},{"title":"Custom Format Conversion","local":"custom-format-conversion","sections":[{"title":"Design Principles","local":"design-principles","sections":[],"depth":3},{"title":"Conversion Methods","local":"conversion-methods","sections":[],"depth":3},{"title":"Using Configs for Dataset Variations","local":"using-configs-for-dataset-variations","sections":[],"depth":3},{"title":"Multi-modal Examples","local":"multi-modal-examples","sections":[],"depth":3}],"depth":2},{"title":"Essential Features","local":"essential-features","sections":[{"title":"Feature Types Overview","local":"feature-types-overview","sections":[],"depth":3}],"depth":2},{"title":"Upload Methods","local":"upload-methods","sections":[],"depth":2},{"title":"Validation","local":"validation","sections":[],"depth":2},{"title":"Common Issues → Solutions","local":"common-issues--solutions","sections":[],"depth":2},{"title":"Dataset Viewer Configuration","local":"dataset-viewer-configuration","sections":[{"title":"When automatic detection works","local":"when-automatic-detection-works","sections":[],"depth":3},{"title":"Manual configuration","local":"manual-configuration","sections":[],"depth":3},{"title":"Common viewer issues","local":"common-viewer-issues","sections":[],"depth":3}],"depth":2},{"title":"Quick Templates","local":"quick-templates","sections":[],"depth":2},{"title":"Documentation","local":"documentation","sections":[],"depth":2},{"title":"Dataset Cards","local":"dataset-cards","sections":[],"depth":2},{"title":"Appendix: Special Cases","local":"appendix-special-cases","sections":[{"title":"WebDataset Structure","local":"webdataset-structure","sections":[],"depth":3},{"title":"Scientific Data","local":"scientific-data","sections":[],"depth":3},{"title":"Community Resources","local":"community-resources","sections":[],"depth":3}],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="hugging-face-dataset-upload-decision-guide" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#hugging-face-dataset-upload-decision-guide"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Hugging Face Dataset Upload Decision Guide</span></h1> <blockquote class="tip" data-svelte-h="svelte-1ci1pys"><p>This guide is primarily designed for LLMs to help users upload datasets to the Hugging Face Hub in the most compatible format. Users can also reference this guide to understand the upload process and best practices.</p></blockquote> <blockquote data-svelte-h="svelte-1fajnvg"><p>Decision guide for uploading datasets to Hugging Face Hub. Optimized for Dataset Viewer compatibility and integration with the Hugging Face ecosystem.</p></blockquote> <h2 class="relative group"><a id="overview" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#overview"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Overview</span></h2> <p data-svelte-h="svelte-hun1os">Your goal is to help a user upload a dataset to the Hugging Face Hub. Ideally, the dataset should be compatible with the Dataset Viewer (and thus the <code>load_dataset</code> function) to ensure easy access and usability. You should aim to meet the following criteria:</p> <table data-svelte-h="svelte-1x2va0m"><thead><tr><th><strong>Criteria</strong></th> <th>Description</th> <th>Priority</th></tr></thead> <tbody><tr><td><strong>Respect repository limits</strong></td> <td>Ensure the dataset adheres to Hugging Face’s storage limits for file sizes, repository sizes, and file counts. See the Critical Constraints section below for specific limits.</td> <td>Required</td></tr> <tr><td><strong>Use hub-compatible formats</strong></td> <td>Use Parquet format when possible (best compression, rich typing, large dataset support). For smaller datasets (<several GB), JSON/JSONL or CSV are acceptable. Raw files work well for images/audio in smaller datasets while respecting repo limits. Use WebDataset (.tar) for large media collections. Domain-specific formats can be used when conversion is impractical.</td> <td>Desired</td></tr> <tr><td><strong>Dataset Viewer compatibility</strong></td> <td>Structure data to work with the automatic Dataset Viewer, enabling preview and easy exploration. This typically means using supported formats and proper file organization. Validation steps are provided later in this guide.</td> <td>Desired</td></tr> <tr><td><strong>Organize data sensibly</strong></td> <td>Use logical folder structures that match Hub conventions (e.g., train/test splits). Configs can be used to define different configurations of the dataset. This facilitates both human understanding and automatic data loading.</td> <td>Desired</td></tr> <tr><td><strong>Use appropriate Features</strong></td> <td>When using the datasets library, specify correct feature types (e.g., Image(), Audio(), ClassLabel()) to ensure proper data handling and viewer functionality. This enables type-specific optimizations and previews.</td> <td>Required (when using datasets library)</td></tr> <tr><td><strong>Document non-standard datasets</strong></td> <td>If conversion to hub-compatible formats is impossible and custom formats must be used, ensure repository limits are strictly followed and provide clear documentation on how to download and load the dataset. Include usage examples and any special requirements.</td> <td>Required (when datasets library isn’t compatible)</td></tr></tbody></table> <h2 class="relative group"><a id="working-without-file-access" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#working-without-file-access"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Working Without File Access</span></h2> <p data-svelte-h="svelte-s1susn">When you don’t have direct access to the user’s files (e.g., web interface), ask the user to run these commands to understand their dataset:</p> <p data-svelte-h="svelte-183nx4y"><strong>Dataset structure</strong>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START --><span class="hljs-comment"># Show directory tree (install with: pip install tree or brew install tree)</span> | |
| tree -L 3 --filelimit 20 | |
| <span class="hljs-comment"># Alternative without tree:</span> | |
| find . -<span class="hljs-built_in">type</span> f -name <span class="hljs-string">"*.csv"</span> -o -name <span class="hljs-string">"*.json"</span> -o -name <span class="hljs-string">"*.parquet"</span> | <span class="hljs-built_in">head</span> -20<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-ddy51v"><strong>Check file sizes</strong>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START --><span class="hljs-comment"># Total dataset size</span> | |
| <span class="hljs-built_in">du</span> -sh . | |
| <span class="hljs-comment"># Individual file sizes</span> | |
| <span class="hljs-built_in">ls</span> -lh data/<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1x9yqrg"><strong>Peek at data format</strong>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START --><span class="hljs-comment"># First few lines of CSV/JSON</span> | |
| <span class="hljs-built_in">head</span> -n 5 data/train.csv | |
| <span class="hljs-comment"># Check image folder structure</span> | |
| <span class="hljs-built_in">ls</span> -la images/ | <span class="hljs-built_in">head</span> -10<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1lf1f1j"><strong>Quick file count</strong>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-bash "><!-- HTML_TAG_START --><span class="hljs-comment"># Count files by type</span> | |
| find . -name <span class="hljs-string">"*.jpg"</span> | <span class="hljs-built_in">wc</span> -l<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="critical-constraints" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#critical-constraints"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Critical Constraints</span></h2> <p data-svelte-h="svelte-15kuw4y"><strong>Storage Limits</strong>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-yaml "><!-- HTML_TAG_START --><span class="hljs-comment"># Machine-readable Hub limits</span> | |
| <span class="hljs-attr">hub_limits:</span> | |
| <span class="hljs-attr">max_file_size_gb:</span> <span class="hljs-number">200</span> <span class="hljs-comment"># absolute hard stop enforced by LFS</span> | |
| <span class="hljs-attr">recommended_file_size_gb:</span> <span class="hljs-number">50</span> <span class="hljs-comment"># best-practice shard size</span> | |
| <span class="hljs-attr">max_files_per_folder:</span> <span class="hljs-number">10000</span> <span class="hljs-comment"># Git performance threshold</span> | |
| <span class="hljs-attr">max_files_per_repo:</span> <span class="hljs-number">100000</span> <span class="hljs-comment"># Repository file count limit</span> | |
| <span class="hljs-attr">recommended_repo_size_gb:</span> <span class="hljs-number">300</span> <span class="hljs-comment"># public-repo soft cap; contact HF if larger</span> | |
| <span class="hljs-attr">viewer_row_size_mb:</span> <span class="hljs-number">2</span> <span class="hljs-comment"># approximate per-row viewer limit</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-16tkzi3"><strong>Human-readable summary</strong>:</p> <ul data-svelte-h="svelte-9dxgr5"><li>Free: 100GB private datasets</li> <li>Pro (for individuals) | Team or Enterprise (for organizations): 1TB+ private storage per seat (see <a href="https://huggingface.co/pricing" rel="nofollow">pricing</a>)</li> <li>Public: 1TB (contact <a href="mailto:datasets@huggingface.co">datasets@huggingface.co</a> for larger)</li> <li>Per file: 200GB max, <50GB recommended</li> <li>Per folder: <10k files</li></ul> <p data-svelte-h="svelte-bpnjb2">See <a href="https://huggingface.co/docs/hub/storage-limits#repository-limitations-and-recommendations" rel="nofollow">https://huggingface.co/docs/hub/storage-limits#repository-limitations-and-recommendations</a> for current limits for current recommendations for repository sizes and file counts.</p> <h2 class="relative group"><a id="quick-reference-by-data-type" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#quick-reference-by-data-type"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Quick Reference by Data Type</span></h2> <table data-svelte-h="svelte-1qthh5o"><thead><tr><th>Your Data</th> <th>Recommended Approach</th> <th>Quick Command</th></tr></thead> <tbody><tr><td><strong>CSV/JSON files</strong></td> <td>Use built-in loaders (handles any size via memory mapping)</td> <td><code>load_dataset("csv", data_files="data.csv").push_to_hub("username/dataset")</code></td></tr> <tr><td><strong>Images in folders</strong></td> <td>Use <a href="https://huggingface.co/docs/datasets/en/image_load" rel="nofollow">imagefolder</a> for automatic class detection</td> <td><code>load_dataset("imagefolder", data_dir="./images").push_to_hub("username/dataset")</code></td></tr> <tr><td><strong>Audio files</strong></td> <td>Use <a href="https://huggingface.co/docs/datasets/en/audio_load" rel="nofollow">audiofolder</a> for automatic organization</td> <td><code>load_dataset("audiofolder", data_dir="./audio").push_to_hub("username/dataset")</code></td></tr> <tr><td><strong>Video files</strong></td> <td>Use <a href="https://huggingface.co/docs/datasets/en/video_load#videofolder" rel="nofollow">videofolder</a> for automatic organization</td> <td><code>load_dataset("videofolder", data_dir="./videos").push_to_hub("username/dataset")</code></td></tr> <tr><td><strong>PDF documents</strong></td> <td>Use <a href="https://huggingface.co/docs/datasets/en/document_dataset#pdffolder" rel="nofollow">pdffolder</a> for text extraction</td> <td><code>load_dataset("pdffolder", data_dir="./pdfs").push_to_hub("username/dataset")</code></td></tr> <tr><td><strong>Very large datasets (100GB+)</strong></td> <td>Use <code>max_shard_size</code> to control memory usage</td> <td><code>dataset.push_to_hub("username/dataset", max_shard_size="5GB")</code></td></tr> <tr><td><strong>Many files / directories (>10k)</strong></td> <td>Use upload_large_folder to avoid Git limitations</td> <td><code>api.upload_large_folder(folder_path="./data", repo_id="username/dataset", repo_type="dataset")</code></td></tr> <tr><td><strong>Streaming large media</strong></td> <td>WebDataset format for efficient streaming</td> <td>Create .tar shards, then <code>upload_large_folder()</code></td></tr> <tr><td><strong>Scientific data (HDF5, NetCDF)</strong></td> <td>Convert to Parquet with Array features</td> <td>See <a href="#scientific-data">Scientific Data</a> section</td></tr> <tr><td><strong>Custom/proprietary formats</strong></td> <td>Document thoroughly if conversion impossible</td> <td><code>upload_large_folder()</code> with comprehensive README</td></tr></tbody></table> <h2 class="relative group"><a id="upload-workflow" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#upload-workflow"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Upload Workflow</span></h2> <ol start="0"><li data-svelte-h="svelte-coqx"><p>✓ <strong>Gather dataset information</strong> (if needed):</p> <ul><li>What type of data? (images, text, audio, CSV, etc.)</li> <li>How is it organized? (folder structure, single file, multiple files)</li> <li>What’s the approximate size?</li> <li>What format are the files in?</li> <li>Any special requirements? (e.g., streaming, private access)</li> <li>Check for existing README or documentation files that describe the dataset</li></ul></li> <li data-svelte-h="svelte-13hzcf6"><p>✓ <strong>Authenticate</strong>:</p> <ul><li>CLI: <code>hf auth login</code></li> <li>Or use token: <code>HfApi(token="hf_...")</code> or set <code>HF_TOKEN</code> environment variable</li></ul></li> <li data-svelte-h="svelte-1x7390r"><p>✓ <strong>Identify your data type</strong>: Check the <a href="#quick-reference-by-data-type">Quick Reference</a> table above</p></li> <li data-svelte-h="svelte-1lpi177"><p>✓ <strong>Choose upload method</strong>:</p> <ul><li><strong>Small files (<1GB) with hub-compatible format</strong>: Can use <a href="https://huggingface.co/new-dataset" rel="nofollow">Hub UI</a> for quick uploads</li> <li><strong>Built-in loader available</strong>: Use the loader + <code>push_to_hub()</code> (see Quick Reference table)</li> <li><strong>Large datasets or many files</strong>: Use <code>upload_large_folder()</code> for files >100GB or >10k files</li> <li><strong>Custom formats</strong>: Convert to hub-compatible format if possible, otherwise document thoroughly</li></ul></li> <li><p data-svelte-h="svelte-docjp3">✓ <strong>Test locally</strong> (if using built-in loader):</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-comment"># Validate your dataset loads correctly before uploading</span> | |
| dataset = load_dataset(<span class="hljs-string">"loader_name"</span>, data_dir=<span class="hljs-string">"./your_data"</span>) | |
| <span class="hljs-built_in">print</span>(dataset)<!-- HTML_TAG_END --></pre></div></li> <li><p data-svelte-h="svelte-8stqwj">✓ <strong>Upload to Hub</strong>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-comment"># Basic upload</span> | |
| dataset.push_to_hub(<span class="hljs-string">"username/dataset-name"</span>) | |
| <span class="hljs-comment"># With options for large datasets</span> | |
| dataset.push_to_hub( | |
| <span class="hljs-string">"username/dataset-name"</span>, | |
| max_shard_size=<span class="hljs-string">"5GB"</span>, <span class="hljs-comment"># Control memory usage</span> | |
| private=<span class="hljs-literal">True</span> <span class="hljs-comment"># For private datasets</span> | |
| )<!-- HTML_TAG_END --></pre></div></li> <li data-svelte-h="svelte-28v5ni"><p>✓ <strong>Verify your upload</strong>:</p> <ul><li>Check Dataset Viewer: <code>https://huggingface.co/datasets/username/dataset-name</code></li> <li>Test loading: <code>load_dataset("username/dataset-name")</code></li> <li>If viewer shows errors, check the <a href="#common-issues--solutions">Troubleshooting</a> section</li></ul></li></ol> <h2 class="relative group"><a id="common-conversion-patterns" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#common-conversion-patterns"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Common Conversion Patterns</span></h2> <p data-svelte-h="svelte-au918z">When built-in loaders don’t match your data structure, use the datasets library as a compatibility layer. Convert your data to a Dataset object, then use <code>push_to_hub()</code> for maximum flexibility and Dataset Viewer compatibility.</p> <h3 class="relative group"><a id="from-dataframes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#from-dataframes"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>From DataFrames</span></h3> <p data-svelte-h="svelte-lvop0e">If you already have your data working in pandas, polars, or other dataframe libraries, you can convert directly:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-comment"># From pandas DataFrame</span> | |
| <span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd | |
| <span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset | |
| df = pd.read_csv(<span class="hljs-string">"your_data.csv"</span>) | |
| dataset = Dataset.from_pandas(df) | |
| dataset.push_to_hub(<span class="hljs-string">"username/dataset-name"</span>) | |
| <span class="hljs-comment"># From polars DataFrame (direct method)</span> | |
| <span class="hljs-keyword">import</span> polars <span class="hljs-keyword">as</span> pl | |
| <span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset | |
| df = pl.read_csv(<span class="hljs-string">"your_data.csv"</span>) | |
| dataset = Dataset.from_polars(df) <span class="hljs-comment"># Direct conversion</span> | |
| dataset.push_to_hub(<span class="hljs-string">"username/dataset-name"</span>) | |
| <span class="hljs-comment"># From PyArrow Table (useful for scientific data)</span> | |
| <span class="hljs-keyword">import</span> pyarrow <span class="hljs-keyword">as</span> pa | |
| <span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset | |
| <span class="hljs-comment"># If you have a PyArrow table</span> | |
| table = pa.table({<span class="hljs-string">'data'</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>], <span class="hljs-string">'labels'</span>: [<span class="hljs-string">'a'</span>, <span class="hljs-string">'b'</span>, <span class="hljs-string">'c'</span>]}) | |
| dataset = Dataset(table) | |
| dataset.push_to_hub(<span class="hljs-string">"username/dataset-name"</span>) | |
| <span class="hljs-comment"># For Spark/Dask dataframes, see https://huggingface.co/docs/hub/datasets-libraries</span><!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="custom-format-conversion" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#custom-format-conversion"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Custom Format Conversion</span></h2> <p data-svelte-h="svelte-ar9k36">When built-in loaders don’t match your data format, convert to Dataset objects following these principles:</p> <h3 class="relative group"><a id="design-principles" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#design-principles"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Design Principles</span></h3> <p data-svelte-h="svelte-iur0js"><strong>1. Prefer wide/flat structures over joins</strong></p> <ul data-svelte-h="svelte-1dkzusn"><li>Denormalize relational data into single rows for better usability</li> <li>Include all relevant information in each example</li> <li>Lean towards bigger but more usable data - Hugging Face’s infrastructure uses advanced deduplication (XetHub) and Parquet optimizations to handle redundancy efficiently</li></ul> <p data-svelte-h="svelte-r65yvz"><strong>2. Use configs for logical dataset variations</strong></p> <ul data-svelte-h="svelte-9qv5vd"><li>Beyond train/test/val splits, use configs for different subsets or views of your data</li> <li>Each config can have different features or data organization</li> <li>Example: language-specific configs, task-specific views, or data modalities</li></ul> <h3 class="relative group"><a id="conversion-methods" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#conversion-methods"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Conversion Methods</span></h3> <p data-svelte-h="svelte-iuck5z"><strong>Small datasets (fits in memory) - use <code>Dataset.from_dict()</code></strong>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-comment"># Parse your custom format into a dictionary</span> | |
| data_dict = { | |
| <span class="hljs-string">"text"</span>: [<span class="hljs-string">"example1"</span>, <span class="hljs-string">"example2"</span>], | |
| <span class="hljs-string">"label"</span>: [<span class="hljs-string">"positive"</span>, <span class="hljs-string">"negative"</span>], | |
| <span class="hljs-string">"score"</span>: [<span class="hljs-number">0.9</span>, <span class="hljs-number">0.2</span>] | |
| } | |
| <span class="hljs-comment"># Create dataset with appropriate features</span> | |
| <span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Value, ClassLabel | |
| features = Features({ | |
| <span class="hljs-string">'text'</span>: Value(<span class="hljs-string">'string'</span>), | |
| <span class="hljs-string">'label'</span>: ClassLabel(names=[<span class="hljs-string">'negative'</span>, <span class="hljs-string">'positive'</span>]), | |
| <span class="hljs-string">'score'</span>: Value(<span class="hljs-string">'float32'</span>) | |
| }) | |
| dataset = Dataset.from_dict(data_dict, features=features) | |
| dataset.push_to_hub(<span class="hljs-string">"username/dataset"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-15huzsb"><strong>Large datasets (memory-efficient) - use <code>Dataset.from_generator()</code></strong>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">def</span> <span class="hljs-title function_">data_generator</span>(): | |
| <span class="hljs-comment"># Parse your custom format progressively</span> | |
| <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> parse_large_file(<span class="hljs-string">"data.custom"</span>): | |
| <span class="hljs-keyword">yield</span> { | |
| <span class="hljs-string">"text"</span>: item[<span class="hljs-string">"content"</span>], | |
| <span class="hljs-string">"label"</span>: item[<span class="hljs-string">"category"</span>], | |
| <span class="hljs-string">"embedding"</span>: item[<span class="hljs-string">"vector"</span>] | |
| } | |
| <span class="hljs-comment"># Specify features for Dataset Viewer compatibility</span> | |
| <span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Features, Value, ClassLabel, <span class="hljs-type">List</span> | |
| features = Features({ | |
| <span class="hljs-string">'text'</span>: Value(<span class="hljs-string">'string'</span>), | |
| <span class="hljs-string">'label'</span>: ClassLabel(names=[<span class="hljs-string">'cat1'</span>, <span class="hljs-string">'cat2'</span>, <span class="hljs-string">'cat3'</span>]), | |
| <span class="hljs-string">'embedding'</span>: <span class="hljs-type">List</span>(feature=Value(<span class="hljs-string">'float32'</span>), length=<span class="hljs-number">768</span>) | |
| }) | |
| dataset = Dataset.from_generator(data_generator, features=features) | |
| dataset.push_to_hub(<span class="hljs-string">"username/dataset"</span>, max_shard_size=<span class="hljs-string">"1GB"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-12bonk0"><strong>Tip</strong>: For large datasets, test with a subset first by adding a limit to your generator or using <code>.select(range(100))</code> after creation.</p> <h3 class="relative group"><a id="using-configs-for-dataset-variations" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#using-configs-for-dataset-variations"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Using Configs for Dataset Variations</span></h3> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-comment"># Push different configurations of your dataset</span> | |
| dataset_en = Dataset.from_dict(english_data, features=features) | |
| dataset_en.push_to_hub(<span class="hljs-string">"username/multilingual-dataset"</span>, config_name=<span class="hljs-string">"english"</span>) | |
| dataset_fr = Dataset.from_dict(french_data, features=features) | |
| dataset_fr.push_to_hub(<span class="hljs-string">"username/multilingual-dataset"</span>, config_name=<span class="hljs-string">"french"</span>) | |
| <span class="hljs-comment"># Users can then load specific configs</span> | |
| dataset = load_dataset(<span class="hljs-string">"username/multilingual-dataset"</span>, <span class="hljs-string">"english"</span>)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="multi-modal-examples" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#multi-modal-examples"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Multi-modal Examples</span></h3> <p data-svelte-h="svelte-1u8i1rn"><strong>Text + Audio (speech recognition)</strong>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">def</span> <span class="hljs-title function_">speech_generator</span>(): | |
| <span class="hljs-keyword">for</span> audio_file <span class="hljs-keyword">in</span> Path(<span class="hljs-string">"audio/"</span>).glob(<span class="hljs-string">"*.wav"</span>): | |
| transcript_file = audio_file.with_suffix(<span class="hljs-string">".txt"</span>) | |
| <span class="hljs-keyword">yield</span> { | |
| <span class="hljs-string">"audio"</span>: <span class="hljs-built_in">str</span>(audio_file), | |
| <span class="hljs-string">"text"</span>: transcript_file.read_text().strip(), | |
| <span class="hljs-string">"speaker_id"</span>: audio_file.stem.split(<span class="hljs-string">"_"</span>)[<span class="hljs-number">0</span>] | |
| } | |
| features = Features({ | |
| <span class="hljs-string">'audio'</span>: Audio(sampling_rate=<span class="hljs-number">16000</span>), | |
| <span class="hljs-string">'text'</span>: Value(<span class="hljs-string">'string'</span>), | |
| <span class="hljs-string">'speaker_id'</span>: Value(<span class="hljs-string">'string'</span>) | |
| }) | |
| dataset = Dataset.from_generator(speech_generator, features=features) | |
| dataset.push_to_hub(<span class="hljs-string">"username/speech-dataset"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1jgfsm0"><strong>Multiple images per example</strong>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-comment"># Before/after images, medical imaging, etc.</span> | |
| data = { | |
| <span class="hljs-string">"image_before"</span>: [<span class="hljs-string">"img1_before.jpg"</span>, <span class="hljs-string">"img2_before.jpg"</span>], | |
| <span class="hljs-string">"image_after"</span>: [<span class="hljs-string">"img1_after.jpg"</span>, <span class="hljs-string">"img2_after.jpg"</span>], | |
| <span class="hljs-string">"treatment"</span>: [<span class="hljs-string">"method_A"</span>, <span class="hljs-string">"method_B"</span>] | |
| } | |
| features = Features({ | |
| <span class="hljs-string">'image_before'</span>: Image(), | |
| <span class="hljs-string">'image_after'</span>: Image(), | |
| <span class="hljs-string">'treatment'</span>: ClassLabel(names=[<span class="hljs-string">'method_A'</span>, <span class="hljs-string">'method_B'</span>]) | |
| }) | |
| dataset = Dataset.from_dict(data, features=features) | |
| dataset.push_to_hub(<span class="hljs-string">"username/before-after-images"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-wsof8w"><strong>Note</strong>: For text + images, consider using ImageFolder with metadata.csv which handles this automatically.</p> <h2 class="relative group"><a id="essential-features" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#essential-features"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Essential Features</span></h2> <p data-svelte-h="svelte-1odksos">Features define the schema and data types for your dataset columns. Specifying correct features ensures:</p> <ul data-svelte-h="svelte-t5ru0o"><li>Proper data handling and type conversion</li> <li>Dataset Viewer functionality (e.g., image/audio previews)</li> <li>Efficient storage and loading</li> <li>Clear documentation of your data structure</li></ul> <p data-svelte-h="svelte-1qjpjc5">For complete feature documentation, see: <a href="https://huggingface.co/docs/datasets/about_dataset_features" rel="nofollow">Dataset Features</a></p> <h3 class="relative group"><a id="feature-types-overview" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#feature-types-overview"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Feature Types Overview</span></h3> <p data-svelte-h="svelte-1qg37o8"><strong>Basic Types</strong>:</p> <ul data-svelte-h="svelte-3fng8i"><li><code>Value</code>: Scalar values - <code>string</code>, <code>int64</code>, <code>float32</code>, <code>bool</code>, <code>binary</code>, and other numeric types</li> <li><code>ClassLabel</code>: Categorical data with named classes</li> <li><code>Sequence</code>: Lists of any feature type</li> <li><code>LargeList</code>: For very large lists</li></ul> <p data-svelte-h="svelte-1x89mat"><strong>Media Types</strong> (enable Dataset Viewer previews):</p> <ul data-svelte-h="svelte-1tm6u7n"><li><code>Image()</code>: Handles various image formats, returns PIL Image objects</li> <li><code>Audio(sampling_rate=16000)</code>: Audio with array data and optional sampling rate</li> <li><code>Video()</code>: Video files</li> <li><code>Pdf()</code>: PDF documents with text extraction</li></ul> <p data-svelte-h="svelte-s170ij"><strong>Array Types</strong> (for tensors/scientific data):</p> <ul data-svelte-h="svelte-1rqoami"><li><code>Array2D</code>, <code>Array3D</code>, <code>Array4D</code>, <code>Array5D</code>: Fixed or variable-length arrays</li> <li>Example: <code>Array2D(shape=(224, 224), dtype='float32')</code></li> <li>First dimension can be <code>None</code> for variable length</li></ul> <p data-svelte-h="svelte-1xno8op"><strong>Translation Types</strong>:</p> <ul data-svelte-h="svelte-swic33"><li><code>Translation</code>: For translation pairs with fixed languages</li> <li><code>TranslationVariableLanguages</code>: For translations with varying language pairs</li></ul> <p data-svelte-h="svelte-1wn0wiu"><strong>Note</strong>: New feature types are added regularly. Check the documentation for the latest additions.</p> <h2 class="relative group"><a id="upload-methods" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#upload-methods"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Upload Methods</span></h2> <p data-svelte-h="svelte-msn4wa"><strong>Dataset objects (use push_to_hub)</strong>: | |
| Use when you’ve loaded/converted data using the datasets library</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START -->dataset.push_to_hub(<span class="hljs-string">"username/dataset"</span>, max_shard_size=<span class="hljs-string">"5GB"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1oorzyp"><strong>Pre-existing files (use upload_large_folder)</strong>: | |
| Use when you have hub-compatible files (e.g., Parquet files) already prepared and organized</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfApi | |
| api = HfApi() | |
| api.upload_large_folder(folder_path=<span class="hljs-string">"./data"</span>, repo_id=<span class="hljs-string">"username/dataset"</span>, repo_type=<span class="hljs-string">"dataset"</span>, num_workers=<span class="hljs-number">16</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-egl09b"><strong>Important</strong>: Before using <code>upload_large_folder</code>, verify the files meet repository limits:</p> <ul data-svelte-h="svelte-j2q96t"><li>Check folder structure if you have file access: ensure no folder contains >10k files</li> <li>Ask the user to confirm: “Are your files in a hub-compatible format (Parquet/CSV/JSON) and organized appropriately?”</li> <li>For non-standard formats, consider converting to Dataset objects first to ensure compatibility</li></ul> <h2 class="relative group"><a id="validation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#validation"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Validation</span></h2> <p data-svelte-h="svelte-tjyq6k"><strong>Consider small reformatting</strong>: If data is close to a built-in loader format, suggest minor changes:</p> <ul data-svelte-h="svelte-1dn48bv"><li>Rename columns (e.g., ‘filename’ → ‘file_name’ for ImageFolder)</li> <li>Reorganize folders (e.g., move images into class subfolders)</li> <li>Rename files to match expected patterns (e.g., ‘data.csv’ → ‘train.csv’)</li></ul> <p data-svelte-h="svelte-1bpf54c"><strong>Pre-upload</strong>:</p> <ul><li data-svelte-h="svelte-v6f4tr"><p>Test locally: <code>load_dataset("imagefolder", data_dir="./data")</code></p></li> <li><p data-svelte-h="svelte-15dlcf2">Verify features work correctly:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-comment"># Test first example</span> | |
| <span class="hljs-built_in">print</span>(dataset[<span class="hljs-number">0</span>]) | |
| <span class="hljs-comment"># For images: verify they load</span> | |
| <span class="hljs-keyword">if</span> <span class="hljs-string">'image'</span> <span class="hljs-keyword">in</span> dataset.features: | |
| dataset[<span class="hljs-number">0</span>][<span class="hljs-string">'image'</span>] <span class="hljs-comment"># Should return PIL Image</span> | |
| <span class="hljs-comment"># Check dataset size before upload</span> | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"Size: <span class="hljs-subst">{<span class="hljs-built_in">len</span>(dataset)}</span> examples"</span>)<!-- HTML_TAG_END --></pre></div></li> <li data-svelte-h="svelte-1qjx0kd"><p>Check metadata.csv has ‘file_name’ column</p></li> <li data-svelte-h="svelte-u9u4qm"><p>Verify relative paths, no leading slashes</p></li> <li data-svelte-h="svelte-1pgq06d"><p>Ensure no folder >10k files</p></li></ul> <p data-svelte-h="svelte-1hyf2bz"><strong>Post-upload</strong>:</p> <ul data-svelte-h="svelte-8u5q5t"><li>Check viewer: <code>https://huggingface.co/datasets/username/dataset</code></li> <li>Test loading: <code>load_dataset("username/dataset")</code></li> <li>Verify features preserved: <code>print(dataset.features)</code></li></ul> <h2 class="relative group"><a id="common-issues--solutions" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#common-issues--solutions"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Common Issues → Solutions</span></h2> <table data-svelte-h="svelte-1lkt6pd"><thead><tr><th>Issue</th> <th>Solution</th></tr></thead> <tbody><tr><td>“Repository not found”</td> <td>Run <code>hf auth login</code></td></tr> <tr><td>Memory errors</td> <td>Use <code>max_shard_size="500MB"</code></td></tr> <tr><td>Dataset viewer not working</td> <td>Wait 5-10min, check README.md config</td></tr> <tr><td>Timeout errors</td> <td>Use <code>multi_commits=True</code></td></tr> <tr><td>Files >50GB</td> <td>Split into smaller files</td></tr> <tr><td>“File not found”</td> <td>Use relative paths in metadata</td></tr></tbody></table> <h2 class="relative group"><a id="dataset-viewer-configuration" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#dataset-viewer-configuration"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Dataset Viewer Configuration</span></h2> <p data-svelte-h="svelte-g003c9"><strong>Note</strong>: This section is primarily for datasets uploaded directly to the Hub (via UI or <code>upload_large_folder</code>). Datasets uploaded with <code>push_to_hub()</code> typically configure the viewer automatically.</p> <h3 class="relative group"><a id="when-automatic-detection-works" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#when-automatic-detection-works"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>When automatic detection works</span></h3> <p data-svelte-h="svelte-16loxkj">The Dataset Viewer automatically detects standard structures:</p> <ul data-svelte-h="svelte-1hto711"><li>Files named: <code>train.csv</code>, <code>test.json</code>, <code>validation.parquet</code></li> <li>Directories named: <code>train/</code>, <code>test/</code>, <code>validation/</code></li> <li>Split names with delimiters: <code>test-data.csv</code> ✓ (not <code>testdata.csv</code> ✗)</li></ul> <h3 class="relative group"><a id="manual-configuration" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#manual-configuration"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Manual configuration</span></h3> <p data-svelte-h="svelte-v9qyvd">For custom structures, add YAML to your README.md:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-yaml "><!-- HTML_TAG_START --><span class="hljs-meta">---</span> | |
| <span class="hljs-attr">configs:</span> | |
| <span class="hljs-bullet">-</span> <span class="hljs-attr">config_name:</span> <span class="hljs-string">default</span> <span class="hljs-comment"># Required even for single config!</span> | |
| <span class="hljs-attr">data_files:</span> | |
| <span class="hljs-bullet">-</span> <span class="hljs-attr">split:</span> <span class="hljs-string">train</span> | |
| <span class="hljs-attr">path:</span> <span class="hljs-string">"data/train/*.parquet"</span> | |
| <span class="hljs-bullet">-</span> <span class="hljs-attr">split:</span> <span class="hljs-string">test</span> | |
| <span class="hljs-attr">path:</span> <span class="hljs-string">"data/test/*.parquet"</span> | |
| <span class="hljs-meta">---</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-mkdef3">Multiple configurations example:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-yaml "><!-- HTML_TAG_START --><span class="hljs-meta">---</span> | |
| <span class="hljs-attr">configs:</span> | |
| <span class="hljs-bullet">-</span> <span class="hljs-attr">config_name:</span> <span class="hljs-string">english</span> | |
| <span class="hljs-attr">data_files:</span> <span class="hljs-string">"en/*.parquet"</span> | |
| <span class="hljs-bullet">-</span> <span class="hljs-attr">config_name:</span> <span class="hljs-string">french</span> | |
| <span class="hljs-attr">data_files:</span> <span class="hljs-string">"fr/*.parquet"</span> | |
| <span class="hljs-meta">---</span><!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="common-viewer-issues" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#common-viewer-issues"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Common viewer issues</span></h3> <ul data-svelte-h="svelte-13hljct"><li><strong>No viewer after upload</strong>: Wait 5-10 minutes for processing</li> <li><strong>“Config names error”</strong>: Add <code>config_name</code> field (required!)</li> <li><strong>Files not detected</strong>: Check naming patterns (needs delimiters)</li> <li><strong>Viewer disabled</strong>: Remove <code>viewer: false</code> from README YAML</li></ul> <h2 class="relative group"><a id="quick-templates" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#quick-templates"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Quick Templates</span></h2> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-comment"># ImageFolder with metadata</span> | |
| dataset = load_dataset(<span class="hljs-string">"imagefolder"</span>, data_dir=<span class="hljs-string">"./images"</span>) | |
| dataset.push_to_hub(<span class="hljs-string">"username/dataset"</span>) | |
| <span class="hljs-comment"># Memory-efficient upload</span> | |
| dataset.push_to_hub(<span class="hljs-string">"username/dataset"</span>, max_shard_size=<span class="hljs-string">"500MB"</span>) | |
| <span class="hljs-comment"># Multiple CSV files</span> | |
| dataset = load_dataset(<span class="hljs-string">'csv'</span>, data_files={<span class="hljs-string">'train'</span>: <span class="hljs-string">'train.csv'</span>, <span class="hljs-string">'test'</span>: <span class="hljs-string">'test.csv'</span>}) | |
| dataset.push_to_hub(<span class="hljs-string">"username/dataset"</span>)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="documentation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#documentation"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Documentation</span></h2> <p data-svelte-h="svelte-vz9qls"><strong>Core docs</strong>: <a href="https://huggingface.co/docs/hub/datasets-adding" rel="nofollow">Adding datasets</a> | <a href="https://huggingface.co/docs/hub/datasets-viewer" rel="nofollow">Dataset viewer</a> | <a href="https://huggingface.co/docs/hub/storage-limits" rel="nofollow">Storage limits</a> | <a href="https://huggingface.co/docs/datasets/upload_dataset" rel="nofollow">Upload guide</a></p> <h2 class="relative group"><a id="dataset-cards" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#dataset-cards"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Dataset Cards</span></h2> <p data-svelte-h="svelte-3if0hu">Remind users to add a dataset card (README.md) with:</p> <ul data-svelte-h="svelte-jr8bcr"><li>Dataset description and usage</li> <li>License information</li> <li>Citation details</li></ul> <p data-svelte-h="svelte-1f9g19l">See <a href="https://huggingface.co/docs/hub/datasets-cards" rel="nofollow">Dataset Cards guide</a> for details.</p> <hr> <h2 class="relative group"><a id="appendix-special-cases" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#appendix-special-cases"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Appendix: Special Cases</span></h2> <h3 class="relative group"><a id="webdataset-structure" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#webdataset-structure"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>WebDataset Structure</span></h3> <p data-svelte-h="svelte-1cttxvh">For streaming large media datasets:</p> <ul data-svelte-h="svelte-1pv9cl9"><li>Create 1-5GB tar shards</li> <li>Consistent internal structure</li> <li>Upload with <code>upload_large_folder</code></li></ul> <h3 class="relative group"><a id="scientific-data" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#scientific-data"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Scientific Data</span></h3> <ul data-svelte-h="svelte-1h00ccq"><li>HDF5/NetCDF → Convert to Parquet with Array features</li> <li>Time series → Array2D(shape=(None, n))</li> <li>Complex metadata → Store as JSON strings</li></ul> <h3 class="relative group"><a id="community-resources" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#community-resources"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Community Resources</span></h3> <p data-svelte-h="svelte-qsd78w">For very specialized or bespoke formats:</p> <ul data-svelte-h="svelte-gadoy2"><li>Search the Hub for similar datasets: <code>https://huggingface.co/datasets</code></li> <li>Ask for advice on the <a href="https://discuss.huggingface.co/c/datasets/10" rel="nofollow">Hugging Face Forums</a></li> <li>Join the <a href="https://hf.co/join/discord" rel="nofollow">Hugging Face Discord</a> for real-time help</li> <li>Many domain-specific formats already have examples on the Hub</li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-upload-guide-llm.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p> | |
| <script> | |
| { | |
| __sveltekit_fm3we3 = { | |
| assets: "/docs/hub/pr_2549/en", | |
| base: "/docs/hub/pr_2549/en", | |
| env: {} | |
| }; | |
| const element = document.currentScript.parentElement; | |
| const data = [null,null]; | |
| Promise.all([ | |
| import("/docs/hub/pr_2549/en/_app/immutable/entry/start.e1a19059.js"), | |
| import("/docs/hub/pr_2549/en/_app/immutable/entry/app.d45f53b9.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 63], | |
| data, | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 103 kB
- Xet hash:
- 9bbe11de06685ea039b5f5932157a8970c95abc5d095e5074adc4af73018c8e0
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.