File size: 2,487 Bytes
cc725be | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 | import os
import random
from pathlib import Path
from datasets import load_dataset
GUTENBERG = "D:/gutenberg_en.txt"
RU_BOOKS = "D:/ru_books.txt"
SCRIPT_DIR = Path(__file__).parent
OUT_DIR = SCRIPT_DIR / "sample"
OUT_DIR.mkdir(parents=True, exist_ok=True)
OUT_EN = OUT_DIR / "gutenberg_sample.txt"
OUT_RU = OUT_DIR / "ru_books_sample.txt"
N_SAMPLES = 5000
MIN_LEN = 100
MAX_LEN = 2000
MAX_SCAN = 2_000_000
def download_gutenberg():
print("downloading gutenberg...")
ds = load_dataset("AdhyanshVerma/pg-en", split="train", streaming=True)
with open(GUTENBERG, "w", encoding="utf-8") as f:
n = 0
for s in ds:
t = s.get("text", "").strip()
if t:
f.write(t + "\n")
n += 1
if n >= 500_000:
break
print(f"gutenberg: {n} lines")
def download_ru_books():
print("downloading ru books...")
ds = load_dataset("maxzt/RuHeritage-Corpus", split="train", streaming=True)
with open(RU_BOOKS, "w", encoding="utf-8") as f:
n = 0
for s in ds:
t = s.get("text", "").strip()
if t:
f.write(t + "\n")
n += 1
if n >= 500_000:
break
print(f"ru books: {n} lines")
def make_sample(src_path, dst_path, n_samples, min_len, max_len):
if not os.path.exists(src_path):
print(f"missing: {src_path}")
return 0
candidates = []
total = 0
with open(src_path, "r", encoding="utf-8", errors="ignore") as f:
for line in f:
total += 1
line = line.strip()
if min_len <= len(line) <= max_len:
candidates.append(line)
if total >= MAX_SCAN:
break
if not candidates:
return 0
random.shuffle(candidates)
selected = candidates[:n_samples]
with open(dst_path, "w", encoding="utf-8") as f:
for line in selected:
f.write(line + "\n")
return len(selected)
if __name__ == "__main__":
random.seed(42)
if not os.path.exists(GUTENBERG):
download_gutenberg()
if not os.path.exists(RU_BOOKS):
download_ru_books()
en = make_sample(GUTENBERG, OUT_EN, N_SAMPLES, MIN_LEN, MAX_LEN)
ru = make_sample(RU_BOOKS, OUT_RU, N_SAMPLES, MIN_LEN, MAX_LEN)
print(f"en: {en}")
print(f"ru: {ru}") |