Rename test.py to TestTokenizer.py
Browse files
test.py → TestTokenizer.py
RENAMED
|
@@ -1,7 +1,6 @@
|
|
| 1 |
-
python -c '
|
| 2 |
from transformers import AutoTokenizer
|
| 3 |
tok = AutoTokenizer.from_pretrained(".")
|
| 4 |
print("Vocab size:", len(tok))
|
| 5 |
print("pad_token_id:", tok.pad_token_id)
|
| 6 |
print("eos_token_id:", tok.eos_token_id)
|
| 7 |
-
|
|
|
|
|
|
|
| 1 |
from transformers import AutoTokenizer
|
| 2 |
tok = AutoTokenizer.from_pretrained(".")
|
| 3 |
print("Vocab size:", len(tok))
|
| 4 |
print("pad_token_id:", tok.pad_token_id)
|
| 5 |
print("eos_token_id:", tok.eos_token_id)
|
| 6 |
+
|