bundle gemma_assets (Gemma tokenizer + config sidecars) - the loader required them but they were never distributed; single-file-Gemma users hit 'sidecar files missing' even with a writable mount
Browse files
gemma_assets/config.json
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"Gemma3ForConditionalGeneration"
|
| 4 |
+
],
|
| 5 |
+
"boi_token_index": 255999,
|
| 6 |
+
"eoi_token_index": 256000,
|
| 7 |
+
"eos_token_id": [
|
| 8 |
+
1,
|
| 9 |
+
106
|
| 10 |
+
],
|
| 11 |
+
"image_token_index": 262144,
|
| 12 |
+
"initializer_range": 0.02,
|
| 13 |
+
"mm_tokens_per_image": 256,
|
| 14 |
+
"model_type": "gemma3",
|
| 15 |
+
"text_config": {
|
| 16 |
+
"hidden_size": 3840,
|
| 17 |
+
"intermediate_size": 15360,
|
| 18 |
+
"model_type": "gemma3_text",
|
| 19 |
+
"num_attention_heads": 16,
|
| 20 |
+
"num_hidden_layers": 48,
|
| 21 |
+
"num_key_value_heads": 8,
|
| 22 |
+
"rope_scaling": {
|
| 23 |
+
"factor": 8.0,
|
| 24 |
+
"rope_type": "linear"
|
| 25 |
+
},
|
| 26 |
+
"sliding_window": 1024
|
| 27 |
+
},
|
| 28 |
+
"torch_dtype": "bfloat16",
|
| 29 |
+
"transformers_version": "4.50.0.dev0",
|
| 30 |
+
"vision_config": {
|
| 31 |
+
"hidden_size": 1152,
|
| 32 |
+
"image_size": 896,
|
| 33 |
+
"intermediate_size": 4304,
|
| 34 |
+
"model_type": "siglip_vision_model",
|
| 35 |
+
"num_attention_heads": 16,
|
| 36 |
+
"num_hidden_layers": 27,
|
| 37 |
+
"patch_size": 14,
|
| 38 |
+
"vision_use_head": false
|
| 39 |
+
}
|
| 40 |
+
}
|
gemma_assets/preprocessor_config.json
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"do_convert_rgb": null,
|
| 3 |
+
"do_normalize": true,
|
| 4 |
+
"do_pan_and_scan": null,
|
| 5 |
+
"do_rescale": true,
|
| 6 |
+
"do_resize": true,
|
| 7 |
+
"image_mean": [
|
| 8 |
+
0.5,
|
| 9 |
+
0.5,
|
| 10 |
+
0.5
|
| 11 |
+
],
|
| 12 |
+
"image_processor_type": "Gemma3ImageProcessor",
|
| 13 |
+
"image_seq_length": 256,
|
| 14 |
+
"image_std": [
|
| 15 |
+
0.5,
|
| 16 |
+
0.5,
|
| 17 |
+
0.5
|
| 18 |
+
],
|
| 19 |
+
"pan_and_scan_max_num_crops": null,
|
| 20 |
+
"pan_and_scan_min_crop_size": null,
|
| 21 |
+
"pan_and_scan_min_ratio_to_activate": null,
|
| 22 |
+
"processor_class": "Gemma3Processor",
|
| 23 |
+
"resample": 2,
|
| 24 |
+
"rescale_factor": 0.00392156862745098,
|
| 25 |
+
"size": {
|
| 26 |
+
"height": 896,
|
| 27 |
+
"width": 896
|
| 28 |
+
}
|
| 29 |
+
}
|
gemma_assets/processor_config.json
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"image_seq_length": 256,
|
| 3 |
+
"processor_class": "Gemma3Processor"
|
| 4 |
+
}
|
gemma_assets/special_tokens_map.json
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"boi_token": "<start_of_image>",
|
| 3 |
+
"bos_token": {
|
| 4 |
+
"content": "<bos>",
|
| 5 |
+
"lstrip": false,
|
| 6 |
+
"normalized": false,
|
| 7 |
+
"rstrip": false,
|
| 8 |
+
"single_word": false
|
| 9 |
+
},
|
| 10 |
+
"eoi_token": "<end_of_image>",
|
| 11 |
+
"eos_token": {
|
| 12 |
+
"content": "<eos>",
|
| 13 |
+
"lstrip": false,
|
| 14 |
+
"normalized": false,
|
| 15 |
+
"rstrip": false,
|
| 16 |
+
"single_word": false
|
| 17 |
+
},
|
| 18 |
+
"image_token": "<image_soft_token>",
|
| 19 |
+
"pad_token": {
|
| 20 |
+
"content": "<pad>",
|
| 21 |
+
"lstrip": false,
|
| 22 |
+
"normalized": false,
|
| 23 |
+
"rstrip": false,
|
| 24 |
+
"single_word": false
|
| 25 |
+
},
|
| 26 |
+
"unk_token": {
|
| 27 |
+
"content": "<unk>",
|
| 28 |
+
"lstrip": false,
|
| 29 |
+
"normalized": false,
|
| 30 |
+
"rstrip": false,
|
| 31 |
+
"single_word": false
|
| 32 |
+
}
|
| 33 |
+
}
|
gemma_assets/tokenizer.model
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c
|
| 3 |
+
size 4689074
|
gemma_assets/tokenizer_config.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
rebels_loaders.py
CHANGED
|
@@ -845,11 +845,15 @@ class RebelsJE_TextEncoder:
|
|
| 845 |
"Searched, in order:\n " + "\n ".join(sidecar_sources) + "\n"
|
| 846 |
"Staging folder: " + temp_folder + "\n"
|
| 847 |
"If a file shows FOUND-but-could-not-stage, the staging folder is "
|
| 848 |
-
"read-only
|
| 849 |
-
"
|
| 850 |
-
"
|
| 851 |
-
"
|
| 852 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 853 |
)
|
| 854 |
|
| 855 |
model_folder = Path(temp_folder)
|
|
|
|
| 845 |
"Searched, in order:\n " + "\n ".join(sidecar_sources) + "\n"
|
| 846 |
"Staging folder: " + temp_folder + "\n"
|
| 847 |
"If a file shows FOUND-but-could-not-stage, the staging folder is "
|
| 848 |
+
"read-only - the pack falls back to a temp dir automatically, so "
|
| 849 |
+
"update the pack. If a file is 'not found in any searched location', "
|
| 850 |
+
"your pack's gemma_assets folder is incomplete: update to the latest "
|
| 851 |
+
"patch (it now bundles gemma_assets), OR copy the six sidecar files "
|
| 852 |
+
"(config.json, preprocessor_config.json, processor_config.json, "
|
| 853 |
+
"special_tokens_map.json, tokenizer_config.json, tokenizer.model) "
|
| 854 |
+
"directly into the same folder as your Gemma weights. Do NOT pull "
|
| 855 |
+
"config.json from the google repo - that is the multimodal variant "
|
| 856 |
+
"and is NOT compatible with this pipeline."
|
| 857 |
)
|
| 858 |
|
| 859 |
model_folder = Path(temp_folder)
|