from authentic CLIPS

Everything copied from clip-s barring the .bin and .safetensors

README.md

@@ -1,8 +1,47 @@
 ---
-tags:
-- clip
-library_name: open_clip
-pipeline_tag: zero-shot-image-classification
-license: mit
+license: apache-2.0
+datasets:
+- UCSC-VLAA/Recap-DataComp-1B
 ---
-# Model card for CLIC-CLIPS-ViT-L-14-224-PixelProse
+# Model Card for ViT-L-14-CLIPS-224-Recap-DataComp-1B
+
+## Model Details
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** https://github.com/UCSC-VLAA/CLIPS
+- **Paper:** https://arxiv.org/abs/2411.16828
+- **Project Page:** https://ucsc-vlaa.github.io/CLIPS/
+
+## Model Usage
+### With OpenCLIP
+#### Note: We made modifications to the tokenizer implementation in open_clip/tokenizer.py.
+#### For more details, refer to https://github.com/UCSC-VLAA/CLIPS.
+
+```
+import torch
+import torch.nn.functional as F
+from urllib.request import urlopen
+from PIL import Image
+from open_clip import create_model_from_pretrained, get_tokenizer
+
+model, preprocess = create_model_from_pretrained('hf-hub:UCSC-VLAA/ViT-L-14-CLIPS-224-Recap-DataComp-1B')
+tokenizer = get_tokenizer('hf-hub:UCSC-VLAA/ViT-L-14-CLIPS-224-Recap-DataComp-1B')
+
+image = Image.open(urlopen(
+    'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png'
+))
+image = preprocess(image).unsqueeze(0)
+
+text = tokenizer(["a diagram", "a dog", "a cat", "a beignet"], context_length=model.context_length)
+
+with torch.no_grad(), torch.cuda.amp.autocast():
+    image_features = model.encode_image(image)
+    text_features = model.encode_text(text)
+    image_features = F.normalize(image_features, dim=-1)
+    text_features = F.normalize(text_features, dim=-1)
+
+    text_probs = (100.0 * image_features @ text_features.T).softmax(dim=-1)
+
+print("Label probs:", text_probs)  # prints: [[0., 0., 0., 1.0]]
+```   

open_clip_config.json

@@ -10,8 +10,8 @@
       "pool_type": "avg",
       "final_ln_after_pool": true,
       "norm_kwargs": {
-        "eps": 1e-06
-      }
+            "eps":  1e-6
+        }
     },
     "text_cfg": {
       "context_length": 80,
@@ -29,22 +29,22 @@
         "approximate": "tanh"
       },
       "norm_kwargs": {
-        "eps": 1e-06
-      }
+            "eps":  1e-6
+        }
     }
   },
   "preprocess_cfg": {
     "mean": [
-      0.48145466,
-      0.4578275,
-      0.40821073
+      0.485,
+      0.456,
+      0.406
     ],
     "std": [
-      0.26862954,
-      0.26130258,
-      0.27577711
+      0.229,
+      0.224,
+      0.225
     ],
-    "interpolation": "bicubic",
-    "resize_mode": "shortest"
+    "interpolation": "bilinear",
+    "resize_mode": "squash"
   }
 }

special_tokens_map.json

@@ -1,6 +1,7 @@
 {
-  "bos_token": "<|startoftext|>",
-  "eos_token": "<|endoftext|>",
-  "pad_token": "<|endoftext|>",
-  "unk_token": "<|endoftext|>"
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
 }

tokenizer_config.json

@@ -1,31 +1,56 @@
 {
-  "add_prefix_space": false,
   "added_tokens_decoder": {
-    "49406": {
-      "content": "<|startoftext|>",
+    "0": {
+      "content": "[PAD]",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49407": {
-      "content": "<|endoftext|>",
+    "100": {
+      "content": "[UNK]",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     }
   },
-  "additional_special_tokens": [],
-  "bos_token": "<|startoftext|>",
-  "clean_up_tokenization_spaces": true,
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
   "do_lower_case": true,
-  "eos_token": "<|endoftext|>",
-  "errors": "replace",
-  "model_max_length": 77,
-  "pad_token": "<|endoftext|>",
-  "tokenizer_class": "CLIPTokenizer",
-  "unk_token": "<|endoftext|>"
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
 }