Trimmed pplx-embed-v1 French 32768 tokens

1_Pooling/config.json

@@ -0,0 +1,10 @@
+{
+    "word_embedding_dimension": 1024,
+    "pooling_mode_cls_token": false,
+    "pooling_mode_mean_tokens": true,
+    "pooling_mode_max_tokens": false,
+    "pooling_mode_mean_sqrt_len_tokens": false,
+    "pooling_mode_weightedmean_tokens": false,
+    "pooling_mode_lasttoken": false,
+    "include_prompt": true
+}

README.md

@@ -0,0 +1,60 @@
+---
+pipeline_tag: sentence-similarity
+language: fra
+license: mit
+tags:
+  - trimmed
+  - sentence-transformers
+  - feature-extraction
+  - bidirectional_pplx_qwen3
+library_name: sentence-transformers
+base_model: perplexity-ai/pplx-embed-v1-0.6b
+base_model_relation: quantized
+datasets:
+  - Lumberjackk/fineweb-2-trimming
+---
+
+# pplx-embed-v1-fra-32768
+
+This model is a **20.47% smaller** version of [perplexity-ai/pplx-embed-v1-0.6b](https://huggingface.co/perplexity-ai/pplx-embed-v1-0.6b) optimized for French language via vocabulary size reduction using the [trimming](https://huggingface.co/blog/introduction-to-trimming) method.
+
+This trimmed model should perform similarly to the original model with only **32,768 tokens** and a much smaller memory footprint. However, it may not perform well for other languages as tokens not commonly used in French were removed from the vocabulary.
+
+## Model Statistics
+
+| Metric | Original | Trimmed | Reduction |
+|--------|----------|---------|-----------|
+| **Vocabulary size** | 151,643 tokens | 32,768 tokens | **78.39%** |
+| **Model size** | 596,049,920 params | 474,021,888 params | **20.47%** |
+
+## Mining Dataset Statistics
+
+- **Number of texts used for mining**: 200,000 texts  
+- **Dataset**: [Lumberjackk/fineweb-2-trimming](https://huggingface.co/datasets/Lumberjackk/fineweb-2-trimming)
+
+## Usage
+
+```python
+from sentence_transformers import SentenceTransformer
+
+model = SentenceTransformer("alphaedge-ai/pplx-embed-v1-fra-32768", trust_remote_code=True)
+texts = [
+    "Chunk 1",
+    "Chunk 2",
+    "Chunk 3",
+]
+embeddings = model.encode(texts)
+print(embeddings.shape)
+```
+
+## Citation
+
+#### pplx-embed
+
+```bibtex
+@article{pplxembed2025,
+  title={pplx-embed: State-of-the-Art Embedding Models for Web-Scale Retrieval},
+  author={Perplexity AI},
+  year={2025}
+}
+```

added_tokens.json

@@ -0,0 +1,18 @@
+{
+  "</think>": 32767,
+  "<think>": 32766,
+  "<|box_end|>": 32758,
+  "<|box_start|>": 32757,
+  "<|endoftext|>": 32752,
+  "<|im_end|>": 32754,
+  "<|im_start|>": 32753,
+  "<|image_pad|>": 32764,
+  "<|object_ref_end|>": 32756,
+  "<|object_ref_start|>": 32755,
+  "<|quad_end|>": 32760,
+  "<|quad_start|>": 32759,
+  "<|video_pad|>": 32765,
+  "<|vision_end|>": 32762,
+  "<|vision_pad|>": 32763,
+  "<|vision_start|>": 32761
+}

config.json

@@ -0,0 +1,71 @@
+{
+  "architectures": [
+    "PPLXQwen3Model"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration.PPLXQwen3Config",
+    "AutoModel": "modeling.PPLXQwen3Model"
+  },
+  "eos_token_id": 32752,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "bidirectional_pplx_qwen3",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": 32752,
+  "rms_norm_eps": 1e-06,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers.js_config": {
+    "use_external_data_format": {
+      "model.onnx": 2,
+      "model_q4.onnx": 1,
+      "model_quantized.onnx": 1
+    }
+  },
+  "transformers_version": "5.3.0.dev0",
+  "use_bidirectional_attention": true,
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 32768,
+  "rope_theta": 1000000,
+  "torch_dtype": "bfloat16"
+}

configuration.py

@@ -0,0 +1,5 @@
+from transformers.models.qwen3.configuration_qwen3 import Qwen3Config
+
+
+class PPLXQwen3Config(Qwen3Config):
+    model_type = "bidirectional_pplx_qwen3"

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b7270a0976ab069f91fc737480d5b358b75bff817e501c77ffe7eabaf6fbea8
+size 948077008

modeling.py

@@ -0,0 +1,83 @@
+from typing import Callable
+import torch
+from transformers import Qwen3Model
+from transformers.cache_utils import Cache
+from transformers.masking_utils import create_causal_mask
+from transformers.modeling_outputs import BaseModelOutputWithPooling
+from transformers.processing_utils import Unpack
+from transformers.utils import TransformersKwargs
+from .configuration import PPLXQwen3Config
+
+
+# From modeling_t5gemma.py
+def bidirectional_mask_function(attention_mask: torch.Tensor | None) -> Callable:
+    """
+    This creates bidirectional attention mask.
+    """
+
+    def inner_mask(batch_idx: int, head_idx: int, q_idx: int, kv_idx: int) -> bool:
+        if attention_mask is None:
+            return torch.ones((), dtype=torch.bool)
+        return attention_mask[batch_idx, kv_idx].to(torch.bool)
+
+    return inner_mask
+
+
+class PPLXQwen3Model(Qwen3Model):
+    _supports_flash_attn = True
+    _supports_sdpa = True
+
+    config_class = PPLXQwen3Config
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.post_init()
+
+    def post_init(self):
+        super().post_init()
+        # Override to set all layers to non-causal attention. This'll work with attn_implementation="flash_attention_2" or "sdpa"
+        for layer in self.layers:
+            layer.self_attn.is_causal = False
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        use_cache: bool | None = None,
+        cache_position: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> BaseModelOutputWithPooling:
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+            input_ids = None
+
+        # We construct a dummy tensor imitating initial positions
+        dummy_cache_position = torch.arange(
+            inputs_embeds.shape[1], device=inputs_embeds.device, dtype=torch.long
+        )
+        attention_mask = {
+            "full_attention": create_causal_mask(
+                config=self.config,
+                input_embeds=inputs_embeds,
+                attention_mask=attention_mask,
+                cache_position=dummy_cache_position,
+                past_key_values=None,
+                position_ids=position_ids,
+                or_mask_function=bidirectional_mask_function(attention_mask),
+            )
+        }
+
+        outputs = super().forward(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            cache_position=cache_position,
+            **kwargs,
+        )
+        return outputs

modules.json

@@ -0,0 +1,21 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "",
+    "type": "st_quantize.FlexibleQuantizer",
+    "kwargs": ["quantization"]
+  }
+]

st_quantize.py

@@ -0,0 +1,122 @@
+import torch
+import numpy as np
+from typing import Literal
+from sentence_transformers.models import Module
+
+
+class Quantizer(torch.nn.Module):
+    def __init__(self, hard: bool = True):
+        """
+        Args:
+            hard: Whether to use hard or soft quantization. Defaults to True.
+        """
+        super().__init__()
+        self._hard = hard
+
+    def _hard_quantize(self, x, *args, **kwargs) -> torch.Tensor:
+        raise NotImplementedError
+
+    def _soft_quantize(self, x, *args, **kwargs) -> torch.Tensor:
+        raise NotImplementedError
+
+    def forward(self, x, *args, **kwargs) -> torch.Tensor:
+        soft = self._soft_quantize(x, *args, **kwargs)
+
+        if not self._hard:
+            result = soft
+        else:
+            result = (
+                self._hard_quantize(x, *args, **kwargs).detach() + soft - soft.detach()
+            )
+
+        return result
+
+
+class Int8TanhQuantizer(Quantizer):
+    def __init__(
+        self,
+        hard: bool = True,
+    ):
+        super().__init__(hard=hard)
+        self.qmin = -128
+        self.qmax = 127
+
+    def _soft_quantize(self, x, *args, **kwargs):
+        return torch.tanh(x)
+
+    def _hard_quantize(self, x, *args, **kwargs):
+        soft = self._soft_quantize(x)
+        int_x = torch.round(soft * self.qmax)
+        int_x = torch.clamp(int_x, self.qmin, self.qmax)
+        return int_x
+
+
+class BinaryTanhQuantizer(Quantizer):
+    def __init__(
+        self,
+        hard: bool = True,
+        scale: float = 1.0,
+    ):
+        super().__init__(hard)
+        self._scale = scale
+
+    def _soft_quantize(self, x, *args, **kwargs):
+        return torch.tanh(self._scale * x)
+
+    def _hard_quantize(self, x, *args, **kwargs):
+        return torch.where(x >= 0, 1.0, -1.0)
+    
+
+class PackedBinaryQuantizer:
+    def __call__(self, x: torch.Tensor) -> torch.Tensor:
+        bits = np.where(x.cpu().numpy() >=  0, True, False)
+        packed = np.packbits(bits, axis=-1)
+        return torch.from_numpy(packed).to(x.device)
+
+
+class FlexibleQuantizer(Module):
+    def __init__(self):
+        super().__init__()
+        self._int8_quantizer = Int8TanhQuantizer()
+        self._binary_quantizer = BinaryTanhQuantizer()
+        self._packed_binary_quantizer = PackedBinaryQuantizer()
+
+    def forward(
+        self,
+        features: dict[str, torch.Tensor],
+        quantization: Literal["int8", "binary", "ubinary"] = "int8",
+        **kwargs
+    ) -> dict[str, torch.Tensor]:
+        if quantization == "int8":
+            features["sentence_embedding"] = self._int8_quantizer(
+                features["sentence_embedding"]
+            )
+        elif quantization == "binary":
+            features["sentence_embedding"] = self._binary_quantizer(
+                features["sentence_embedding"]
+            )
+        elif quantization == "ubinary":
+            features["sentence_embedding"] = self._packed_binary_quantizer(
+                features["sentence_embedding"]
+            )
+        else:
+            raise ValueError(
+                f"Invalid quantization type: {quantization}. Must be 'binary', 'ubinary', or 'int8'."
+            )
+        return features
+
+    @classmethod
+    def load(
+        cls,
+        model_name_or_path: str,
+        subfolder: str = "",
+        token: bool | str | None = None,
+        cache_folder: str | None = None,
+        revision: str | None = None,
+        local_files_only: bool = False,
+        **kwargs,
+    ):
+        return cls()
+        
+    def save(self, output_path: str, *args, **kwargs) -> None: 
+        return

tokenizer_config.json

@@ -0,0 +1,163 @@
+{
+  "tokenizer_class": "Qwen2Tokenizer",
+  "model_max_length": 131072,
+  "clean_up_tokenization_spaces": false,
+  "errors": "replace",
+  "split_special_tokens": false,
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "eos_token": "<|endoftext|>",
+  "eos_token_id": 32752,
+  "pad_token": "<|endoftext|>",
+  "pad_token_id": 32752,
+  "bos_token": null,
+  "unk_token": null,
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>",
+    "<think>",
+    "</think>"
+  ],
+  "added_tokens_decoder": {
+    "32752": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32753": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32754": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32755": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32756": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32757": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32758": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32759": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32760": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32761": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32762": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32763": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32764": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32765": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32766": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32767": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  }
+}