Upload folder using huggingface_hub

README.md

@@ -62,23 +62,46 @@ from transformers import AutoModel, AutoTokenizer
 tokenizer = AutoTokenizer.from_pretrained("evm-alpha/semantic-evm-mlm-chkp1000")
 model = AutoModel.from_pretrained("evm-alpha/semantic-evm-mlm-chkp1000")
 
-# Encode bytecode (as hex string)
-bytecode = "0x608060405234801561001057600080fd5b50"
-inputs = tokenizer(bytecode, return_tensors="pt")
+# This tokenizer uses byte-level BPE (GPT-2 style).
+# You must convert raw bytes to the GPT-2 unicode format:
+
+def bytes_to_unicode_map():
+    """GPT-2 byte-to-unicode mapping."""
+    bs = list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
+    cs = bs[:]
+    n = 0
+    for b in range(256):
+        if b not in bs:
+            bs.append(b)
+            cs.append(256 + n)
+            n += 1
+    return dict(zip(bs, [chr(c) for c in cs]))
+
+BYTE_MAP = bytes_to_unicode_map()
+
+def encode_evm_bytes(data: bytes) -> str:
+    """Convert raw EVM bytes to tokenizer input string."""
+    return "".join(BYTE_MAP[b] for b in data)
+
+# Example: encode raw EVM bytecode
+bytecode_hex = "608060405234801561001057600080fd5b50"
+raw_bytes = bytes.fromhex(bytecode_hex)
+tokenizer_input = encode_evm_bytes(raw_bytes)
+
+inputs = tokenizer(tokenizer_input, return_tensors="pt")
 outputs = model(**inputs)
 
-# Get embeddings (mean pooling across sequence)
+# Get embeddings (mean pooling)
 embeddings = outputs.last_hidden_state  # [batch, seq_len, hidden_size]
-attention_mask = inputs["attention_mask"]  # [batch, seq_len]
-masked_embeddings = embeddings * attention_mask.unsqueeze(-1)
-pooled = masked_embeddings.sum(1) / attention_mask.sum(1, keepdim=True)  # mean pooling
+mask = inputs["attention_mask"].unsqueeze(-1)
+pooled = (embeddings * mask).sum(1) / mask.sum(1)
 ```
 
 ## Limitations
 
+- Trained on specific EVM chains; may not generalize to all EVM variants
 - Max context: 512 tokens (~256 bytes of bytecode)
 - Optimized for EVM bytecode patterns; not intended for natural language
-- Trained only across ethereum, optimism, base, unichain, bsc and bera
 
 ## License