Duplicate from Akahsizrr/Cyber-Prime-1-2.6B

Co-authored-by: Vasko Djack <Akahsizrr@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,37 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+benchmark.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,186 @@
+---
+license: apache-2.0
+language:
+  - en
+library_name: transformers
+tags:
+  - cybersecurity
+  - security
+  - ner
+  - threat-intelligence
+  - phishing
+  - http
+  - fine-tuned
+  - small-language-model
+base_model: LiquidAI/LFM2-2.6B
+model-index:
+  - name: Cyber-Prime-1-2.6B
+    results:
+      - task:
+          type: named-entity-recognition
+          name: CyNER
+        dataset:
+          name: CyNER
+          type: cyberbench
+        metrics:
+          - type: f1
+            value: 0.382
+            name: F1
+      - task:
+          type: named-entity-recognition
+          name: APTNER
+        dataset:
+          name: APTNER
+          type: cyberbench
+        metrics:
+          - type: f1
+            value: 0.413
+            name: F1
+      - task:
+          type: summarization
+          name: CyNews
+        dataset:
+          name: CyNews
+          type: cyberbench
+        metrics:
+          - type: rouge1
+            value: 0.354
+            name: ROUGE-1
+      - task:
+          type: multiple-choice
+          name: SecMMLU
+        dataset:
+          name: SecMMLU
+          type: cyberbench
+        metrics:
+          - type: accuracy
+            value: 0.580
+            name: Accuracy
+      - task:
+          type: multiple-choice
+          name: CyQuiz
+        dataset:
+          name: CyQuiz
+          type: cyberbench
+        metrics:
+          - type: accuracy
+            value: 0.570
+            name: Accuracy
+      - task:
+          type: text-classification
+          name: Email Phishing Detection
+        dataset:
+          name: Email
+          type: cyberbench
+        metrics:
+          - type: f1
+            value: 0.728
+            name: F1
+      - task:
+          type: text-classification
+          name: HTTP Attack Detection
+        dataset:
+          name: HTTP
+          type: cyberbench
+        metrics:
+          - type: f1
+            value: 0.483
+            name: F1
+---
+
+# Cyber-Prime 1 (2.6B)
+
+A fine-tuned cybersecurity specialist built on [LiquidAI/LFM2-2.6B](https://huggingface.co/LiquidAI/LFM2-2.6B). Despite having only 2.6 billion parameters, Cyber-Prime 1 outperforms Llama-2-7B on every CyberBench task and beats GPT-3.5-Turbo on named entity recognition and threat intelligence summarization.
+
+![Cyber-Prime 1 Benchmark Results](benchmark.png)
+
+## Overview
+
+Cyber-Prime 1 is a surgical fine-tune of the LFM2.5-2.6B base model, trained on a curated mix of:
+
+- **NER repair data** — 6,000+ rows fixing JSON format extraction for cybersecurity entities
+- **HTTP reasoning traces** — 5,000 rows with authored chain-of-thought reasoning for attack detection (XSS, SQLi, path traversal, command injection)
+- **Email classification** — 5,000 direct-mode rows for phishing vs. safe classification
+- **CyNews summarization** — 2,000 rows for threat intelligence headline generation
+- **Source data** — 2,000 rows from GHSA, KEV, and ATT&CK sources
+- **Multiple choice** — security knowledge and cyber quiz gold rows
+
+The model uses two distinct modes:
+- **Direct mode** for simple classification (email phishing, HTTP detection) and summarization
+- **Think mode** (chain-of-thought) for tasks benefiting from reasoning (HTTP analysis, NER extraction)
+
+## Benchmark Results
+
+Evaluated on [CyberBench](https://github.com/jpmorganchase/CyberBench) (Liu et al., AAAI-24 AICS Workshop).
+
+| Dataset | Metric | GPT-4 | GPT-3.5 Turbo | Mistral-7B Instruct | Llama-2-7B | **Cyber-Prime 1 (2.6B)** |
+|---------|--------|-------|---------------|---------------------|------------|----------------|
+| CyNER | F1 | 0.554 | 0.334 | 0.323 | 0.263 | **0.382** |
+| APTNER | F1 | 0.500 | 0.409 | 0.262 | 0.280 | **0.413** |
+| CyNews | ROUGE-1 | 0.275 | 0.271 | 0.217 | 0.003 | **0.354** |
+| SecMMLU | Accuracy | 0.830 | 0.780 | 0.720 | 0.630 | **0.580** |
+| CyQuiz | Accuracy | 0.810 | 0.830 | 0.690 | 0.620 | **0.570** |
+| Email | F1 | 0.939 | 0.789 | 0.889 | 0.942 | **0.728** |
+| HTTP | F1 | 0.841 | 0.831 | 0.472 | 0.428 | **0.483** |
+| **Average** | — | **0.721** | **0.609** | **0.511** | **0.451** | **0.501** |
+
+### Key Results
+
+- **Beats Llama-2-7B on all 7 tasks** — a 2.6B model sweeping a 7B model across the board
+- **Beats Mistral-7B-Instruct on 4/7 tasks** — APTNER (+0.151), CyNews (+0.137), CyNER (+0.059), HTTP (+0.011)
+- **Beats GPT-3.5-Turbo on 3/7 tasks** — CyNews (+0.083), APTNER (+0.004), CyNER (+0.048)
+- **Beats GPT-4 on CyNews** — 0.354 vs 0.275 ROUGE-1 (+0.079), a 2.6B model out-summarizing GPT-4
+
+## Usage
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model = AutoModelForCausalLM.from_pretrained(
+    "Akahsizrr/Cyber-Prime-1-2.6B",
+    torch_dtype="auto",
+    device_map="auto",
+)
+tokenizer = AutoTokenizer.from_pretrained("Akahsizrr/Cyber-Prime-1-2.6B")
+
+# NER extraction (Alpaca format)
+prompt = """### Instruction:
+Extract cybersecurity entities from the given text.
+
+### Input:
+APT29 used WELLMAIL to compromise Microsoft Exchange servers via CVE-2021-26855.
+
+### Response:
+"""
+
+inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+output = model.generate(**inputs, max_new_tokens=200, do_sample=False)
+print(tokenizer.decode(output[0], skip_special_tokens=True))
+```
+
+## Limitations
+
+- **Knowledge tasks (SecMMLU, CyQuiz):** Limited by parameter count — a 2.6B model cannot store broad cybersecurity knowledge as well as larger models
+- **Email classification:** Slightly below GPT-3.5-Turbo due to reasoning mode interference from HTTP training data
+- **HTTP detection:** Reasoning improves detection of obvious attacks but may miss subtle injection patterns
+- **Not a security tool:** This model is a research artifact for benchmark evaluation, not a production security system
+
+## Citation
+
+```bibtex
+@misc{cyberprime1,
+  title={Cyber-Prime 1: A Small Cybersecurity Language Model},
+  author={Akahsizrr},
+  year={2025},
+  url={https://huggingface.co/Akahsizrr/Cyber-Prime-1-2.6B}
+}
+```
+
+```bibtex
+@misc{liu2024cyberbench,
+  title={Cyberbench: A multi-task benchmark for evaluating large language models in cybersecurity},
+  author={Liu, Zefang and Shi, Jialei and Buford, John F},
+  howpublished={AAAI-24 Workshop on Artificial Intelligence for Cyber Security (AICS)},
+  year={2024}
+}
+```

chat_template.jinja

@@ -0,0 +1,125 @@
+{{- bos_token -}}
+{%- set preserve_thinking = preserve_thinking | default(false) -%}
+
+{%- macro format_arg_value(arg_value) -%}
+    {%- if arg_value is string -%}
+        {{- "'" + (arg_value | replace("\\", "\\\\") | replace("'", "\\'") | replace("\n", "\\n") | replace("\r", "\\r")) + "'" -}}
+    {%- elif arg_value is mapping or arg_value is iterable -%}
+        {{- arg_value | tojson -}}
+    {%- else -%}
+        {{- arg_value | string -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro parse_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- elif content is mapping -%}
+        {{- content | tojson -}}
+    {%- elif content is iterable -%}
+        {%- set _ns = namespace(result="") -%}
+        {%- for item in content -%}
+            {%- if item is string -%}
+                {%- set _ns.result = _ns.result + item -%}
+            {%- elif item is mapping and item.get("type") == "image" -%}
+                {%- set _ns.result = _ns.result + "<image>" -%}
+            {%- elif item is mapping and item.get("type") == "text" -%}
+                {%- set _ns.result = _ns.result + ((item.get("text") or "") | string) -%}
+            {%- else -%}
+                {%- set _ns.result = _ns.result + (item | tojson) -%}
+            {%- endif -%}
+        {%- endfor -%}
+        {{- _ns.result -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_tool_calls(tool_calls) -%}
+    {%- set tool_calls_ns = namespace(tool_calls=[]) -%}
+    {%- for tool_call in tool_calls -%}
+        {%- set func = tool_call["function"] if "function" in tool_call else tool_call -%}
+        {%- set func_name = func["name"] -%}
+        {%- set func_args = func.get("arguments") -%}
+        {%- set args_ns = namespace(arg_strings=[]) -%}
+        {%- if func_args is mapping -%}
+            {%- for arg_name, arg_value in func_args.items() -%}
+                {%- set args_ns.arg_strings = args_ns.arg_strings + [arg_name + "=" + format_arg_value(arg_value)] -%}
+            {%- endfor -%}
+        {%- elif func_args is string and (func_args | trim) not in ["", "{}", "null"] -%}
+            {{- raise_exception("Tool call arguments must be a mapping, got a JSON-encoded string: parse arguments with json.loads() before applying the chat template") -}}
+        {%- endif -%}
+        {%- set tool_calls_ns.tool_calls = tool_calls_ns.tool_calls + [func_name + "(" + (args_ns.arg_strings | join(", ")) + ")"] -%}
+    {%- endfor -%}
+    {{- "<|tool_call_start|>[" + (tool_calls_ns.tool_calls | join(", ")) + "]<|tool_call_end|>" -}}
+{%- endmacro -%}
+
+{%- set ns = namespace(system_prompt="", last_user_index=-1) -%}
+{%- if messages and messages[0]["role"] == "system" -%}
+    {%- if messages[0].get("content") -%}
+        {%- set ns.system_prompt = parse_content(messages[0]["content"]) -%}
+    {%- endif -%}
+    {%- set messages = messages[1:] -%}
+{%- endif -%}
+{%- if tools -%}
+    {%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "List of tools: [" -%}
+    {%- for tool in tools -%}
+        {%- if tool is not string -%}
+            {%- set tool = tool | tojson -%}
+        {%- endif -%}
+        {%- set ns.system_prompt = ns.system_prompt + tool -%}
+        {%- if not loop.last -%}
+            {%- set ns.system_prompt = ns.system_prompt + ", " -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {%- set ns.system_prompt = ns.system_prompt + "]" -%}
+{%- endif -%}
+{%- if ns.system_prompt -%}
+    {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
+{%- endif -%}
+{%- for message in messages -%}
+    {%- if message["role"] == "user" -%}
+        {%- set ns.last_user_index = loop.index0 -%}
+    {%- endif -%}
+{%- endfor -%}
+{%- for message in messages -%}
+    {{- "<|im_start|>" + message.role + "\n" -}}
+    {%- if message.role == "assistant" -%}
+        {%- generation -%}
+        {%- set keep_thinking = preserve_thinking or loop.index0 > ns.last_user_index -%}
+        {%- set thinking = message.thinking or message.reasoning or message.reasoning_content -%}
+        {%- set thinking = thinking if thinking is string else "" -%}
+        {%- if thinking and keep_thinking -%}
+            {{- "<think>" + thinking + "</think>" -}}
+        {%- endif -%}
+        {%- set _cfm_tag = "CONTINUE_FINAL_MESSAGE_TAG " -%}
+        {%- set _has_cfm = false -%}
+        {%- set content = "" -%}
+        {%- if message.get("content") -%}
+            {%- set content = parse_content(message.content) -%}
+        {%- endif -%}
+        {%- if not keep_thinking and "</think>" in content -%}
+            {%- set content = content.split("</think>")[-1] | trim -%}
+        {%- endif -%}
+        {%- if content.endswith(_cfm_tag) -%}
+            {%- set _has_cfm = true -%}
+            {%- set _trunc_len = (content | length) - (_cfm_tag | length) -%}
+            {%- set content = content[:_trunc_len] -%}
+        {%- endif -%}
+        {{- content -}}
+        {%- if message.tool_calls -%}
+            {{- render_tool_calls(message.tool_calls) -}}
+        {%- endif -%}
+        {%- if _has_cfm -%}
+            {{- _cfm_tag -}}
+        {%- endif -%}
+        {{- "<|im_end|>\n" -}}
+        {%- endgeneration -%}
+    {%- else %}
+        {%- if message.get("content") -%}
+            {{- parse_content(message["content"]) -}}
+        {%- endif -%}
+        {{- "<|im_end|>\n" -}}
+    {%- endif %}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{- "<|im_start|>assistant\n<think>" -}}
+{%- endif -%}

config.json

@@ -0,0 +1,73 @@
+{
+  "architectures": [
+    "Lfm2ForCausalLM"
+  ],
+  "block_auto_adjust_ff_dim": false,
+  "block_dim": 2048,
+  "block_ffn_dim_multiplier": 1.0,
+  "block_mlp_init_scale": 1.0,
+  "block_multiple_of": 256,
+  "block_norm_eps": 1e-05,
+  "block_out_init_scale": 1.0,
+  "block_use_swiglu": true,
+  "block_use_xavier_init": true,
+  "bos_token_id": 124894,
+  "conv_L_cache": 3,
+  "conv_bias": false,
+  "conv_dim": 2048,
+  "conv_use_xavier_init": true,
+  "dtype": "bfloat16",
+  "eos_token_id": 124900,
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 10752,
+  "layer_types": [
+    "conv",
+    "conv",
+    "full_attention",
+    "conv",
+    "conv",
+    "full_attention",
+    "conv",
+    "conv",
+    "conv",
+    "full_attention",
+    "conv",
+    "conv",
+    "conv",
+    "full_attention",
+    "conv",
+    "conv",
+    "conv",
+    "full_attention",
+    "conv",
+    "conv",
+    "conv",
+    "full_attention",
+    "conv",
+    "conv",
+    "full_attention",
+    "conv",
+    "conv",
+    "full_attention",
+    "conv",
+    "conv"
+  ],
+  "max_position_embeddings": 131072,
+  "model_type": "lfm2",
+  "norm_eps": 1e-05,
+  "num_attention_heads": 32,
+  "num_heads": 32,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 8,
+  "pad_token_id": 124893,
+  "rope_parameters": {
+    "rope_theta": 10000000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.2.0",
+  "use_cache": false,
+  "use_pos_enc": true,
+  "vocab_size": 128000
+}

generation_config.json

@@ -0,0 +1,16 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 124894,
+  "do_sample": true,
+  "eos_token_id": [
+    124900
+  ],
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 124893,
+  "repetition_penalty": 1.1,
+  "temperature": 0.1,
+  "top_k": 50,
+  "transformers_version": "5.2.0",
+  "use_cache": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d470990718d31720ee64ea4806d0d5007e17707a833ea988eeaf0784511b3fe0
+size 5394427448

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:695be7802a0e4b8a81048f0ff5ebb7fc811a0ba5a6be63dbb24deb5a81096f41
+size 17905598

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "is_local": true,
+  "legacy": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend",
+  "use_default_system_prompt": false
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d0a27e6886682ee007bd67a9fe8ab7e5e500be7282f54b2e2fba3b0e4ec09ff
+size 5201