Spaces:
Running on Zero
Running on Zero
Upload folder using huggingface_hub (part 7)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +4 -0
- backend/llama.cpp/README.md +604 -0
- backend/llama.cpp/SECURITY.md +97 -0
- backend/llama.cpp/models/templates/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.jinja +204 -0
- backend/llama.cpp/models/templates/NVIDIA-Nemotron-Nano-v2.jinja +162 -0
- backend/llama.cpp/models/templates/NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja +152 -0
- backend/llama.cpp/models/templates/NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja +152 -0
- backend/llama.cpp/models/templates/Qwen-QwQ-32B.jinja +63 -0
- backend/llama.cpp/models/templates/Qwen-Qwen2.5-7B-Instruct.jinja +54 -0
- backend/llama.cpp/models/templates/Qwen-Qwen3-0.6B.jinja +85 -0
- backend/llama.cpp/models/templates/Qwen3-Coder.jinja +117 -0
- backend/llama.cpp/models/templates/Qwen3.5-4B.jinja +154 -0
- backend/llama.cpp/models/templates/README.md +26 -0
- backend/llama.cpp/models/templates/Reka-Edge.jinja +161 -0
- backend/llama.cpp/models/templates/StepFun3.5-Flash.jinja +80 -0
- backend/llama.cpp/models/templates/moonshotai-Kimi-K2.jinja +43 -0
- backend/llama.cpp/models/templates/openai-gpt-oss-120b.jinja +331 -0
- backend/llama.cpp/models/templates/openbmb-MiniCPM5-1B.jinja +179 -0
- backend/llama.cpp/models/templates/unsloth-Apriel-1.5.jinja +126 -0
- backend/llama.cpp/models/templates/unsloth-mistral-Devstral-Small-2507.jinja +105 -0
- backend/llama.cpp/models/templates/upstage-Solar-Open-100B.jinja +156 -0
- backend/llama.cpp/mypy.ini +7 -0
- backend/llama.cpp/pocs/CMakeLists.txt +14 -0
- backend/llama.cpp/pocs/vdot/CMakeLists.txt +9 -0
- backend/llama.cpp/pocs/vdot/q8dot.cpp +173 -0
- backend/llama.cpp/pocs/vdot/vdot.cpp +311 -0
- backend/llama.cpp/pyproject.toml +67 -0
- backend/llama.cpp/pyrightconfig.json +22 -0
- backend/llama.cpp/requirements.txt +13 -0
- backend/llama.cpp/requirements/requirements-all.txt +18 -0
- backend/llama.cpp/requirements/requirements-compare-llama-bench.txt +3 -0
- backend/llama.cpp/requirements/requirements-convert_hf_to_gguf.txt +9 -0
- backend/llama.cpp/requirements/requirements-convert_hf_to_gguf_update.txt +1 -0
- backend/llama.cpp/requirements/requirements-convert_legacy_llama.txt +7 -0
- backend/llama.cpp/requirements/requirements-convert_llama_ggml_to_gguf.txt +1 -0
- backend/llama.cpp/requirements/requirements-convert_lora_to_gguf.txt +4 -0
- backend/llama.cpp/requirements/requirements-gguf_editor_gui.txt +3 -0
- backend/llama.cpp/requirements/requirements-pydantic.txt +3 -0
- backend/llama.cpp/requirements/requirements-server-bench.txt +5 -0
- backend/llama.cpp/requirements/requirements-test-tokenizer-random.txt +1 -0
- backend/llama.cpp/requirements/requirements-tool_bench.txt +11 -0
- backend/llama.cpp/scripts/apple/validate-apps.sh +5 -0
- backend/llama.cpp/scripts/apple/validate-ios.sh +820 -0
- backend/llama.cpp/scripts/apple/validate-macos.sh +781 -0
- backend/llama.cpp/scripts/apple/validate-tvos.sh +813 -0
- backend/llama.cpp/scripts/apple/validate-visionos.sh +811 -0
- backend/llama.cpp/scripts/bench-models.sh +82 -0
- backend/llama.cpp/scripts/build-info.sh +30 -0
- backend/llama.cpp/scripts/check-requirements.sh +179 -0
- backend/llama.cpp/scripts/compare-commits.sh +66 -0
.gitattributes
CHANGED
|
@@ -61,3 +61,7 @@ backend/llama.cpp/models/ggml-vocab-qwen2.gguf filter=lfs diff=lfs merge=lfs -te
|
|
| 61 |
backend/llama.cpp/models/ggml-vocab-qwen35.gguf filter=lfs diff=lfs merge=lfs -text
|
| 62 |
backend/llama.cpp/models/ggml-vocab-refact.gguf filter=lfs diff=lfs merge=lfs -text
|
| 63 |
backend/llama.cpp/models/ggml-vocab-starcoder.gguf filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
backend/llama.cpp/models/ggml-vocab-qwen35.gguf filter=lfs diff=lfs merge=lfs -text
|
| 62 |
backend/llama.cpp/models/ggml-vocab-refact.gguf filter=lfs diff=lfs merge=lfs -text
|
| 63 |
backend/llama.cpp/models/ggml-vocab-starcoder.gguf filter=lfs diff=lfs merge=lfs -text
|
| 64 |
+
backend/llama.cpp/tools/mtmd/test-1.jpeg filter=lfs diff=lfs merge=lfs -text
|
| 65 |
+
backend/llama.cpp/tools/mtmd/test-2.mp3 filter=lfs diff=lfs merge=lfs -text
|
| 66 |
+
backend/llama.cpp/tools/mtmd/test-3.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 67 |
+
backend/llama.cpp/tools/mtmd/tests/test-1-positive.png filter=lfs diff=lfs merge=lfs -text
|
backend/llama.cpp/README.md
ADDED
|
@@ -0,0 +1,604 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# llama.cpp
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
[](https://opensource.org/licenses/MIT)
|
| 6 |
+
[](https://github.com/ggml-org/llama.cpp/releases)
|
| 7 |
+
[](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml)
|
| 8 |
+
[](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
|
| 9 |
+
[](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
|
| 10 |
+
|
| 11 |
+
[Manifesto](https://github.com/ggml-org/llama.cpp/discussions/205) / [ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md)
|
| 12 |
+
|
| 13 |
+
LLM inference in C/C++
|
| 14 |
+
|
| 15 |
+
## Recent API changes
|
| 16 |
+
|
| 17 |
+
- [Changelog for `libllama` API](https://github.com/ggml-org/llama.cpp/issues/9289)
|
| 18 |
+
- [Changelog for `llama-server` REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
|
| 19 |
+
|
| 20 |
+
## Hot topics
|
| 21 |
+
|
| 22 |
+
- **Hugging Face cache migration: models downloaded with `-hf` are now stored in the standard Hugging Face cache directory, enabling sharing with other HF tools.**
|
| 23 |
+
- **[guide : using the new WebUI of llama.cpp](https://github.com/ggml-org/llama.cpp/discussions/16938)**
|
| 24 |
+
- [guide : running gpt-oss with llama.cpp](https://github.com/ggml-org/llama.cpp/discussions/15396)
|
| 25 |
+
- [[FEEDBACK] Better packaging for llama.cpp to support downstream consumers 🤗](https://github.com/ggml-org/llama.cpp/discussions/15313)
|
| 26 |
+
- Support for the `gpt-oss` model with native MXFP4 format has been added | [PR](https://github.com/ggml-org/llama.cpp/pull/15091) | [Collaboration with NVIDIA](https://blogs.nvidia.com/blog/rtx-ai-garage-openai-oss) | [Comment](https://github.com/ggml-org/llama.cpp/discussions/15095)
|
| 27 |
+
- Multimodal support arrived in `llama-server`: [#12898](https://github.com/ggml-org/llama.cpp/pull/12898) | [documentation](./docs/multimodal.md)
|
| 28 |
+
- VS Code extension for FIM completions: https://github.com/ggml-org/llama.vscode
|
| 29 |
+
- Vim/Neovim plugin for FIM completions: https://github.com/ggml-org/llama.vim
|
| 30 |
+
- Hugging Face Inference Endpoints now support GGUF out of the box! https://github.com/ggml-org/llama.cpp/discussions/9669
|
| 31 |
+
- Hugging Face GGUF editor: [discussion](https://github.com/ggml-org/llama.cpp/discussions/9268) | [tool](https://huggingface.co/spaces/CISCai/gguf-editor)
|
| 32 |
+
- WebGPU support is now available in the browser, see a blog/demo introducing it [here](https://reeselevine.github.io/llamas-on-the-web/).
|
| 33 |
+
|
| 34 |
+
----
|
| 35 |
+
|
| 36 |
+
## Quick start
|
| 37 |
+
|
| 38 |
+
Getting started with llama.cpp is straightforward. Here are several ways to install it on your machine:
|
| 39 |
+
|
| 40 |
+
- Install `llama.cpp` using [brew, nix, winget, or conda-forge](docs/install.md)
|
| 41 |
+
- Run with Docker - see our [Docker documentation](docs/docker.md)
|
| 42 |
+
- Download pre-built binaries from the [releases page](https://github.com/ggml-org/llama.cpp/releases)
|
| 43 |
+
- Build from source by cloning this repository - check out [our build guide](docs/build.md)
|
| 44 |
+
|
| 45 |
+
Once installed, you'll need a model to work with. Head to the [Obtaining and quantizing models](#obtaining-and-quantizing-models) section to learn more.
|
| 46 |
+
|
| 47 |
+
Example command:
|
| 48 |
+
|
| 49 |
+
```sh
|
| 50 |
+
# Use a local model file
|
| 51 |
+
llama-cli -m my_model.gguf
|
| 52 |
+
|
| 53 |
+
# Or download and run a model directly from Hugging Face
|
| 54 |
+
llama-cli -hf ggml-org/gemma-3-1b-it-GGUF
|
| 55 |
+
|
| 56 |
+
# Launch OpenAI-compatible API server
|
| 57 |
+
llama-server -hf ggml-org/gemma-3-1b-it-GGUF
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
## Description
|
| 61 |
+
|
| 62 |
+
The main goal of `llama.cpp` is to enable LLM inference with minimal setup and state-of-the-art performance on a wide
|
| 63 |
+
range of hardware - locally and in the cloud.
|
| 64 |
+
|
| 65 |
+
- Plain C/C++ implementation without any dependencies
|
| 66 |
+
- Apple silicon is a first-class citizen - optimized via ARM NEON, Accelerate and Metal frameworks
|
| 67 |
+
- AVX, AVX2, AVX512 and AMX support for x86 architectures
|
| 68 |
+
- RVV, ZVFH, ZFH, ZICBOP and ZIHINTPAUSE support for RISC-V architectures
|
| 69 |
+
- 1.5-bit, 2-bit, 3-bit, 4-bit, 5-bit, 6-bit, and 8-bit integer quantization for faster inference and reduced memory use
|
| 70 |
+
- Custom CUDA kernels for running LLMs on NVIDIA GPUs (support for AMD GPUs via HIP and Moore Threads GPUs via MUSA)
|
| 71 |
+
- Vulkan and SYCL backend support
|
| 72 |
+
- CPU+GPU hybrid inference to partially accelerate models larger than the total VRAM capacity
|
| 73 |
+
|
| 74 |
+
The `llama.cpp` project is the main playground for developing new features for the [ggml](https://github.com/ggml-org/ggml) library.
|
| 75 |
+
|
| 76 |
+
<details>
|
| 77 |
+
<summary>Models</summary>
|
| 78 |
+
|
| 79 |
+
Typically finetunes of the base models below are supported as well.
|
| 80 |
+
|
| 81 |
+
Instructions for adding support for new models: [HOWTO-add-model.md](docs/development/HOWTO-add-model.md)
|
| 82 |
+
|
| 83 |
+
#### Text-only
|
| 84 |
+
|
| 85 |
+
- [X] LLaMA 🦙
|
| 86 |
+
- [x] LLaMA 2 🦙🦙
|
| 87 |
+
- [x] LLaMA 3 🦙🦙🦙
|
| 88 |
+
- [X] [Mistral 7B](https://huggingface.co/mistralai/Mistral-7B-v0.1)
|
| 89 |
+
- [x] [Mixtral MoE](https://huggingface.co/models?search=mistral-ai/Mixtral)
|
| 90 |
+
- [x] [DBRX](https://huggingface.co/databricks/dbrx-instruct)
|
| 91 |
+
- [x] [Jamba](https://huggingface.co/ai21labs)
|
| 92 |
+
- [X] [Falcon](https://huggingface.co/models?search=tiiuae/falcon)
|
| 93 |
+
- [X] [Chinese LLaMA / Alpaca](https://github.com/ymcui/Chinese-LLaMA-Alpaca) and [Chinese LLaMA-2 / Alpaca-2](https://github.com/ymcui/Chinese-LLaMA-Alpaca-2)
|
| 94 |
+
- [X] [Vigogne (French)](https://github.com/bofenghuang/vigogne)
|
| 95 |
+
- [X] [BERT](https://github.com/ggml-org/llama.cpp/pull/5423)
|
| 96 |
+
- [X] [Koala](https://bair.berkeley.edu/blog/2023/04/03/koala/)
|
| 97 |
+
- [X] [Baichuan 1 & 2](https://huggingface.co/models?search=baichuan-inc/Baichuan) + [derivations](https://huggingface.co/hiyouga/baichuan-7b-sft)
|
| 98 |
+
- [X] [Aquila 1 & 2](https://huggingface.co/models?search=BAAI/Aquila)
|
| 99 |
+
- [X] [Starcoder models](https://github.com/ggml-org/llama.cpp/pull/3187)
|
| 100 |
+
- [X] [Refact](https://huggingface.co/smallcloudai/Refact-1_6B-fim)
|
| 101 |
+
- [X] [MPT](https://github.com/ggml-org/llama.cpp/pull/3417)
|
| 102 |
+
- [X] [Bloom](https://github.com/ggml-org/llama.cpp/pull/3553)
|
| 103 |
+
- [x] [Yi models](https://huggingface.co/models?search=01-ai/Yi)
|
| 104 |
+
- [X] [StableLM models](https://huggingface.co/stabilityai)
|
| 105 |
+
- [x] [Deepseek models](https://huggingface.co/models?search=deepseek-ai/deepseek)
|
| 106 |
+
- [x] [Qwen models](https://huggingface.co/models?search=Qwen/Qwen)
|
| 107 |
+
- [x] [PLaMo-13B](https://github.com/ggml-org/llama.cpp/pull/3557)
|
| 108 |
+
- [x] [Phi models](https://huggingface.co/models?search=microsoft/phi)
|
| 109 |
+
- [x] [PhiMoE](https://github.com/ggml-org/llama.cpp/pull/11003)
|
| 110 |
+
- [x] [GPT-2](https://huggingface.co/gpt2)
|
| 111 |
+
- [x] [Orion 14B](https://github.com/ggml-org/llama.cpp/pull/5118)
|
| 112 |
+
- [x] [InternLM2](https://huggingface.co/models?search=internlm2)
|
| 113 |
+
- [x] [CodeShell](https://github.com/WisdomShell/codeshell)
|
| 114 |
+
- [x] [Gemma](https://ai.google.dev/gemma)
|
| 115 |
+
- [x] [Mamba](https://github.com/state-spaces/mamba)
|
| 116 |
+
- [x] [Grok-1](https://huggingface.co/keyfan/grok-1-hf)
|
| 117 |
+
- [x] [Xverse](https://huggingface.co/models?search=xverse)
|
| 118 |
+
- [x] [Command-R models](https://huggingface.co/models?search=CohereForAI/c4ai-command-r)
|
| 119 |
+
- [x] [SEA-LION](https://huggingface.co/models?search=sea-lion)
|
| 120 |
+
- [x] [GritLM-7B](https://huggingface.co/GritLM/GritLM-7B) + [GritLM-8x7B](https://huggingface.co/GritLM/GritLM-8x7B)
|
| 121 |
+
- [x] [OLMo](https://allenai.org/olmo)
|
| 122 |
+
- [x] [OLMo 2](https://allenai.org/olmo)
|
| 123 |
+
- [x] [OLMoE](https://huggingface.co/allenai/OLMoE-1B-7B-0924)
|
| 124 |
+
- [x] [Granite models](https://huggingface.co/collections/ibm-granite/granite-code-models-6624c5cec322e4c148c8b330)
|
| 125 |
+
- [x] [GPT-NeoX](https://github.com/EleutherAI/gpt-neox) + [Pythia](https://github.com/EleutherAI/pythia)
|
| 126 |
+
- [x] [Snowflake-Arctic MoE](https://huggingface.co/collections/Snowflake/arctic-66290090abe542894a5ac520)
|
| 127 |
+
- [x] [Smaug](https://huggingface.co/models?search=Smaug)
|
| 128 |
+
- [x] [Poro 34B](https://huggingface.co/LumiOpen/Poro-34B)
|
| 129 |
+
- [x] [Bitnet b1.58 models](https://huggingface.co/1bitLLM)
|
| 130 |
+
- [x] [Flan T5](https://huggingface.co/models?search=flan-t5)
|
| 131 |
+
- [x] [Open Elm models](https://huggingface.co/collections/apple/openelm-instruct-models-6619ad295d7ae9f868b759ca)
|
| 132 |
+
- [x] [ChatGLM3-6b](https://huggingface.co/THUDM/chatglm3-6b) + [ChatGLM4-9b](https://huggingface.co/THUDM/glm-4-9b) + [GLMEdge-1.5b](https://huggingface.co/THUDM/glm-edge-1.5b-chat) + [GLMEdge-4b](https://huggingface.co/THUDM/glm-edge-4b-chat)
|
| 133 |
+
- [x] [GLM-4-0414](https://huggingface.co/collections/THUDM/glm-4-0414-67f3cbcb34dd9d252707cb2e)
|
| 134 |
+
- [x] [SmolLM](https://huggingface.co/collections/HuggingFaceTB/smollm-6695016cad7167254ce15966)
|
| 135 |
+
- [x] [EXAONE-3.0-7.8B-Instruct](https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct)
|
| 136 |
+
- [x] [FalconMamba Models](https://huggingface.co/collections/tiiuae/falconmamba-7b-66b9a580324dd1598b0f6d4a)
|
| 137 |
+
- [x] [Jais](https://huggingface.co/inceptionai/jais-13b-chat)
|
| 138 |
+
- [x] [Bielik-11B-v2.3](https://huggingface.co/collections/speakleash/bielik-11b-v23-66ee813238d9b526a072408a)
|
| 139 |
+
- [x] [RWKV-7](https://huggingface.co/collections/shoumenchougou/rwkv7-gxx-gguf)
|
| 140 |
+
- [x] [RWKV-6](https://github.com/BlinkDL/RWKV-LM)
|
| 141 |
+
- [x] [QRWKV-6](https://huggingface.co/recursal/QRWKV6-32B-Instruct-Preview-v0.1)
|
| 142 |
+
- [x] [GigaChat-20B-A3B](https://huggingface.co/ai-sage/GigaChat-20B-A3B-instruct)
|
| 143 |
+
- [X] [Trillion-7B-preview](https://huggingface.co/trillionlabs/Trillion-7B-preview)
|
| 144 |
+
- [x] [Ling models](https://huggingface.co/collections/inclusionAI/ling-67c51c85b34a7ea0aba94c32)
|
| 145 |
+
- [x] [Liquid LFM2 models](https://huggingface.co/collections/LiquidAI/lfm2)
|
| 146 |
+
- [x] [Liquid LFM2.5 models](https://huggingface.co/collections/LiquidAI/lfm25)
|
| 147 |
+
- [x] [Liquid Nanos](https://huggingface.co/collections/LiquidAI/liquid-nanos)
|
| 148 |
+
- [x] [Hunyuan models](https://huggingface.co/collections/tencent/hunyuan-dense-model-6890632cda26b19119c9c5e7)
|
| 149 |
+
- [x] [BailingMoeV2 (Ring/Ling 2.0) models](https://huggingface.co/collections/inclusionAI/ling-v2-68bf1dd2fc34c306c1fa6f86)
|
| 150 |
+
- [x] [Mellum models](https://huggingface.co/JetBrains/models?search=mellum)
|
| 151 |
+
|
| 152 |
+
#### Multimodal
|
| 153 |
+
|
| 154 |
+
- [x] [LLaVA 1.5 models](https://huggingface.co/collections/liuhaotian/llava-15-653aac15d994e992e2677a7e), [LLaVA 1.6 models](https://huggingface.co/collections/liuhaotian/llava-16-65b9e40155f60fd046a5ccf2)
|
| 155 |
+
- [x] [BakLLaVA](https://huggingface.co/models?search=SkunkworksAI/Bakllava)
|
| 156 |
+
- [x] [Obsidian](https://huggingface.co/NousResearch/Obsidian-3B-V0.5)
|
| 157 |
+
- [x] [ShareGPT4V](https://huggingface.co/models?search=Lin-Chen/ShareGPT4V)
|
| 158 |
+
- [x] [MobileVLM 1.7B/3B models](https://huggingface.co/models?search=mobileVLM)
|
| 159 |
+
- [x] [Yi-VL](https://huggingface.co/models?search=Yi-VL)
|
| 160 |
+
- [x] [Mini CPM](https://huggingface.co/models?search=MiniCPM)
|
| 161 |
+
- [x] [Moondream](https://huggingface.co/vikhyatk/moondream2)
|
| 162 |
+
- [x] [Bunny](https://github.com/BAAI-DCAI/Bunny)
|
| 163 |
+
- [x] [GLM-EDGE](https://huggingface.co/models?search=glm-edge)
|
| 164 |
+
- [x] [Qwen2-VL](https://huggingface.co/collections/Qwen/qwen2-vl-66cee7455501d7126940800d)
|
| 165 |
+
- [x] [LFM2-VL](https://huggingface.co/collections/LiquidAI/lfm2-vl-68963bbc84a610f7638d5ffa)
|
| 166 |
+
|
| 167 |
+
</details>
|
| 168 |
+
|
| 169 |
+
<details>
|
| 170 |
+
<summary>Bindings</summary>
|
| 171 |
+
|
| 172 |
+
- Python: [ddh0/easy-llama](https://github.com/ddh0/easy-llama)
|
| 173 |
+
- Python: [abetlen/llama-cpp-python](https://github.com/abetlen/llama-cpp-python)
|
| 174 |
+
- Go: [go-skynet/go-llama.cpp](https://github.com/go-skynet/go-llama.cpp)
|
| 175 |
+
- Node.js: [withcatai/node-llama-cpp](https://github.com/withcatai/node-llama-cpp)
|
| 176 |
+
- JS/TS (llama.cpp server client): [lgrammel/modelfusion](https://modelfusion.dev/integration/model-provider/llamacpp)
|
| 177 |
+
- JS/TS (Programmable Prompt Engine CLI): [offline-ai/cli](https://github.com/offline-ai/cli)
|
| 178 |
+
- JavaScript/Wasm (works in browser): [tangledgroup/llama-cpp-wasm](https://github.com/tangledgroup/llama-cpp-wasm)
|
| 179 |
+
- Typescript/Wasm (nicer API, available on npm): [ngxson/wllama](https://github.com/ngxson/wllama)
|
| 180 |
+
- Ruby: [yoshoku/llama_cpp.rb](https://github.com/yoshoku/llama_cpp.rb)
|
| 181 |
+
- Ruby: [docusealco/rllama](https://github.com/docusealco/rllama)
|
| 182 |
+
- Rust (more features): [edgenai/llama_cpp-rs](https://github.com/edgenai/llama_cpp-rs)
|
| 183 |
+
- Rust (nicer API): [mdrokz/rust-llama.cpp](https://github.com/mdrokz/rust-llama.cpp)
|
| 184 |
+
- Rust (more direct bindings): [utilityai/llama-cpp-rs](https://github.com/utilityai/llama-cpp-rs)
|
| 185 |
+
- Rust (automated build from crates.io): [ShelbyJenkins/llm_client](https://github.com/ShelbyJenkins/llm_client)
|
| 186 |
+
- C#/.NET: [SciSharp/LLamaSharp](https://github.com/SciSharp/LLamaSharp)
|
| 187 |
+
- C#/VB.NET (more features - community license): [LM-Kit.NET](https://docs.lm-kit.com/lm-kit-net/index.html)
|
| 188 |
+
- Scala 3: [donderom/llm4s](https://github.com/donderom/llm4s)
|
| 189 |
+
- Clojure: [phronmophobic/llama.clj](https://github.com/phronmophobic/llama.clj)
|
| 190 |
+
- React Native: [mybigday/llama.rn](https://github.com/mybigday/llama.rn)
|
| 191 |
+
- Java: [kherud/java-llama.cpp](https://github.com/kherud/java-llama.cpp)
|
| 192 |
+
- Java: [QuasarByte/llama-cpp-jna](https://github.com/QuasarByte/llama-cpp-jna)
|
| 193 |
+
- Zig: [deins/llama.cpp.zig](https://github.com/Deins/llama.cpp.zig)
|
| 194 |
+
- Flutter/Dart: [netdur/llama_cpp_dart](https://github.com/netdur/llama_cpp_dart)
|
| 195 |
+
- Flutter: [xuegao-tzx/Fllama](https://github.com/xuegao-tzx/Fllama)
|
| 196 |
+
- PHP (API bindings and features built on top of llama.cpp): [distantmagic/resonance](https://github.com/distantmagic/resonance) [(more info)](https://github.com/ggml-org/llama.cpp/pull/6326)
|
| 197 |
+
- Guile Scheme: [guile_llama_cpp](https://savannah.nongnu.org/projects/guile-llama-cpp)
|
| 198 |
+
- Swift [srgtuszy/llama-cpp-swift](https://github.com/srgtuszy/llama-cpp-swift)
|
| 199 |
+
- Swift [ShenghaiWang/SwiftLlama](https://github.com/ShenghaiWang/SwiftLlama)
|
| 200 |
+
- Delphi [Embarcadero/llama-cpp-delphi](https://github.com/Embarcadero/llama-cpp-delphi)
|
| 201 |
+
- Go (no CGo needed): [hybridgroup/yzma](https://github.com/hybridgroup/yzma)
|
| 202 |
+
- Android: [llama.android](/examples/llama.android)
|
| 203 |
+
|
| 204 |
+
</details>
|
| 205 |
+
|
| 206 |
+
<details>
|
| 207 |
+
<summary>UIs</summary>
|
| 208 |
+
|
| 209 |
+
*(to have a project listed here, it should clearly state that it depends on `llama.cpp`)*
|
| 210 |
+
|
| 211 |
+
- [AI Sublime Text plugin](https://github.com/yaroslavyaroslav/OpenAI-sublime-text) (MIT)
|
| 212 |
+
- [BonzAI App](https://apps.apple.com/us/app/bonzai-your-local-ai-agent/id6752847988) (proprietary)
|
| 213 |
+
- [cztomsik/ava](https://github.com/cztomsik/ava) (MIT)
|
| 214 |
+
- [Dot](https://github.com/alexpinel/Dot) (GPL)
|
| 215 |
+
- [eva](https://github.com/ylsdamxssjxxdd/eva) (MIT)
|
| 216 |
+
- [iohub/collama](https://github.com/iohub/coLLaMA) (Apache-2.0)
|
| 217 |
+
- [janhq/jan](https://github.com/janhq/jan) (AGPL)
|
| 218 |
+
- [johnbean393/Sidekick](https://github.com/johnbean393/Sidekick) (MIT)
|
| 219 |
+
- [KanTV](https://github.com/zhouwg/kantv?tab=readme-ov-file) (Apache-2.0)
|
| 220 |
+
- [KodiBot](https://github.com/firatkiral/kodibot) (GPL)
|
| 221 |
+
- [llama.vim](https://github.com/ggml-org/llama.vim) (MIT)
|
| 222 |
+
- [LARS](https://github.com/abgulati/LARS) (AGPL)
|
| 223 |
+
- [Llama Assistant](https://github.com/vietanhdev/llama-assistant) (GPL)
|
| 224 |
+
- [LlamaLib](https://github.com/undreamai/LlamaLib) (Apache-2.0)
|
| 225 |
+
- [LLMFarm](https://github.com/guinmoon/LLMFarm?tab=readme-ov-file) (MIT)
|
| 226 |
+
- [LLMUnity](https://github.com/undreamai/LLMUnity) (MIT)
|
| 227 |
+
- [LMStudio](https://lmstudio.ai/) (proprietary)
|
| 228 |
+
- [LocalAI](https://github.com/mudler/LocalAI) (MIT)
|
| 229 |
+
- [LostRuins/koboldcpp](https://github.com/LostRuins/koboldcpp) (AGPL)
|
| 230 |
+
- [MindMac](https://mindmac.app) (proprietary)
|
| 231 |
+
- [MindWorkAI/AI-Studio](https://github.com/MindWorkAI/AI-Studio) (FSL-1.1-MIT)
|
| 232 |
+
- [Mobile-Artificial-Intelligence/maid](https://github.com/Mobile-Artificial-Intelligence/maid) (MIT)
|
| 233 |
+
- [Mozilla-Ocho/llamafile](https://github.com/Mozilla-Ocho/llamafile) (Apache-2.0)
|
| 234 |
+
- [nat/openplayground](https://github.com/nat/openplayground) (MIT)
|
| 235 |
+
- [nomic-ai/gpt4all](https://github.com/nomic-ai/gpt4all) (MIT)
|
| 236 |
+
- [ollama/ollama](https://github.com/ollama/ollama) (MIT)
|
| 237 |
+
- [oobabooga/text-generation-webui](https://github.com/oobabooga/text-generation-webui) (AGPL)
|
| 238 |
+
- [PocketPal AI](https://github.com/a-ghorbani/pocketpal-ai) (MIT)
|
| 239 |
+
- [psugihara/FreeChat](https://github.com/psugihara/FreeChat) (MIT)
|
| 240 |
+
- [ptsochantaris/emeltal](https://github.com/ptsochantaris/emeltal) (MIT)
|
| 241 |
+
- [pythops/tenere](https://github.com/pythops/tenere) (AGPL)
|
| 242 |
+
- [ramalama](https://github.com/containers/ramalama) (MIT)
|
| 243 |
+
- [semperai/amica](https://github.com/semperai/amica) (MIT)
|
| 244 |
+
- [withcatai/catai](https://github.com/withcatai/catai) (MIT)
|
| 245 |
+
- [Autopen](https://github.com/blackhole89/autopen) (GPL)
|
| 246 |
+
|
| 247 |
+
</details>
|
| 248 |
+
|
| 249 |
+
<details>
|
| 250 |
+
<summary>Tools</summary>
|
| 251 |
+
|
| 252 |
+
- [akx/ggify](https://github.com/akx/ggify) – download PyTorch models from Hugging Face Hub and convert them to GGML
|
| 253 |
+
- [akx/ollama-dl](https://github.com/akx/ollama-dl) – download models from the Ollama library to be used directly with llama.cpp
|
| 254 |
+
- [crashr/gppm](https://github.com/crashr/gppm) – launch llama.cpp instances utilizing NVIDIA Tesla P40 or P100 GPUs with reduced idle power consumption
|
| 255 |
+
- [gpustack/gguf-parser](https://github.com/gpustack/gguf-parser-go/tree/main/cmd/gguf-parser) - review/check the GGUF file and estimate the memory usage
|
| 256 |
+
- [Styled Lines](https://marketplace.unity.com/packages/tools/generative-ai/styled-lines-llama-cpp-model-292902) (proprietary licensed, async wrapper of inference part for game development in Unity3d with pre-built Mobile and Web platform wrappers and a model example)
|
| 257 |
+
- [unslothai/unsloth](https://github.com/unslothai/unsloth) – 🦥 exports/saves fine-tuned and trained models to GGUF (Apache-2.0)
|
| 258 |
+
|
| 259 |
+
</details>
|
| 260 |
+
|
| 261 |
+
<details>
|
| 262 |
+
<summary>Infrastructure</summary>
|
| 263 |
+
|
| 264 |
+
- [Paddler](https://github.com/intentee/paddler) - Open-source LLMOps platform for hosting and scaling AI in your own infrastructure
|
| 265 |
+
- [GPUStack](https://github.com/gpustack/gpustack) - Manage GPU clusters for running LLMs
|
| 266 |
+
- [llama_cpp_canister](https://github.com/onicai/llama_cpp_canister) - llama.cpp as a smart contract on the Internet Computer, using WebAssembly
|
| 267 |
+
- [llama-swap](https://github.com/mostlygeek/llama-swap) - transparent proxy that adds automatic model switching with llama-server
|
| 268 |
+
- [Kalavai](https://github.com/kalavai-net/kalavai-client) - Crowdsource end to end LLM deployment at any scale
|
| 269 |
+
- [llmaz](https://github.com/InftyAI/llmaz) - ☸️ Easy, advanced inference platform for large language models on Kubernetes.
|
| 270 |
+
- [LLMKube](https://github.com/defilantech/llmkube) - Kubernetes operator for llama.cpp with multi-GPU and Apple Silicon Metal
|
| 271 |
+
support"
|
| 272 |
+
</details>
|
| 273 |
+
|
| 274 |
+
<details>
|
| 275 |
+
<summary>Games</summary>
|
| 276 |
+
|
| 277 |
+
- [Lucy's Labyrinth](https://github.com/MorganRO8/Lucys_Labyrinth) - A simple maze game where agents controlled by an AI model will try to trick you.
|
| 278 |
+
|
| 279 |
+
</details>
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
## Supported backends
|
| 283 |
+
|
| 284 |
+
| Backend | Target devices |
|
| 285 |
+
| --- | --- |
|
| 286 |
+
| [Metal](docs/build.md#metal-build) | Apple Silicon |
|
| 287 |
+
| [BLAS](docs/build.md#blas-build) | All |
|
| 288 |
+
| [BLIS](docs/backend/BLIS.md) | All |
|
| 289 |
+
| [SYCL](docs/backend/SYCL.md) | Intel GPU |
|
| 290 |
+
| [OpenVINO [In Progress]](docs/backend/OPENVINO.md) | Intel CPUs, GPUs, and NPUs |
|
| 291 |
+
| [MUSA](docs/build.md#musa) | Moore Threads GPU |
|
| 292 |
+
| [CUDA](docs/build.md#cuda) | Nvidia GPU |
|
| 293 |
+
| [HIP](docs/build.md#hip) | AMD GPU |
|
| 294 |
+
| [ZenDNN](docs/build.md#zendnn) | AMD CPU |
|
| 295 |
+
| [Vulkan](docs/build.md#vulkan) | GPU |
|
| 296 |
+
| [CANN](docs/build.md#cann) | Ascend NPU |
|
| 297 |
+
| [OpenCL](docs/backend/OPENCL.md) | Adreno GPU |
|
| 298 |
+
| [IBM zDNN](docs/backend/zDNN.md) | IBM Z & LinuxONE |
|
| 299 |
+
| [WebGPU](docs/build.md#webgpu) | All |
|
| 300 |
+
| [RPC](https://github.com/ggml-org/llama.cpp/tree/master/tools/rpc) | All |
|
| 301 |
+
| [Hexagon [In Progress]](docs/backend/snapdragon/README.md) | Snapdragon |
|
| 302 |
+
| [VirtGPU](docs/backend/VirtGPU.md) | VirtGPU APIR |
|
| 303 |
+
|
| 304 |
+
## Obtaining and quantizing models
|
| 305 |
+
|
| 306 |
+
The [Hugging Face](https://huggingface.co) platform hosts a [number of LLMs](https://huggingface.co/models?library=gguf&sort=trending) compatible with `llama.cpp`:
|
| 307 |
+
|
| 308 |
+
- [Trending](https://huggingface.co/models?library=gguf&sort=trending)
|
| 309 |
+
- [LLaMA](https://huggingface.co/models?sort=trending&search=llama+gguf)
|
| 310 |
+
|
| 311 |
+
You can either manually download the GGUF file or directly use any `llama.cpp`-compatible models from [Hugging Face](https://huggingface.co/) or other model hosting sites, by using this CLI argument: `-hf <user>/<model>[:quant]`. For example:
|
| 312 |
+
|
| 313 |
+
```sh
|
| 314 |
+
llama-cli -hf ggml-org/gemma-3-1b-it-GGUF
|
| 315 |
+
```
|
| 316 |
+
|
| 317 |
+
By default, the CLI would download from Hugging Face, you can switch to other options with the environment variable `MODEL_ENDPOINT`. The `MODEL_ENDPOINT` must point to a Hugging Face compatible API endpoint.
|
| 318 |
+
|
| 319 |
+
After downloading a model, use the CLI tools to run it locally - see below.
|
| 320 |
+
|
| 321 |
+
`llama.cpp` requires the model to be stored in the [GGUF](https://github.com/ggml-org/ggml/blob/master/docs/gguf.md) file format. Models in other data formats can be converted to GGUF using the `convert_*.py` Python scripts in this repo.
|
| 322 |
+
|
| 323 |
+
The Hugging Face platform provides a variety of online tools for converting, quantizing and hosting models with `llama.cpp`:
|
| 324 |
+
|
| 325 |
+
- Use the [GGUF-my-repo space](https://huggingface.co/spaces/ggml-org/gguf-my-repo) to convert to GGUF format and quantize model weights to smaller sizes
|
| 326 |
+
- Use the [GGUF-my-LoRA space](https://huggingface.co/spaces/ggml-org/gguf-my-lora) to convert LoRA adapters to GGUF format (more info: https://github.com/ggml-org/llama.cpp/discussions/10123)
|
| 327 |
+
- Use the [GGUF-editor space](https://huggingface.co/spaces/CISCai/gguf-editor) to edit GGUF meta data in the browser (more info: https://github.com/ggml-org/llama.cpp/discussions/9268)
|
| 328 |
+
- Use the [Inference Endpoints](https://ui.endpoints.huggingface.co/) to directly host `llama.cpp` in the cloud (more info: https://github.com/ggml-org/llama.cpp/discussions/9669)
|
| 329 |
+
|
| 330 |
+
To learn more about model quantization, [read this documentation](tools/quantize/README.md)
|
| 331 |
+
|
| 332 |
+
## [`llama-cli`](tools/cli)
|
| 333 |
+
|
| 334 |
+
#### A CLI tool for accessing and experimenting with most of `llama.cpp`'s functionality.
|
| 335 |
+
|
| 336 |
+
- <details open>
|
| 337 |
+
<summary>Run in conversation mode</summary>
|
| 338 |
+
|
| 339 |
+
Models with a built-in chat template will automatically activate conversation mode. If this doesn't occur, you can manually enable it by adding `-cnv` and specifying a suitable chat template with `--chat-template NAME`
|
| 340 |
+
|
| 341 |
+
```bash
|
| 342 |
+
llama-cli -m model.gguf
|
| 343 |
+
|
| 344 |
+
# > hi, who are you?
|
| 345 |
+
# Hi there! I'm your helpful assistant! I'm an AI-powered chatbot designed to assist and provide information to users like you. I'm here to help answer your questions, provide guidance, and offer support on a wide range of topics. I'm a friendly and knowledgeable AI, and I'm always happy to help with anything you need. What's on your mind, and how can I assist you today?
|
| 346 |
+
#
|
| 347 |
+
# > what is 1+1?
|
| 348 |
+
# Easy peasy! The answer to 1+1 is... 2!
|
| 349 |
+
```
|
| 350 |
+
|
| 351 |
+
</details>
|
| 352 |
+
|
| 353 |
+
- <details>
|
| 354 |
+
<summary>Run in conversation mode with custom chat template</summary>
|
| 355 |
+
|
| 356 |
+
```bash
|
| 357 |
+
# use the "chatml" template (use -h to see the list of supported templates)
|
| 358 |
+
llama-cli -m model.gguf -cnv --chat-template chatml
|
| 359 |
+
|
| 360 |
+
# use a custom template
|
| 361 |
+
llama-cli -m model.gguf -cnv --in-prefix 'User: ' --reverse-prompt 'User:'
|
| 362 |
+
```
|
| 363 |
+
|
| 364 |
+
</details>
|
| 365 |
+
|
| 366 |
+
- <details>
|
| 367 |
+
<summary>Constrain the output with a custom grammar</summary>
|
| 368 |
+
|
| 369 |
+
```bash
|
| 370 |
+
llama-cli -m model.gguf -n 256 --grammar-file grammars/json.gbnf -p 'Request: schedule a call at 8pm; Command:'
|
| 371 |
+
|
| 372 |
+
# {"appointmentTime": "8pm", "appointmentDetails": "schedule a a call"}
|
| 373 |
+
```
|
| 374 |
+
|
| 375 |
+
The [grammars/](grammars/) folder contains a handful of sample grammars. To write your own, check out the [GBNF Guide](grammars/README.md).
|
| 376 |
+
|
| 377 |
+
For authoring more complex JSON grammars, check out https://grammar.intrinsiclabs.ai/
|
| 378 |
+
|
| 379 |
+
</details>
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
## [`llama-server`](tools/server)
|
| 383 |
+
|
| 384 |
+
#### A lightweight, [OpenAI API](https://github.com/openai/openai-openapi) compatible, HTTP server for serving LLMs.
|
| 385 |
+
|
| 386 |
+
- <details open>
|
| 387 |
+
<summary>Start a local HTTP server with default configuration on port 8080</summary>
|
| 388 |
+
|
| 389 |
+
```bash
|
| 390 |
+
llama-server -m model.gguf --port 8080
|
| 391 |
+
|
| 392 |
+
# Basic web UI can be accessed via browser: http://localhost:8080
|
| 393 |
+
# Chat completion endpoint: http://localhost:8080/v1/chat/completions
|
| 394 |
+
```
|
| 395 |
+
|
| 396 |
+
</details>
|
| 397 |
+
|
| 398 |
+
- <details>
|
| 399 |
+
<summary>Support multiple-users and parallel decoding</summary>
|
| 400 |
+
|
| 401 |
+
```bash
|
| 402 |
+
# up to 4 concurrent requests, each with 4096 max context
|
| 403 |
+
llama-server -m model.gguf -c 16384 -np 4
|
| 404 |
+
```
|
| 405 |
+
|
| 406 |
+
</details>
|
| 407 |
+
|
| 408 |
+
- <details>
|
| 409 |
+
<summary>Enable speculative decoding</summary>
|
| 410 |
+
|
| 411 |
+
```bash
|
| 412 |
+
# the draft.gguf model should be a small variant of the target model.gguf
|
| 413 |
+
llama-server -m model.gguf -md draft.gguf
|
| 414 |
+
```
|
| 415 |
+
|
| 416 |
+
</details>
|
| 417 |
+
|
| 418 |
+
- <details>
|
| 419 |
+
<summary>Serve an embedding model</summary>
|
| 420 |
+
|
| 421 |
+
```bash
|
| 422 |
+
# use the /embedding endpoint
|
| 423 |
+
llama-server -m model.gguf --embedding --pooling cls -ub 8192
|
| 424 |
+
```
|
| 425 |
+
|
| 426 |
+
</details>
|
| 427 |
+
|
| 428 |
+
- <details>
|
| 429 |
+
<summary>Serve a reranking model</summary>
|
| 430 |
+
|
| 431 |
+
```bash
|
| 432 |
+
# use the /reranking endpoint
|
| 433 |
+
llama-server -m model.gguf --reranking
|
| 434 |
+
```
|
| 435 |
+
|
| 436 |
+
</details>
|
| 437 |
+
|
| 438 |
+
- <details>
|
| 439 |
+
<summary>Constrain all outputs with a grammar</summary>
|
| 440 |
+
|
| 441 |
+
```bash
|
| 442 |
+
# custom grammar
|
| 443 |
+
llama-server -m model.gguf --grammar-file grammar.gbnf
|
| 444 |
+
|
| 445 |
+
# JSON
|
| 446 |
+
llama-server -m model.gguf --grammar-file grammars/json.gbnf
|
| 447 |
+
```
|
| 448 |
+
|
| 449 |
+
</details>
|
| 450 |
+
|
| 451 |
+
|
| 452 |
+
## [`llama-perplexity`](tools/perplexity)
|
| 453 |
+
|
| 454 |
+
#### A tool for measuring the [perplexity](tools/perplexity/README.md) [^1] (and other quality metrics) of a model over a given text.
|
| 455 |
+
|
| 456 |
+
- <details open>
|
| 457 |
+
<summary>Measure the perplexity over a text file</summary>
|
| 458 |
+
|
| 459 |
+
```bash
|
| 460 |
+
llama-perplexity -m model.gguf -f file.txt
|
| 461 |
+
|
| 462 |
+
# [1]15.2701,[2]5.4007,[3]5.3073,[4]6.2965,[5]5.8940,[6]5.6096,[7]5.7942,[8]4.9297, ...
|
| 463 |
+
# Final estimate: PPL = 5.4007 +/- 0.67339
|
| 464 |
+
```
|
| 465 |
+
|
| 466 |
+
</details>
|
| 467 |
+
|
| 468 |
+
- <details>
|
| 469 |
+
<summary>Measure KL divergence</summary>
|
| 470 |
+
|
| 471 |
+
```bash
|
| 472 |
+
# TODO
|
| 473 |
+
```
|
| 474 |
+
|
| 475 |
+
</details>
|
| 476 |
+
|
| 477 |
+
[^1]: [https://huggingface.co/docs/transformers/perplexity](https://huggingface.co/docs/transformers/perplexity)
|
| 478 |
+
|
| 479 |
+
## [`llama-bench`](tools/llama-bench)
|
| 480 |
+
|
| 481 |
+
#### Benchmark the performance of the inference for various parameters.
|
| 482 |
+
|
| 483 |
+
- <details open>
|
| 484 |
+
<summary>Run default benchmark</summary>
|
| 485 |
+
|
| 486 |
+
```bash
|
| 487 |
+
llama-bench -m model.gguf
|
| 488 |
+
|
| 489 |
+
# Output:
|
| 490 |
+
# | model | size | params | backend | threads | test | t/s |
|
| 491 |
+
# | ------------------- | ---------: | ---------: | ---------- | ------: | ------------: | -------------------: |
|
| 492 |
+
# | qwen2 1.5B Q4_0 | 885.97 MiB | 1.54 B | Metal,BLAS | 16 | pp512 | 5765.41 ± 20.55 |
|
| 493 |
+
# | qwen2 1.5B Q4_0 | 885.97 MiB | 1.54 B | Metal,BLAS | 16 | tg128 | 197.71 ± 0.81 |
|
| 494 |
+
#
|
| 495 |
+
# build: 3e0ba0e60 (4229)
|
| 496 |
+
```
|
| 497 |
+
|
| 498 |
+
</details>
|
| 499 |
+
|
| 500 |
+
## [`llama-simple`](examples/simple)
|
| 501 |
+
|
| 502 |
+
#### A minimal example for implementing apps with `llama.cpp`. Useful for developers.
|
| 503 |
+
|
| 504 |
+
- <details>
|
| 505 |
+
<summary>Basic text completion</summary>
|
| 506 |
+
|
| 507 |
+
```bash
|
| 508 |
+
llama-simple -m model.gguf
|
| 509 |
+
|
| 510 |
+
# Hello my name is Kaitlyn and I am a 16 year old girl. I am a junior in high school and I am currently taking a class called "The Art of
|
| 511 |
+
```
|
| 512 |
+
|
| 513 |
+
</details>
|
| 514 |
+
|
| 515 |
+
|
| 516 |
+
## Contributing
|
| 517 |
+
|
| 518 |
+
- Contributors can open PRs
|
| 519 |
+
- Collaborators will be invited based on contributions
|
| 520 |
+
- Maintainers can push to branches in the `llama.cpp` repo and merge PRs into the `master` branch
|
| 521 |
+
- Any help with managing issues, PRs and projects is very appreciated!
|
| 522 |
+
- See [good first issues](https://github.com/ggml-org/llama.cpp/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) for tasks suitable for first contributions
|
| 523 |
+
- Read the [CONTRIBUTING.md](CONTRIBUTING.md) for more information
|
| 524 |
+
- Make sure to read this: [Inference at the edge](https://github.com/ggml-org/llama.cpp/discussions/205)
|
| 525 |
+
- A bit of backstory for those who are interested: [Changelog podcast](https://changelog.com/podcast/532)
|
| 526 |
+
|
| 527 |
+
## Other documentation
|
| 528 |
+
|
| 529 |
+
- [cli](tools/cli/README.md)
|
| 530 |
+
- [completion](tools/completion/README.md)
|
| 531 |
+
- [server](tools/server/README.md)
|
| 532 |
+
- [GBNF grammars](grammars/README.md)
|
| 533 |
+
|
| 534 |
+
#### Development documentation
|
| 535 |
+
|
| 536 |
+
- [How to build](docs/build.md)
|
| 537 |
+
- [Running on Docker](docs/docker.md)
|
| 538 |
+
- [Build on Android](docs/android.md)
|
| 539 |
+
- [Multi-GPU usage](docs/multi-gpu.md)
|
| 540 |
+
- [Performance troubleshooting](docs/development/token_generation_performance_tips.md)
|
| 541 |
+
- [GGML tips & tricks](https://github.com/ggml-org/llama.cpp/wiki/GGML-Tips-&-Tricks)
|
| 542 |
+
|
| 543 |
+
#### Seminal papers and background on the models
|
| 544 |
+
|
| 545 |
+
If your issue is with model generation quality, then please at least scan the following links and papers to understand the limitations of LLaMA models. This is especially important when choosing an appropriate model size and appreciating both the significant and subtle differences between LLaMA models and ChatGPT:
|
| 546 |
+
- LLaMA:
|
| 547 |
+
- [Introducing LLaMA: A foundational, 65-billion-parameter large language model](https://ai.facebook.com/blog/large-language-model-llama-meta-ai/)
|
| 548 |
+
- [LLaMA: Open and Efficient Foundation Language Models](https://arxiv.org/abs/2302.13971)
|
| 549 |
+
- GPT-3
|
| 550 |
+
- [Language Models are Few-Shot Learners](https://arxiv.org/abs/2005.14165)
|
| 551 |
+
- GPT-3.5 / InstructGPT / ChatGPT:
|
| 552 |
+
- [Aligning language models to follow instructions](https://openai.com/research/instruction-following)
|
| 553 |
+
- [Training language models to follow instructions with human feedback](https://arxiv.org/abs/2203.02155)
|
| 554 |
+
|
| 555 |
+
## XCFramework
|
| 556 |
+
The XCFramework is a precompiled version of the library for iOS, visionOS, tvOS,
|
| 557 |
+
and macOS. It can be used in Swift projects without the need to compile the
|
| 558 |
+
library from source. For example:
|
| 559 |
+
```swift
|
| 560 |
+
// swift-tools-version: 5.10
|
| 561 |
+
// The swift-tools-version declares the minimum version of Swift required to build this package.
|
| 562 |
+
|
| 563 |
+
import PackageDescription
|
| 564 |
+
|
| 565 |
+
let package = Package(
|
| 566 |
+
name: "MyLlamaPackage",
|
| 567 |
+
targets: [
|
| 568 |
+
.executableTarget(
|
| 569 |
+
name: "MyLlamaPackage",
|
| 570 |
+
dependencies: [
|
| 571 |
+
"LlamaFramework"
|
| 572 |
+
]),
|
| 573 |
+
.binaryTarget(
|
| 574 |
+
name: "LlamaFramework",
|
| 575 |
+
url: "https://github.com/ggml-org/llama.cpp/releases/download/b5046/llama-b5046-xcframework.zip",
|
| 576 |
+
checksum: "c19be78b5f00d8d29a25da41042cb7afa094cbf6280a225abe614b03b20029ab"
|
| 577 |
+
)
|
| 578 |
+
]
|
| 579 |
+
)
|
| 580 |
+
```
|
| 581 |
+
The above example is using an intermediate build `b5046` of the library. This can be modified
|
| 582 |
+
to use a different version by changing the URL and checksum.
|
| 583 |
+
|
| 584 |
+
## Completions
|
| 585 |
+
Command-line completion is available for some environments.
|
| 586 |
+
|
| 587 |
+
#### Bash Completion
|
| 588 |
+
```bash
|
| 589 |
+
$ build/bin/llama-cli --completion-bash > ~/.llama-completion.bash
|
| 590 |
+
$ source ~/.llama-completion.bash
|
| 591 |
+
```
|
| 592 |
+
Optionally this can be added to your `.bashrc` or `.bash_profile` to load it
|
| 593 |
+
automatically. For example:
|
| 594 |
+
```console
|
| 595 |
+
$ echo "source ~/.llama-completion.bash" >> ~/.bashrc
|
| 596 |
+
```
|
| 597 |
+
|
| 598 |
+
## Dependencies
|
| 599 |
+
|
| 600 |
+
- [yhirose/cpp-httplib](https://github.com/yhirose/cpp-httplib) - Single-header HTTP server, used by `llama-server` - MIT license
|
| 601 |
+
- [stb-image](https://github.com/nothings/stb) - Single-header image format decoder, used by multimodal subsystem - Public domain
|
| 602 |
+
- [nlohmann/json](https://github.com/nlohmann/json) - Single-header JSON library, used by various tools/examples - MIT License
|
| 603 |
+
- [miniaudio.h](https://github.com/mackron/miniaudio) - Single-header audio format decoder, used by multimodal subsystem - Public domain
|
| 604 |
+
- [subprocess.h](https://github.com/sheredom/subprocess.h) - Single-header process launching solution for C and C++ - Public domain
|
backend/llama.cpp/SECURITY.md
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Security Policy
|
| 2 |
+
|
| 3 |
+
- [**Reporting a vulnerability**](#reporting-a-vulnerability)
|
| 4 |
+
- [**Requirements**](#requirements)
|
| 5 |
+
- [**Covered Topics**](#covered-topics)
|
| 6 |
+
- [**Using llama.cpp securely**](#using-llamacpp-securely)
|
| 7 |
+
- [Untrusted models](#untrusted-models)
|
| 8 |
+
- [Untrusted inputs](#untrusted-inputs)
|
| 9 |
+
- [Data privacy](#data-privacy)
|
| 10 |
+
- [Untrusted environments or networks](#untrusted-environments-or-networks)
|
| 11 |
+
- [Multi-Tenant environments](#multi-tenant-environments)
|
| 12 |
+
|
| 13 |
+
## Reporting a vulnerability
|
| 14 |
+
|
| 15 |
+
> [!IMPORTANT]
|
| 16 |
+
> The private security disclosure program is disabled until further notice. Please submit patches with fixes directly to the repo as public PRs. Emails will be ignored.
|
| 17 |
+
|
| 18 |
+
If you have discovered a security vulnerability in this project that falls inside the [covered topics](#covered-topics), please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released.
|
| 19 |
+
|
| 20 |
+
Please disclose it as a private [security advisory](https://github.com/ggml-org/llama.cpp/security/advisories/new).
|
| 21 |
+
|
| 22 |
+
A team of volunteers on a reasonable-effort basis maintains this project. As such, please give us at least 90 days to work on a fix before public exposure.
|
| 23 |
+
|
| 24 |
+
### Requirements
|
| 25 |
+
|
| 26 |
+
Before submitting your report, ensure you meet the following requirements:
|
| 27 |
+
|
| 28 |
+
- You have read this policy and fully understand it.
|
| 29 |
+
- AI is only permitted in an assistive capacity as stated in [AGENTS.md](AGENTS.md). We do not accept reports that are written exclusively by AI.
|
| 30 |
+
- Your report must include a working Proof-of-Concept in the form of a script and/or attached files.
|
| 31 |
+
|
| 32 |
+
Maintainers reserve the right to close the report if these requirements are not fulfilled.
|
| 33 |
+
|
| 34 |
+
### Covered Topics
|
| 35 |
+
|
| 36 |
+
Only vulnerabilities that fall within these parts of the project are considered valid. For problems falling outside of this list, please report them as issues.
|
| 37 |
+
|
| 38 |
+
- `src/**/*`
|
| 39 |
+
- `ggml/**/*`
|
| 40 |
+
- `gguf-py/**/*`
|
| 41 |
+
- `tools/server/*`, **excluding** the following topics:
|
| 42 |
+
- Web UI
|
| 43 |
+
- Features marked as experimental
|
| 44 |
+
- Features not recommended for use in untrusted environments (e.g., router, MCP)
|
| 45 |
+
- Bugs that can lead to Denial-of-Service attack
|
| 46 |
+
|
| 47 |
+
Note that none of the topics under [Using llama.cpp securely](#using-llamacpp-securely) are considered vulnerabilities in LLaMA C++.
|
| 48 |
+
|
| 49 |
+
For vulnerabilities that fall within the `vendor` directory, please report them directly to the third-party project.
|
| 50 |
+
|
| 51 |
+
## Using llama.cpp securely
|
| 52 |
+
|
| 53 |
+
### Untrusted models
|
| 54 |
+
Be careful when running untrusted models. This classification includes models created by unknown developers or utilizing data obtained from unknown sources.
|
| 55 |
+
|
| 56 |
+
*Always execute untrusted models within a secure, isolated environment such as a sandbox* (e.g., containers, virtual machines). This helps protect your system from potentially malicious code.
|
| 57 |
+
|
| 58 |
+
> [!NOTE]
|
| 59 |
+
> The trustworthiness of a model is not binary. You must always determine the proper level of caution depending on the specific model and how it matches your use case and risk tolerance.
|
| 60 |
+
|
| 61 |
+
### Untrusted inputs
|
| 62 |
+
|
| 63 |
+
Some models accept various input formats (text, images, audio, etc.). The libraries converting these inputs have varying security levels, so it's crucial to isolate the model and carefully pre-process inputs to mitigate script injection risks.
|
| 64 |
+
|
| 65 |
+
For maximum security when handling untrusted inputs, you may need to employ the following:
|
| 66 |
+
|
| 67 |
+
* Sandboxing: Isolate the environment where the inference happens.
|
| 68 |
+
* Pre-analysis: Check how the model performs by default when exposed to prompt injection (e.g. using [fuzzing for prompt injection](https://github.com/FonduAI/awesome-prompt-injection?tab=readme-ov-file#tools)). This will give you leads on how hard you will have to work on the next topics.
|
| 69 |
+
* Updates: Keep both LLaMA C++ and your libraries updated with the latest security patches.
|
| 70 |
+
* Input Sanitation: Before feeding data to the model, sanitize inputs rigorously. This involves techniques such as:
|
| 71 |
+
* Validation: Enforce strict rules on allowed characters and data types.
|
| 72 |
+
* Filtering: Remove potentially malicious scripts or code fragments.
|
| 73 |
+
* Encoding: Convert special characters into safe representations.
|
| 74 |
+
* Verification: Run tooling that identifies potential script injections (e.g. [models that detect prompt injection attempts](https://python.langchain.com/docs/guides/safety/hugging_face_prompt_injection)).
|
| 75 |
+
|
| 76 |
+
### Data privacy
|
| 77 |
+
|
| 78 |
+
To protect sensitive data from potential leaks or unauthorized access, it is crucial to sandbox the model execution. This means running the model in a secure, isolated environment, which helps mitigate many attack vectors.
|
| 79 |
+
|
| 80 |
+
### Untrusted environments or networks
|
| 81 |
+
|
| 82 |
+
If you can't run your models in a secure and isolated environment or if it must be exposed to an untrusted network, make sure to take the following security precautions:
|
| 83 |
+
* Do not use the RPC backend, [ggml-rpc-server](https://github.com/ggml-org/llama.cpp/tree/master/tools/rpc) and [llama-server](https://github.com/ggml-org/llama.cpp/tree/master/tools/server) functionality (see https://github.com/ggml-org/llama.cpp/pull/13061).
|
| 84 |
+
* Confirm the hash of any downloaded artifact (e.g. pre-trained model weights) matches a known-good value.
|
| 85 |
+
* Encrypt your data if sending it over the network.
|
| 86 |
+
|
| 87 |
+
### Multi-Tenant environments
|
| 88 |
+
|
| 89 |
+
If you intend to run multiple models in parallel with shared memory, it is your responsibility to ensure the models do not interact or access each other's data. The primary areas of concern are tenant isolation, resource allocation, model sharing and hardware attacks.
|
| 90 |
+
|
| 91 |
+
1. Tenant Isolation: Models should run separately with strong isolation methods to prevent unwanted data access. Separating networks is crucial for isolation, as it prevents unauthorized access to data or models and malicious users from sending graphs to execute under another tenant's identity.
|
| 92 |
+
|
| 93 |
+
2. Resource Allocation: A denial of service caused by one model can impact the overall system health. Implement safeguards like rate limits, access controls, and health monitoring.
|
| 94 |
+
|
| 95 |
+
3. Model Sharing: In a multitenant model sharing design, tenants and users must understand the security risks of running code provided by others. Since there are no reliable methods to detect malicious models, sandboxing the model execution is the recommended approach to mitigate the risk.
|
| 96 |
+
|
| 97 |
+
4. Hardware Attacks: GPUs or TPUs can also be attacked. [Researches](https://scholar.google.com/scholar?q=gpu+side+channel) has shown that side channel attacks on GPUs are possible, which can make data leak from other models or processes running on the same system at the same time.
|
backend/llama.cpp/models/templates/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.jinja
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{% macro render_extra_keys(json_dict, handled_keys) %}
|
| 2 |
+
{%- if json_dict is mapping %}
|
| 3 |
+
{%- for json_key in json_dict if json_key not in handled_keys %}
|
| 4 |
+
{%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
|
| 5 |
+
{{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
|
| 6 |
+
{%- else %}
|
| 7 |
+
{{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
|
| 8 |
+
{%- endif %}
|
| 9 |
+
{%- endfor %}
|
| 10 |
+
{%- endif %}
|
| 11 |
+
{% endmacro %}
|
| 12 |
+
{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %}
|
| 13 |
+
{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %}
|
| 14 |
+
|
| 15 |
+
{%- set ns = namespace(last_user_idx = -1) %}
|
| 16 |
+
{%- set loop_messages = messages %}
|
| 17 |
+
{%- for m in loop_messages %}
|
| 18 |
+
{%- if m["role"] == "user" %}
|
| 19 |
+
{%- set ns.last_user_idx = loop.index0 %}
|
| 20 |
+
{%- endif %}
|
| 21 |
+
{%- endfor %}
|
| 22 |
+
|
| 23 |
+
{%- if messages[0]["role"] == "system" %}
|
| 24 |
+
{%- set system_message = messages[0]["content"] %}
|
| 25 |
+
{%- set loop_messages = messages[1:] %}
|
| 26 |
+
{%- else %}
|
| 27 |
+
{%- set system_message = "" %}
|
| 28 |
+
{%- set loop_messages = messages %}
|
| 29 |
+
{%- endif %}
|
| 30 |
+
{%- if not tools is defined %}
|
| 31 |
+
{%- set tools = [] %}
|
| 32 |
+
{%- endif %}
|
| 33 |
+
{# Recompute last_user_idx relative to loop_messages after handling system #}
|
| 34 |
+
{%- set ns = namespace(last_user_idx = -1) %}
|
| 35 |
+
{%- for m in loop_messages %}
|
| 36 |
+
{%- if m["role"] == "user" %}
|
| 37 |
+
{%- set ns.last_user_idx = loop.index0 %}
|
| 38 |
+
{%- endif %}
|
| 39 |
+
{%- endfor %}
|
| 40 |
+
{%- if system_message is defined %}
|
| 41 |
+
{{- "<|im_start|>system\n" + system_message }}
|
| 42 |
+
{%- else %}
|
| 43 |
+
{%- if tools is iterable and tools | length > 0 %}
|
| 44 |
+
{{- "<|im_start|>system\n" }}
|
| 45 |
+
{%- endif %}
|
| 46 |
+
{%- endif %}
|
| 47 |
+
{%- if tools is iterable and tools | length > 0 %}
|
| 48 |
+
{%- if system_message is defined and system_message | length > 0 %}
|
| 49 |
+
{{- "\n\n" }}
|
| 50 |
+
{%- endif %}
|
| 51 |
+
{{- "# Tools\n\nYou have access to the following functions:\n\n" }}
|
| 52 |
+
{{- "<tools>" }}
|
| 53 |
+
{%- for tool in tools %}
|
| 54 |
+
{%- if tool.function is defined %}
|
| 55 |
+
{%- set tool = tool.function %}
|
| 56 |
+
{%- endif %}
|
| 57 |
+
{{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
|
| 58 |
+
{%- if tool.description is defined %}
|
| 59 |
+
{{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
|
| 60 |
+
{%- endif %}
|
| 61 |
+
{{- '\n<parameters>' }}
|
| 62 |
+
{%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
|
| 63 |
+
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
| 64 |
+
{{- '\n<parameter>' }}
|
| 65 |
+
{{- '\n<name>' ~ param_name ~ '</name>' }}
|
| 66 |
+
{%- if param_fields.type is defined %}
|
| 67 |
+
{{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
|
| 68 |
+
{%- endif %}
|
| 69 |
+
{%- if param_fields.description is defined %}
|
| 70 |
+
{{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
|
| 71 |
+
{%- endif %}
|
| 72 |
+
{%- if param_fields.enum is defined %}
|
| 73 |
+
{{- '\n<enum>' ~ (param_fields.enum | tojson | safe) ~ '</enum>' }}
|
| 74 |
+
{%- endif %}
|
| 75 |
+
{%- set handled_keys = ['name', 'type', 'description', 'enum'] %}
|
| 76 |
+
{{- render_extra_keys(param_fields, handled_keys) }}
|
| 77 |
+
{{- '\n</parameter>' }}
|
| 78 |
+
{%- endfor %}
|
| 79 |
+
{%- endif %}
|
| 80 |
+
{% set handled_keys = ['type', 'properties', 'required'] %}
|
| 81 |
+
{{- render_extra_keys(tool.parameters, handled_keys) }}
|
| 82 |
+
{%- if tool.parameters is defined and tool.parameters.required is defined %}
|
| 83 |
+
{{- '\n<required>' ~ (tool.parameters.required | tojson | safe) ~ '</required>' }}
|
| 84 |
+
{%- endif %}
|
| 85 |
+
{{- '\n</parameters>' }}
|
| 86 |
+
{%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
|
| 87 |
+
{{- render_extra_keys(tool, handled_keys) }}
|
| 88 |
+
{{- '\n</function>' }}
|
| 89 |
+
{%- endfor %}
|
| 90 |
+
{{- "\n</tools>" }}
|
| 91 |
+
|
| 92 |
+
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
| 93 |
+
{%- endif %}
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
{%- if system_message is defined %}
|
| 97 |
+
{{- '<|im_end|>\n' }}
|
| 98 |
+
{%- else %}
|
| 99 |
+
{%- if tools is iterable and tools | length > 0 %}
|
| 100 |
+
{{- '<|im_end|>\n' }}
|
| 101 |
+
{%- endif %}
|
| 102 |
+
{%- endif %}
|
| 103 |
+
|
| 104 |
+
{%- for message in loop_messages %}
|
| 105 |
+
{%- if message.role == "assistant" %}
|
| 106 |
+
{# Add reasoning content in to content field for unified processing below. #}
|
| 107 |
+
{%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}
|
| 108 |
+
{%- set content = "<think>\n" ~ message.reasoning_content ~ "\n</think>\n" ~ (message.content | default('', true)) %}
|
| 109 |
+
{%- else %}
|
| 110 |
+
{%- set content = message.content | default('', true) %}
|
| 111 |
+
{%- if content is string -%}
|
| 112 |
+
{# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}
|
| 113 |
+
{%- if '<think>' not in content and '</think>' not in content -%}
|
| 114 |
+
{%- set content = "<think></think>" ~ content -%}
|
| 115 |
+
{%- endif -%}
|
| 116 |
+
{%- else -%}
|
| 117 |
+
{%- set content = content -%}
|
| 118 |
+
{%- endif -%}
|
| 119 |
+
{%- endif %}
|
| 120 |
+
{%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
|
| 121 |
+
{# Assistant message has tool calls. #}
|
| 122 |
+
{{- '<|im_start|>assistant\n' }}
|
| 123 |
+
{%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
| 124 |
+
{%- if content is string and content | trim | length > 0 %}
|
| 125 |
+
{%- if include_content %}
|
| 126 |
+
{{- (content | trim) ~ '\n' -}}
|
| 127 |
+
{%- else %}
|
| 128 |
+
{%- set c = (content | string) %}
|
| 129 |
+
{%- if '</think>' in c %}
|
| 130 |
+
{# Keep only content after the last closing think. Also generation prompt causes this. #}
|
| 131 |
+
{%- set c = c.split('</think>')[-1] %}
|
| 132 |
+
{%- elif '<think>' in c %}
|
| 133 |
+
{# If <think> was opened but never closed, drop the trailing think segment #}
|
| 134 |
+
{%- set c = c.split('<think>')[0] %}
|
| 135 |
+
{%- endif %}
|
| 136 |
+
{%- set c = "<think></think>" ~ c | trim %}
|
| 137 |
+
{%- if c | length > 0 %}
|
| 138 |
+
{{- c ~ '\n' -}}
|
| 139 |
+
{%- endif %}
|
| 140 |
+
{%- endif %}
|
| 141 |
+
{%- else %}
|
| 142 |
+
{{- "<think></think>" -}}
|
| 143 |
+
{%- endif %}
|
| 144 |
+
{%- for tool_call in message.tool_calls %}
|
| 145 |
+
{%- if tool_call.function is defined %}
|
| 146 |
+
{%- set tool_call = tool_call.function %}
|
| 147 |
+
{%- endif %}
|
| 148 |
+
{{- '<tool_call>\n<function=' ~ tool_call.name ~ '>\n' -}}
|
| 149 |
+
{%- if tool_call.arguments is defined %}
|
| 150 |
+
{%- for args_name, args_value in tool_call.arguments|items %}
|
| 151 |
+
{{- '<parameter=' ~ args_name ~ '>\n' -}}
|
| 152 |
+
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
| 153 |
+
{{- args_value ~ '\n</parameter>\n' -}}
|
| 154 |
+
{%- endfor %}
|
| 155 |
+
{%- endif %}
|
| 156 |
+
{{- '</function>\n</tool_call>\n' -}}
|
| 157 |
+
{%- endfor %}
|
| 158 |
+
{{- '<|im_end|>\n' }}
|
| 159 |
+
{%- else %}
|
| 160 |
+
{# Assistant message doesn't have tool calls. #}
|
| 161 |
+
{%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
| 162 |
+
{{- '<|im_start|>assistant\n' ~ (content | default('', true) | string | trim) ~ '<|im_end|>\n' }}
|
| 163 |
+
{%- else %}
|
| 164 |
+
{%- set c = (content | default('', true) | string) %}
|
| 165 |
+
{%- if '<think>' in c and '</think>' in c %}
|
| 166 |
+
{%- set c = "<think></think>" ~ c.split('</think>')[-1] %}
|
| 167 |
+
{%- endif %}
|
| 168 |
+
{%- set c = c | trim %}
|
| 169 |
+
{%- if c | length > 0 %}
|
| 170 |
+
{{- '<|im_start|>assistant\n' ~ c ~ '<|im_end|>\n' }}
|
| 171 |
+
{%- else %}
|
| 172 |
+
{{- '<|im_start|>assistant\n<|im_end|>\n' }}
|
| 173 |
+
{%- endif %}
|
| 174 |
+
{%- endif %}
|
| 175 |
+
{%- endif %}
|
| 176 |
+
{%- elif message.role == "user" or message.role == "system" %}
|
| 177 |
+
{{- '<|im_start|>' + message.role + '\n' }}
|
| 178 |
+
{%- set content = message.content | string %}
|
| 179 |
+
{{- content }}
|
| 180 |
+
{{- '<|im_end|>\n' }}
|
| 181 |
+
{%- elif message.role == "tool" %}
|
| 182 |
+
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
| 183 |
+
{{- '<|im_start|>user\n' }}
|
| 184 |
+
{%- endif %}
|
| 185 |
+
{{- '<tool_response>\n' }}
|
| 186 |
+
{{- message.content }}
|
| 187 |
+
{{- '\n</tool_response>\n' }}
|
| 188 |
+
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
| 189 |
+
{{- '<|im_end|>\n' }}
|
| 190 |
+
{%- elif loop.last %}
|
| 191 |
+
{{- '<|im_end|>\n' }}
|
| 192 |
+
{%- endif %}
|
| 193 |
+
{%- else %}
|
| 194 |
+
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
|
| 195 |
+
{%- endif %}
|
| 196 |
+
{%- endfor %}
|
| 197 |
+
|
| 198 |
+
{%- if add_generation_prompt %}
|
| 199 |
+
{%- if enable_thinking %}
|
| 200 |
+
{{- '<|im_start|>assistant\n<think>\n' }}
|
| 201 |
+
{%- else %}
|
| 202 |
+
{{- '<|im_start|>assistant\n<think></think>' }}
|
| 203 |
+
{%- endif %}
|
| 204 |
+
{%- endif %}
|
backend/llama.cpp/models/templates/NVIDIA-Nemotron-Nano-v2.jinja
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- set ns = namespace(enable_thinking=true) -%}
|
| 2 |
+
{%- for message in messages -%}
|
| 3 |
+
{%- set content = message['content'] -%}
|
| 4 |
+
{%- if message['role'] == 'user' or message['role'] == 'system' -%}
|
| 5 |
+
{%- if '/think' in content -%}
|
| 6 |
+
{%- set ns.enable_thinking = true -%}
|
| 7 |
+
{%- elif '/no_think' in content -%}
|
| 8 |
+
{%- set ns.enable_thinking = false -%}
|
| 9 |
+
{%- endif -%}
|
| 10 |
+
{%- endif -%}
|
| 11 |
+
{%- endfor -%}
|
| 12 |
+
|
| 13 |
+
{%- if messages[0]['role'] != 'system' -%}
|
| 14 |
+
{%- set ns.non_tool_system_content = '' -%}
|
| 15 |
+
{{- '<SPECIAL_10>System
|
| 16 |
+
' -}}
|
| 17 |
+
{%- else -%}
|
| 18 |
+
{%- set ns.non_tool_system_content = (messages[0]['content'] | default('', true)).replace('/think', '').replace('/no_think', '').strip() -%}
|
| 19 |
+
{{- '<SPECIAL_10>System
|
| 20 |
+
' + ns.non_tool_system_content }}
|
| 21 |
+
{%- endif -%}
|
| 22 |
+
|
| 23 |
+
{%- if tools -%}
|
| 24 |
+
{%- if ns.non_tool_system_content is defined and ns.non_tool_system_content != '' -%}
|
| 25 |
+
{{- '
|
| 26 |
+
|
| 27 |
+
' -}}
|
| 28 |
+
{%- endif -%}
|
| 29 |
+
{{- 'You can use the following tools to assist the user if required:' -}}
|
| 30 |
+
{{- '
|
| 31 |
+
<AVAILABLE_TOOLS>[' -}}
|
| 32 |
+
{%- for tool in tools -%}
|
| 33 |
+
{{- (tool.function if tool.function is defined else tool) | tojson -}}
|
| 34 |
+
{{- ', ' if not loop.last else '' -}}
|
| 35 |
+
{%- endfor -%}
|
| 36 |
+
{{- ']</AVAILABLE_TOOLS>
|
| 37 |
+
|
| 38 |
+
' -}}
|
| 39 |
+
{{- 'If you decide to call any tool(s), use the following format:
|
| 40 |
+
' -}}
|
| 41 |
+
{{- '<TOOLCALL>[{{"name": "tool_name1", "arguments": "tool_args1"}}, ' -}}
|
| 42 |
+
{{- '{{"name": "tool_name2", "arguments": "tool_args2"}}]</TOOLCALL>
|
| 43 |
+
|
| 44 |
+
' -}}
|
| 45 |
+
{{- 'The user will execute tool-calls and return responses from tool(s) in this format:
|
| 46 |
+
' -}}
|
| 47 |
+
{{- '<TOOL_RESPONSE>[{{"tool_response1"}}, {{"tool_response2"}}]</TOOL_RESPONSE>
|
| 48 |
+
|
| 49 |
+
' -}}
|
| 50 |
+
{{- 'Based on the tool responses, you can call additional tools if needed, correct tool calls if any errors are found, or just respond to the user.' -}}
|
| 51 |
+
{%- endif -%}
|
| 52 |
+
{{- '
|
| 53 |
+
|
| 54 |
+
' -}}
|
| 55 |
+
{%- set messages = messages[1:] if messages[0]['role'] == 'system' else messages -%}
|
| 56 |
+
{%- if messages[-1]['role'] == 'assistant' -%}
|
| 57 |
+
{%- set ns.last_turn_assistant_content = (messages[-1]['content'] | default('', true)).strip() -%}
|
| 58 |
+
{%- set ns.last_turn_assistant_tool_calls = messages[-1]['tool_calls'] if 'tool_calls' in messages[-1] else [] -%}
|
| 59 |
+
{%- set messages = messages[:-1] -%}
|
| 60 |
+
{%- endif -%}
|
| 61 |
+
|
| 62 |
+
{%- for message in messages %}
|
| 63 |
+
{%- set content = message['content'] %}
|
| 64 |
+
{%- if message['role'] == 'user' -%}
|
| 65 |
+
{{- '<SPECIAL_11>User
|
| 66 |
+
' + (content | default('', true)).replace('/think', '').replace('/no_think', '').strip() + '
|
| 67 |
+
' }}
|
| 68 |
+
{%- elif message['role'] == 'tool' -%}
|
| 69 |
+
{%- if loop.first or (messages[loop.index0 - 1].role != 'tool') -%}
|
| 70 |
+
{{- '<SPECIAL_11>User
|
| 71 |
+
' + '<TOOL_RESPONSE>[' }}
|
| 72 |
+
{%- endif -%}
|
| 73 |
+
{{- message['content'] -}}
|
| 74 |
+
{{- ', ' if not loop.last and (messages[loop.index0 + 1].role == 'tool') else '' -}}
|
| 75 |
+
{%- if loop.last or (messages[loop.index0 + 1].role != 'tool') -%}
|
| 76 |
+
{{- ']</TOOL_RESPONSE>' -}}
|
| 77 |
+
{%- endif -%}
|
| 78 |
+
{%- elif message['role'] == 'assistant' -%}
|
| 79 |
+
{%- if content and '</think>' in content -%}
|
| 80 |
+
{%- set content = (content.split('</think>')[1] | default('', true)).strip() %}
|
| 81 |
+
{%- endif -%}
|
| 82 |
+
{{- '<SPECIAL_11>Assistant
|
| 83 |
+
' + ((content | default('', true)).strip() if content is not none else '') }}
|
| 84 |
+
{%- if message.tool_calls -%}
|
| 85 |
+
{%- if (content | default('', true)).strip() != '' -%}
|
| 86 |
+
{{- '
|
| 87 |
+
' -}}
|
| 88 |
+
{%- endif -%}
|
| 89 |
+
{{- '<TOOLCALL>[' -}}
|
| 90 |
+
{%- for call in message.tool_calls -%}
|
| 91 |
+
{%- set fn = call.function if call.function is defined else call -%}
|
| 92 |
+
{{- '{"name": "' + fn.name + '", "arguments": ' -}}
|
| 93 |
+
{%- if fn.arguments is string -%}
|
| 94 |
+
{{- fn.arguments -}}
|
| 95 |
+
{%- else -%}
|
| 96 |
+
{{- fn.arguments | tojson -}}
|
| 97 |
+
{%- endif -%}
|
| 98 |
+
{{- '}' + (', ' if not loop.last else '') -}}
|
| 99 |
+
{%- endfor -%}
|
| 100 |
+
{{- ']</TOOLCALL>' -}}
|
| 101 |
+
{%- endif -%}
|
| 102 |
+
{{- '
|
| 103 |
+
<SPECIAL_12>
|
| 104 |
+
' -}}
|
| 105 |
+
{%- endif -%}
|
| 106 |
+
{%- endfor -%}
|
| 107 |
+
|
| 108 |
+
{%- if add_generation_prompt -%}
|
| 109 |
+
{{- '<SPECIAL_11>Assistant
|
| 110 |
+
' -}}
|
| 111 |
+
{%- if ns.enable_thinking is defined and ns.enable_thinking is false -%}
|
| 112 |
+
{{- '<think></think>' -}}
|
| 113 |
+
{%- else -%}
|
| 114 |
+
{{- '<think>
|
| 115 |
+
' -}}
|
| 116 |
+
{%- endif -%}
|
| 117 |
+
{%- if ns.last_turn_assistant_content is defined and ns.last_turn_assistant_content != '' -%}
|
| 118 |
+
{{- ns.last_turn_assistant_content -}}
|
| 119 |
+
{%- endif -%}
|
| 120 |
+
{%- else -%}
|
| 121 |
+
{%- if ns.last_turn_assistant_content is defined and ns.last_turn_assistant_content != '' -%}
|
| 122 |
+
{{- '<SPECIAL_11>Assistant
|
| 123 |
+
' -}}
|
| 124 |
+
{%- if ns.enable_thinking is defined and ns.enable_thinking is false -%}
|
| 125 |
+
{{- '<think></think>' -}}
|
| 126 |
+
{%- else -%}
|
| 127 |
+
{{- '<think>
|
| 128 |
+
' -}}
|
| 129 |
+
{%- endif -%}
|
| 130 |
+
{{- ns.last_turn_assistant_content -}}
|
| 131 |
+
{%- if continue_final_message is defined -%}
|
| 132 |
+
{%- if continue_final_message is false -%}
|
| 133 |
+
{{- '
|
| 134 |
+
<SPECIAL_12>
|
| 135 |
+
' -}}
|
| 136 |
+
{%- endif -%}
|
| 137 |
+
{%- else -%}
|
| 138 |
+
{{- '
|
| 139 |
+
<SPECIAL_12>
|
| 140 |
+
' -}}
|
| 141 |
+
{%- endif -%}
|
| 142 |
+
{%- endif -%}
|
| 143 |
+
{%- if ns.last_turn_assistant_tool_calls is defined and ns.last_turn_assistant_tool_calls | length > 0 -%}
|
| 144 |
+
{{- '<SPECIAL_11>Assistant
|
| 145 |
+
' -}}
|
| 146 |
+
{{- '<TOOLCALL>[' -}}
|
| 147 |
+
{%- for call in ns.last_turn_assistant_tool_calls -%}
|
| 148 |
+
{%- set fn = call.function if call.function is defined else call -%}
|
| 149 |
+
{{- '{"name": "' + fn.name + '", "arguments": ' -}}
|
| 150 |
+
{%- if fn.arguments is string -%}
|
| 151 |
+
{{- fn.arguments -}}
|
| 152 |
+
{%- else -%}
|
| 153 |
+
{{- fn.arguments | tojson -}}
|
| 154 |
+
{%- endif -%}
|
| 155 |
+
{{- '}' + (', ' if not loop.last else '') -}}
|
| 156 |
+
{%- endfor -%}
|
| 157 |
+
{{- ']</TOOLCALL>' -}}
|
| 158 |
+
{{- '<SPECIAL_12>
|
| 159 |
+
|
| 160 |
+
' -}}
|
| 161 |
+
{%- endif -%}
|
| 162 |
+
{%- endif -%}
|
backend/llama.cpp/models/templates/NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- macro json_to_python_type(json_spec) %}
|
| 2 |
+
{%- set basic_type_map = {
|
| 3 |
+
"string": "str",
|
| 4 |
+
"number": "float",
|
| 5 |
+
"integer": "int",
|
| 6 |
+
"boolean": "bool"
|
| 7 |
+
} %}
|
| 8 |
+
|
| 9 |
+
{%- if basic_type_map[json_spec.type] is defined %}
|
| 10 |
+
{{- basic_type_map[json_spec.type] }}
|
| 11 |
+
{%- elif json_spec.type == "array" %}
|
| 12 |
+
{{- "list[" + json_to_python_type(json_spec|items) + "]"}}
|
| 13 |
+
{%- elif json_spec.type == "object" %}
|
| 14 |
+
{%- if json_spec.additionalProperties is defined %}
|
| 15 |
+
{{- "dict[str, " + json_to_python_type(json_spec.additionalProperties) + ']'}}
|
| 16 |
+
{%- else %}
|
| 17 |
+
{{- "dict" }}
|
| 18 |
+
{%- endif %}
|
| 19 |
+
{%- elif json_spec.type is iterable %}
|
| 20 |
+
{{- "Union[" }}
|
| 21 |
+
{%- for t in json_spec.type %}
|
| 22 |
+
{{- json_to_python_type({"type": t}) }}
|
| 23 |
+
{%- if not loop.last %}
|
| 24 |
+
{{- "," }}
|
| 25 |
+
{%- endif %}
|
| 26 |
+
{%- endfor %}
|
| 27 |
+
{{- "]" }}
|
| 28 |
+
{%- else %}
|
| 29 |
+
{{- "Any" }}
|
| 30 |
+
{%- endif %}
|
| 31 |
+
{%- endmacro %}
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
{{- bos_token }}
|
| 35 |
+
{{- '<|im_start|>system
|
| 36 |
+
' }}
|
| 37 |
+
{{- "You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: <tools> " }}
|
| 38 |
+
{%- for tool in tools %}
|
| 39 |
+
{%- if tool.function is defined %}
|
| 40 |
+
{%- set tool = tool.function %}
|
| 41 |
+
{%- endif %}
|
| 42 |
+
{{- '{"type": "function", "function": ' }}
|
| 43 |
+
{{- '{"name": "' + tool.name + '", ' }}
|
| 44 |
+
{{- '"description": "' + tool.name + '(' }}
|
| 45 |
+
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
| 46 |
+
{{- param_name + ": " + json_to_python_type(param_fields) }}
|
| 47 |
+
{%- if not loop.last %}
|
| 48 |
+
{{- ", " }}
|
| 49 |
+
{%- endif %}
|
| 50 |
+
{%- endfor %}
|
| 51 |
+
{{- ")" }}
|
| 52 |
+
{%- if tool.return is defined %}
|
| 53 |
+
{{- " -> " + json_to_python_type(tool.return) }}
|
| 54 |
+
{%- endif %}
|
| 55 |
+
{{- " - " + tool.description + "
|
| 56 |
+
|
| 57 |
+
" }}
|
| 58 |
+
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
| 59 |
+
{%- if loop.first %}
|
| 60 |
+
{{- " Args:
|
| 61 |
+
" }}
|
| 62 |
+
{%- endif %}
|
| 63 |
+
{{- " " + param_name + "(" + json_to_python_type(param_fields) + "): " + param_fields.description|trim }}
|
| 64 |
+
{%- endfor %}
|
| 65 |
+
{%- if tool.return is defined and tool.return.description is defined %}
|
| 66 |
+
{{- "
|
| 67 |
+
Returns:
|
| 68 |
+
" + tool.return.description }}
|
| 69 |
+
{%- endif %}
|
| 70 |
+
{{- '"' }}
|
| 71 |
+
{{- ', "parameters": ' }}
|
| 72 |
+
{%- if tool.parameters.properties | length == 0 %}
|
| 73 |
+
{{- "{}" }}
|
| 74 |
+
{%- else %}
|
| 75 |
+
{{- tool.parameters|tojson }}
|
| 76 |
+
{%- endif %}
|
| 77 |
+
{{- "}" }}
|
| 78 |
+
{%- if not loop.last %}
|
| 79 |
+
{{- "
|
| 80 |
+
" }}
|
| 81 |
+
{%- endif %}
|
| 82 |
+
{%- endfor %}
|
| 83 |
+
{{- " </tools>" }}
|
| 84 |
+
{{- 'Use the following pydantic model json schema for each tool call you will make: {"properties": {"name": {"title": "Name", "type": "string"}, "arguments": {"title": "Arguments", "type": "object"}}, "required": ["name", "arguments"], "title": "FunctionCall", "type": "object"}}
|
| 85 |
+
' }}
|
| 86 |
+
{{- "For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:
|
| 87 |
+
" }}
|
| 88 |
+
{{- "<tool_call>
|
| 89 |
+
" }}
|
| 90 |
+
{{- '{"name": <function-name>, "arguments": <args-dict>}
|
| 91 |
+
' }}
|
| 92 |
+
{{- '</tool_call><|im_end|>
|
| 93 |
+
' }}
|
| 94 |
+
{%- for message in messages %}
|
| 95 |
+
{%- if message.role == "user" or message.role == "system" or (message.role == "assistant" and message.tool_calls is not defined) %}
|
| 96 |
+
{{- '<|im_start|>' + message.role + '
|
| 97 |
+
' + message.content + '<|im_end|>' + '
|
| 98 |
+
' }}
|
| 99 |
+
{%- elif message.role == "assistant" %}
|
| 100 |
+
{{- '<|im_start|>' + message.role }}
|
| 101 |
+
{%- for tool_call in message.tool_calls %}
|
| 102 |
+
{{- '
|
| 103 |
+
<tool_call>
|
| 104 |
+
' }} {%- if tool_call.function is defined %}
|
| 105 |
+
{%- set tool_call = tool_call.function %}
|
| 106 |
+
{%- endif %}
|
| 107 |
+
{{- '{' }}
|
| 108 |
+
{{- '"name": "' }}
|
| 109 |
+
{{- tool_call.name }}
|
| 110 |
+
{{- '"' }}
|
| 111 |
+
{{- ', '}}
|
| 112 |
+
{%- if tool_call.arguments is defined %}
|
| 113 |
+
{{- '"arguments": ' }}
|
| 114 |
+
{%- if tool_call.arguments is string %}
|
| 115 |
+
{{- tool_call.arguments }}
|
| 116 |
+
{%- else %}
|
| 117 |
+
{{- tool_call.arguments|tojson }}
|
| 118 |
+
{%- endif %}
|
| 119 |
+
{%- endif %}
|
| 120 |
+
{{- '}' }}
|
| 121 |
+
{{- '
|
| 122 |
+
</tool_call>' }}
|
| 123 |
+
{%- endfor %}
|
| 124 |
+
{{- '<|im_end|>
|
| 125 |
+
' }}
|
| 126 |
+
{%- elif message.role == "tool" %}
|
| 127 |
+
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
| 128 |
+
{{- '<|im_start|>tool
|
| 129 |
+
' }}
|
| 130 |
+
{%- endif %}
|
| 131 |
+
{{- '<tool_response>
|
| 132 |
+
' }}
|
| 133 |
+
{{- message.content }}
|
| 134 |
+
{%- if not loop.last %}
|
| 135 |
+
{{- '
|
| 136 |
+
</tool_response>
|
| 137 |
+
' }}
|
| 138 |
+
{%- else %}
|
| 139 |
+
{{- '
|
| 140 |
+
</tool_response>' }}
|
| 141 |
+
{%- endif %}
|
| 142 |
+
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
| 143 |
+
{{- '<|im_end|>' }}
|
| 144 |
+
{%- elif loop.last %}
|
| 145 |
+
{{- '<|im_end|>' }}
|
| 146 |
+
{%- endif %}
|
| 147 |
+
{%- endif %}
|
| 148 |
+
{%- endfor %}
|
| 149 |
+
{%- if add_generation_prompt %}
|
| 150 |
+
{{- '<|im_start|>assistant
|
| 151 |
+
' }}
|
| 152 |
+
{%- endif %}
|
backend/llama.cpp/models/templates/NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- macro json_to_python_type(json_spec) %}
|
| 2 |
+
{%- set basic_type_map = {
|
| 3 |
+
"string": "str",
|
| 4 |
+
"number": "float",
|
| 5 |
+
"integer": "int",
|
| 6 |
+
"boolean": "bool"
|
| 7 |
+
} %}
|
| 8 |
+
|
| 9 |
+
{%- if basic_type_map[json_spec.type] is defined %}
|
| 10 |
+
{{- basic_type_map[json_spec.type] }}
|
| 11 |
+
{%- elif json_spec.type == "array" %}
|
| 12 |
+
{{- "list[" + json_to_python_type(json_spec|items) + "]"}}
|
| 13 |
+
{%- elif json_spec.type == "object" %}
|
| 14 |
+
{%- if json_spec.additionalProperties is defined %}
|
| 15 |
+
{{- "dict[str, " + json_to_python_type(json_spec.additionalProperties) + ']'}}
|
| 16 |
+
{%- else %}
|
| 17 |
+
{{- "dict" }}
|
| 18 |
+
{%- endif %}
|
| 19 |
+
{%- elif json_spec.type is iterable %}
|
| 20 |
+
{{- "Union[" }}
|
| 21 |
+
{%- for t in json_spec.type %}
|
| 22 |
+
{{- json_to_python_type({"type": t}) }}
|
| 23 |
+
{%- if not loop.last %}
|
| 24 |
+
{{- "," }}
|
| 25 |
+
{%- endif %}
|
| 26 |
+
{%- endfor %}
|
| 27 |
+
{{- "]" }}
|
| 28 |
+
{%- else %}
|
| 29 |
+
{{- "Any" }}
|
| 30 |
+
{%- endif %}
|
| 31 |
+
{%- endmacro %}
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
{{- bos_token }}
|
| 35 |
+
{{- '<|im_start|>system
|
| 36 |
+
' }}
|
| 37 |
+
{{- "You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: <tools> " }}
|
| 38 |
+
{%- for tool in tools %}
|
| 39 |
+
{%- if tool.function is defined %}
|
| 40 |
+
{%- set tool = tool.function %}
|
| 41 |
+
{%- endif %}
|
| 42 |
+
{{- '{"type": "function", "function": ' }}
|
| 43 |
+
{{- '{"name": "' + tool.name + '", ' }}
|
| 44 |
+
{{- '"description": "' + tool.name + '(' }}
|
| 45 |
+
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
| 46 |
+
{{- param_name + ": " + json_to_python_type(param_fields) }}
|
| 47 |
+
{%- if not loop.last %}
|
| 48 |
+
{{- ", " }}
|
| 49 |
+
{%- endif %}
|
| 50 |
+
{%- endfor %}
|
| 51 |
+
{{- ")" }}
|
| 52 |
+
{%- if tool.return is defined %}
|
| 53 |
+
{{- " -> " + json_to_python_type(tool.return) }}
|
| 54 |
+
{%- endif %}
|
| 55 |
+
{{- " - " + tool.description + "
|
| 56 |
+
|
| 57 |
+
" }}
|
| 58 |
+
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
| 59 |
+
{%- if loop.first %}
|
| 60 |
+
{{- " Args:
|
| 61 |
+
" }}
|
| 62 |
+
{%- endif %}
|
| 63 |
+
{{- " " + param_name + "(" + json_to_python_type(param_fields) + "): " + param_fields.description|trim }}
|
| 64 |
+
{%- endfor %}
|
| 65 |
+
{%- if tool.return is defined and tool.return.description is defined %}
|
| 66 |
+
{{- "
|
| 67 |
+
Returns:
|
| 68 |
+
" + tool.return.description }}
|
| 69 |
+
{%- endif %}
|
| 70 |
+
{{- '"' }}
|
| 71 |
+
{{- ', "parameters": ' }}
|
| 72 |
+
{%- if tool.parameters.properties | length == 0 %}
|
| 73 |
+
{{- "{}" }}
|
| 74 |
+
{%- else %}
|
| 75 |
+
{{- tool.parameters|tojson }}
|
| 76 |
+
{%- endif %}
|
| 77 |
+
{{- "}" }}
|
| 78 |
+
{%- if not loop.last %}
|
| 79 |
+
{{- "
|
| 80 |
+
" }}
|
| 81 |
+
{%- endif %}
|
| 82 |
+
{%- endfor %}
|
| 83 |
+
{{- " </tools>" }}
|
| 84 |
+
{{- 'Use the following pydantic model json schema for each tool call you will make: {"properties": {"name": {"title": "Name", "type": "string"}, "arguments": {"title": "Arguments", "type": "object"}}, "required": ["name", "arguments"], "title": "FunctionCall", "type": "object"}}
|
| 85 |
+
' }}
|
| 86 |
+
{{- "For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:
|
| 87 |
+
" }}
|
| 88 |
+
{{- "<tool_call>
|
| 89 |
+
" }}
|
| 90 |
+
{{- '{"name": <function-name>, "arguments": <args-dict>}
|
| 91 |
+
' }}
|
| 92 |
+
{{- '</tool_call><|im_end|>
|
| 93 |
+
' }}
|
| 94 |
+
{%- for message in messages %}
|
| 95 |
+
{%- if message.role == "user" or message.role == "system" or (message.role == "assistant" and message.tool_calls is not defined) %}
|
| 96 |
+
{{- '<|im_start|>' + message.role + '
|
| 97 |
+
' + message.content + '<|im_end|>' + '
|
| 98 |
+
' }}
|
| 99 |
+
{%- elif message.role == "assistant" %}
|
| 100 |
+
{{- '<|im_start|>' + message.role }}
|
| 101 |
+
{%- for tool_call in message.tool_calls %}
|
| 102 |
+
{{- '
|
| 103 |
+
<tool_call>
|
| 104 |
+
' }} {%- if tool_call.function is defined %}
|
| 105 |
+
{%- set tool_call = tool_call.function %}
|
| 106 |
+
{%- endif %}
|
| 107 |
+
{{- '{' }}
|
| 108 |
+
{{- '"name": "' }}
|
| 109 |
+
{{- tool_call.name }}
|
| 110 |
+
{{- '"' }}
|
| 111 |
+
{{- ', '}}
|
| 112 |
+
{%- if tool_call.arguments is defined %}
|
| 113 |
+
{{- '"arguments": ' }}
|
| 114 |
+
{%- if tool_call.arguments is string %}
|
| 115 |
+
{{- tool_call.arguments }}
|
| 116 |
+
{%- else %}
|
| 117 |
+
{{- tool_call.arguments|tojson }}
|
| 118 |
+
{%- endif %}
|
| 119 |
+
{%- endif %}
|
| 120 |
+
{{- '}' }}
|
| 121 |
+
{{- '
|
| 122 |
+
</tool_call>' }}
|
| 123 |
+
{%- endfor %}
|
| 124 |
+
{{- '<|im_end|>
|
| 125 |
+
' }}
|
| 126 |
+
{%- elif message.role == "tool" %}
|
| 127 |
+
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
| 128 |
+
{{- '<|im_start|>tool
|
| 129 |
+
' }}
|
| 130 |
+
{%- endif %}
|
| 131 |
+
{{- '<tool_response>
|
| 132 |
+
' }}
|
| 133 |
+
{{- message.content }}
|
| 134 |
+
{%- if not loop.last %}
|
| 135 |
+
{{- '
|
| 136 |
+
</tool_response>
|
| 137 |
+
' }}
|
| 138 |
+
{%- else %}
|
| 139 |
+
{{- '
|
| 140 |
+
</tool_response>' }}
|
| 141 |
+
{%- endif %}
|
| 142 |
+
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
| 143 |
+
{{- '<|im_end|>' }}
|
| 144 |
+
{%- elif loop.last %}
|
| 145 |
+
{{- '<|im_end|>' }}
|
| 146 |
+
{%- endif %}
|
| 147 |
+
{%- endif %}
|
| 148 |
+
{%- endfor %}
|
| 149 |
+
{%- if add_generation_prompt %}
|
| 150 |
+
{{- '<|im_start|>assistant
|
| 151 |
+
' }}
|
| 152 |
+
{%- endif %}
|
backend/llama.cpp/models/templates/Qwen-QwQ-32B.jinja
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- if tools %}
|
| 2 |
+
{{- '<|im_start|>system\n' }}
|
| 3 |
+
{%- if messages[0]['role'] == 'system' %}
|
| 4 |
+
{{- messages[0]['content'] }}
|
| 5 |
+
{%- else %}
|
| 6 |
+
{{- '' }}
|
| 7 |
+
{%- endif %}
|
| 8 |
+
{{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
| 9 |
+
{%- for tool in tools %}
|
| 10 |
+
{{- "\n" }}
|
| 11 |
+
{{- tool | tojson }}
|
| 12 |
+
{%- endfor %}
|
| 13 |
+
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
| 14 |
+
{%- else %}
|
| 15 |
+
{%- if messages[0]['role'] == 'system' %}
|
| 16 |
+
{{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
|
| 17 |
+
{%- endif %}
|
| 18 |
+
{%- endif %}
|
| 19 |
+
{%- for message in messages %}
|
| 20 |
+
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
|
| 21 |
+
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
| 22 |
+
{%- elif message.role == "assistant" and not message.tool_calls %}
|
| 23 |
+
{%- set content = message.content %}
|
| 24 |
+
{%- if not loop.last %}
|
| 25 |
+
{%- set content = message.content.split('</think>')[-1].lstrip('\n') %}
|
| 26 |
+
{%- endif %}
|
| 27 |
+
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
| 28 |
+
{%- elif message.role == "assistant" %}
|
| 29 |
+
{%- set content = message.content %}
|
| 30 |
+
{%- if not loop.last %}
|
| 31 |
+
{%- set content = message.content.split('</think>')[-1].lstrip('\n') %}
|
| 32 |
+
{%- endif %}
|
| 33 |
+
{{- '<|im_start|>' + message.role }}
|
| 34 |
+
{%- if message.content %}
|
| 35 |
+
{{- '\n' + content }}
|
| 36 |
+
{%- endif %}
|
| 37 |
+
{%- for tool_call in message.tool_calls %}
|
| 38 |
+
{%- if tool_call.function is defined %}
|
| 39 |
+
{%- set tool_call = tool_call.function %}
|
| 40 |
+
{%- endif %}
|
| 41 |
+
{{- '\n<tool_call>\n{"name": "' }}
|
| 42 |
+
{{- tool_call.name }}
|
| 43 |
+
{{- '", "arguments": ' }}
|
| 44 |
+
{{- tool_call.arguments | tojson }}
|
| 45 |
+
{{- '}\n</tool_call>' }}
|
| 46 |
+
{%- endfor %}
|
| 47 |
+
{{- '<|im_end|>\n' }}
|
| 48 |
+
{%- elif message.role == "tool" %}
|
| 49 |
+
{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
|
| 50 |
+
{{- '<|im_start|>user' }}
|
| 51 |
+
{%- endif %}
|
| 52 |
+
{{- '\n<tool_response>\n' }}
|
| 53 |
+
{{- message.content }}
|
| 54 |
+
{{- '\n</tool_response>' }}
|
| 55 |
+
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
| 56 |
+
{{- '<|im_end|>\n' }}
|
| 57 |
+
{%- endif %}
|
| 58 |
+
{%- endif %}
|
| 59 |
+
{%- endfor %}
|
| 60 |
+
{%- if add_generation_prompt %}
|
| 61 |
+
{{- '<|im_start|>assistant\n<think>\n' }}
|
| 62 |
+
{%- if not enable_thinking -%}{{- '</think>' -}}{%- endif -%}
|
| 63 |
+
{%- endif %}
|
backend/llama.cpp/models/templates/Qwen-Qwen2.5-7B-Instruct.jinja
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- if tools %}
|
| 2 |
+
{{- '<|im_start|>system\n' }}
|
| 3 |
+
{%- if messages[0]['role'] == 'system' %}
|
| 4 |
+
{{- messages[0]['content'] }}
|
| 5 |
+
{%- else %}
|
| 6 |
+
{{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
|
| 7 |
+
{%- endif %}
|
| 8 |
+
{{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
| 9 |
+
{%- for tool in tools %}
|
| 10 |
+
{{- "\n" }}
|
| 11 |
+
{{- tool | tojson }}
|
| 12 |
+
{%- endfor %}
|
| 13 |
+
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
| 14 |
+
{%- else %}
|
| 15 |
+
{%- if messages[0]['role'] == 'system' %}
|
| 16 |
+
{{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
|
| 17 |
+
{%- else %}
|
| 18 |
+
{{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
|
| 19 |
+
{%- endif %}
|
| 20 |
+
{%- endif %}
|
| 21 |
+
{%- for message in messages %}
|
| 22 |
+
{%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
|
| 23 |
+
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
| 24 |
+
{%- elif message.role == "assistant" %}
|
| 25 |
+
{{- '<|im_start|>' + message.role }}
|
| 26 |
+
{%- if message.content %}
|
| 27 |
+
{{- '\n' + message.content }}
|
| 28 |
+
{%- endif %}
|
| 29 |
+
{%- for tool_call in message.tool_calls %}
|
| 30 |
+
{%- if tool_call.function is defined %}
|
| 31 |
+
{%- set tool_call = tool_call.function %}
|
| 32 |
+
{%- endif %}
|
| 33 |
+
{{- '\n<tool_call>\n{"name": "' }}
|
| 34 |
+
{{- tool_call.name }}
|
| 35 |
+
{{- '", "arguments": ' }}
|
| 36 |
+
{{- tool_call.arguments | tojson }}
|
| 37 |
+
{{- '}\n</tool_call>' }}
|
| 38 |
+
{%- endfor %}
|
| 39 |
+
{{- '<|im_end|>\n' }}
|
| 40 |
+
{%- elif message.role == "tool" %}
|
| 41 |
+
{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
|
| 42 |
+
{{- '<|im_start|>user' }}
|
| 43 |
+
{%- endif %}
|
| 44 |
+
{{- '\n<tool_response>\n' }}
|
| 45 |
+
{{- message.content }}
|
| 46 |
+
{{- '\n</tool_response>' }}
|
| 47 |
+
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
| 48 |
+
{{- '<|im_end|>\n' }}
|
| 49 |
+
{%- endif %}
|
| 50 |
+
{%- endif %}
|
| 51 |
+
{%- endfor %}
|
| 52 |
+
{%- if add_generation_prompt %}
|
| 53 |
+
{{- '<|im_start|>assistant\n' }}
|
| 54 |
+
{%- endif %}
|
backend/llama.cpp/models/templates/Qwen-Qwen3-0.6B.jinja
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- if tools %}
|
| 2 |
+
{{- '<|im_start|>system\n' }}
|
| 3 |
+
{%- if messages[0].role == 'system' %}
|
| 4 |
+
{{- messages[0].content + '\n\n' }}
|
| 5 |
+
{%- endif %}
|
| 6 |
+
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
| 7 |
+
{%- for tool in tools %}
|
| 8 |
+
{{- "\n" }}
|
| 9 |
+
{{- tool | tojson }}
|
| 10 |
+
{%- endfor %}
|
| 11 |
+
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
| 12 |
+
{%- else %}
|
| 13 |
+
{%- if messages[0].role == 'system' %}
|
| 14 |
+
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
|
| 15 |
+
{%- endif %}
|
| 16 |
+
{%- endif %}
|
| 17 |
+
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
| 18 |
+
{%- for message in messages[::-1] %}
|
| 19 |
+
{%- set index = (messages|length - 1) - loop.index0 %}
|
| 20 |
+
{%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
|
| 21 |
+
{%- set ns.multi_step_tool = false %}
|
| 22 |
+
{%- set ns.last_query_index = index %}
|
| 23 |
+
{%- endif %}
|
| 24 |
+
{%- endfor %}
|
| 25 |
+
{%- for message in messages %}
|
| 26 |
+
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
|
| 27 |
+
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
| 28 |
+
{%- elif message.role == "assistant" %}
|
| 29 |
+
{%- set content = message.content %}
|
| 30 |
+
{%- set reasoning_content = '' %}
|
| 31 |
+
{%- if message.reasoning_content is defined and message.reasoning_content is not none %}
|
| 32 |
+
{%- set reasoning_content = message.reasoning_content %}
|
| 33 |
+
{%- else %}
|
| 34 |
+
{%- if '</think>' in message.content %}
|
| 35 |
+
{%- set content = message.content.split('</think>')[-1].lstrip('\n') %}
|
| 36 |
+
{%- set reasoning_content = message.content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
| 37 |
+
{%- endif %}
|
| 38 |
+
{%- endif %}
|
| 39 |
+
{%- if loop.index0 > ns.last_query_index %}
|
| 40 |
+
{%- if loop.last or (not loop.last and reasoning_content) %}
|
| 41 |
+
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
|
| 42 |
+
{%- else %}
|
| 43 |
+
{{- '<|im_start|>' + message.role + '\n' + content }}
|
| 44 |
+
{%- endif %}
|
| 45 |
+
{%- else %}
|
| 46 |
+
{{- '<|im_start|>' + message.role + '\n' + content }}
|
| 47 |
+
{%- endif %}
|
| 48 |
+
{%- if message.tool_calls %}
|
| 49 |
+
{%- for tool_call in message.tool_calls %}
|
| 50 |
+
{%- if (loop.first and content) or (not loop.first) %}
|
| 51 |
+
{{- '\n' }}
|
| 52 |
+
{%- endif %}
|
| 53 |
+
{%- if tool_call.function %}
|
| 54 |
+
{%- set tool_call = tool_call.function %}
|
| 55 |
+
{%- endif %}
|
| 56 |
+
{{- '<tool_call>\n{"name": "' }}
|
| 57 |
+
{{- tool_call.name }}
|
| 58 |
+
{{- '", "arguments": ' }}
|
| 59 |
+
{%- if tool_call.arguments is string %}
|
| 60 |
+
{{- tool_call.arguments }}
|
| 61 |
+
{%- else %}
|
| 62 |
+
{{- tool_call.arguments | tojson }}
|
| 63 |
+
{%- endif %}
|
| 64 |
+
{{- '}\n</tool_call>' }}
|
| 65 |
+
{%- endfor %}
|
| 66 |
+
{%- endif %}
|
| 67 |
+
{{- '<|im_end|>\n' }}
|
| 68 |
+
{%- elif message.role == "tool" %}
|
| 69 |
+
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
| 70 |
+
{{- '<|im_start|>user' }}
|
| 71 |
+
{%- endif %}
|
| 72 |
+
{{- '\n<tool_response>\n' }}
|
| 73 |
+
{{- message.content }}
|
| 74 |
+
{{- '\n</tool_response>' }}
|
| 75 |
+
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
| 76 |
+
{{- '<|im_end|>\n' }}
|
| 77 |
+
{%- endif %}
|
| 78 |
+
{%- endif %}
|
| 79 |
+
{%- endfor %}
|
| 80 |
+
{%- if add_generation_prompt %}
|
| 81 |
+
{{- '<|im_start|>assistant\n' }}
|
| 82 |
+
{%- if enable_thinking is defined and enable_thinking is false %}
|
| 83 |
+
{{- '<think>\n\n</think>\n\n' }}
|
| 84 |
+
{%- endif %}
|
| 85 |
+
{%- endif %}
|
backend/llama.cpp/models/templates/Qwen3-Coder.jinja
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{% macro render_extra_keys(json_dict, handled_keys) %}
|
| 2 |
+
{%- if json_dict is mapping %}
|
| 3 |
+
{%- for json_key in json_dict if json_key not in handled_keys %}
|
| 4 |
+
{%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
|
| 5 |
+
{{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
|
| 6 |
+
{%- else %}
|
| 7 |
+
{{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
|
| 8 |
+
{%- endif %}
|
| 9 |
+
{%- endfor %}
|
| 10 |
+
{%- endif %}
|
| 11 |
+
{% endmacro %}
|
| 12 |
+
|
| 13 |
+
{%- if messages[0]["role"] == "system" %}
|
| 14 |
+
{%- set system_message = messages[0]["content"] %}
|
| 15 |
+
{%- set loop_messages = messages[1:] %}
|
| 16 |
+
{%- else %}
|
| 17 |
+
{%- set loop_messages = messages %}
|
| 18 |
+
{%- endif %}
|
| 19 |
+
|
| 20 |
+
{%- if not tools is defined %}
|
| 21 |
+
{%- set tools = [] %}
|
| 22 |
+
{%- endif %}
|
| 23 |
+
|
| 24 |
+
{%- if system_message is defined %}
|
| 25 |
+
{{- "<|im_start|>system\n" + system_message }}
|
| 26 |
+
{%- else %}
|
| 27 |
+
{%- if tools is iterable and tools | length > 0 %}
|
| 28 |
+
{{- "<|im_start|>system\nYou are Qwen, a helpful AI assistant that can interact with a computer to solve tasks." }}
|
| 29 |
+
{%- endif %}
|
| 30 |
+
{%- endif %}
|
| 31 |
+
{%- if tools is iterable and tools | length > 0 %}
|
| 32 |
+
{{- "\n\n# Tools\n\nYou have access to the following tools:\n\n" }}
|
| 33 |
+
{{- "<tools>" }}
|
| 34 |
+
{%- for tool in tools %}
|
| 35 |
+
{%- if tool.function is defined %}
|
| 36 |
+
{%- set tool = tool.function %}
|
| 37 |
+
{%- endif %}
|
| 38 |
+
{{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
|
| 39 |
+
{%- if tool.description is defined %}
|
| 40 |
+
{{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
|
| 41 |
+
{%- endif %}
|
| 42 |
+
{{- '\n<parameters>' }}
|
| 43 |
+
{%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
|
| 44 |
+
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
| 45 |
+
{{- '\n<parameter>' }}
|
| 46 |
+
{{- '\n<name>' ~ param_name ~ '</name>' }}
|
| 47 |
+
{%- if param_fields.type is defined %}
|
| 48 |
+
{{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
|
| 49 |
+
{%- endif %}
|
| 50 |
+
{%- if param_fields.description is defined %}
|
| 51 |
+
{{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
|
| 52 |
+
{%- endif %}
|
| 53 |
+
{%- set handled_keys = ['name', 'type', 'description'] %}
|
| 54 |
+
{{- render_extra_keys(param_fields, handled_keys) }}
|
| 55 |
+
{{- '\n</parameter>' }}
|
| 56 |
+
{%- endfor %}
|
| 57 |
+
{%- endif %}
|
| 58 |
+
{% set handled_keys = ['type', 'properties'] %}
|
| 59 |
+
{{- render_extra_keys(tool.parameters, handled_keys) }}
|
| 60 |
+
{{- '\n</parameters>' }}
|
| 61 |
+
{%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
|
| 62 |
+
{{- render_extra_keys(tool, handled_keys) }}
|
| 63 |
+
{{- '\n</function>' }}
|
| 64 |
+
{%- endfor %}
|
| 65 |
+
{{- "\n</tools>" }}
|
| 66 |
+
{{- '\n\nIf you choose to call a tool ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nvalue_2\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: the tool calling block MUST begin with an opening <tool_call> tag and end with a closing </tool_call> tag.\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
| 67 |
+
{%- endif %}
|
| 68 |
+
{%- if system_message is defined %}
|
| 69 |
+
{{- '<|im_end|>\n' }}
|
| 70 |
+
{%- else %}
|
| 71 |
+
{%- if tools is iterable and tools | length > 0 %}
|
| 72 |
+
{{- '<|im_end|>\n' }}
|
| 73 |
+
{%- endif %}
|
| 74 |
+
{%- endif %}
|
| 75 |
+
{%- for message in loop_messages %}
|
| 76 |
+
{%- if message.role == "assistant" and message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
|
| 77 |
+
{{- '<|im_start|>' + message.role }}
|
| 78 |
+
{%- if message.content is defined and message.content is string and message.content | trim | length > 0 %}
|
| 79 |
+
{{- '\n' + message.content | trim + '\n' }}
|
| 80 |
+
{%- endif %}
|
| 81 |
+
{%- for tool_call in message.tool_calls %}
|
| 82 |
+
{%- if tool_call.function is defined %}
|
| 83 |
+
{%- set tool_call = tool_call.function %}
|
| 84 |
+
{%- endif %}
|
| 85 |
+
{{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 86 |
+
{%- if tool_call.arguments is defined %}
|
| 87 |
+
{%- for args_name, args_value in tool_call.arguments|items %}
|
| 88 |
+
{{- '<parameter=' + args_name + '>\n' }}
|
| 89 |
+
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
| 90 |
+
{{- args_value }}
|
| 91 |
+
{{- '\n</parameter>\n' }}
|
| 92 |
+
{%- endfor %}
|
| 93 |
+
{%- endif %}
|
| 94 |
+
{{- '</function>\n</tool_call>' }}
|
| 95 |
+
{%- endfor %}
|
| 96 |
+
{{- '<|im_end|>\n' }}
|
| 97 |
+
{%- elif message.role == "user" or message.role == "system" or message.role == "assistant" %}
|
| 98 |
+
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
| 99 |
+
{%- elif message.role == "tool" %}
|
| 100 |
+
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
| 101 |
+
{{- '<|im_start|>user\n' }}
|
| 102 |
+
{%- endif %}
|
| 103 |
+
{{- '<tool_response>\n' }}
|
| 104 |
+
{{- message.content }}
|
| 105 |
+
{{- '\n</tool_response>\n' }}
|
| 106 |
+
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
| 107 |
+
{{- '<|im_end|>\n' }}
|
| 108 |
+
{%- elif loop.last %}
|
| 109 |
+
{{- '<|im_end|>\n' }}
|
| 110 |
+
{%- endif %}
|
| 111 |
+
{%- else %}
|
| 112 |
+
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
|
| 113 |
+
{%- endif %}
|
| 114 |
+
{%- endfor %}
|
| 115 |
+
{%- if add_generation_prompt %}
|
| 116 |
+
{{- '<|im_start|>assistant\n' }}
|
| 117 |
+
{%- endif %}
|
backend/llama.cpp/models/templates/Qwen3.5-4B.jinja
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- set image_count = namespace(value=0) %}
|
| 2 |
+
{%- set video_count = namespace(value=0) %}
|
| 3 |
+
{%- macro render_content(content, do_vision_count, is_system_content=false) %}
|
| 4 |
+
{%- if content is string %}
|
| 5 |
+
{{- content }}
|
| 6 |
+
{%- elif content is iterable and content is not mapping %}
|
| 7 |
+
{%- for item in content %}
|
| 8 |
+
{%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
|
| 9 |
+
{%- if is_system_content %}
|
| 10 |
+
{{- raise_exception('System message cannot contain images.') }}
|
| 11 |
+
{%- endif %}
|
| 12 |
+
{%- if do_vision_count %}
|
| 13 |
+
{%- set image_count.value = image_count.value + 1 %}
|
| 14 |
+
{%- endif %}
|
| 15 |
+
{%- if add_vision_id %}
|
| 16 |
+
{{- 'Picture ' ~ image_count.value ~ ': ' }}
|
| 17 |
+
{%- endif %}
|
| 18 |
+
{{- '<|vision_start|><|image_pad|><|vision_end|>' }}
|
| 19 |
+
{%- elif 'video' in item or item.type == 'video' %}
|
| 20 |
+
{%- if is_system_content %}
|
| 21 |
+
{{- raise_exception('System message cannot contain videos.') }}
|
| 22 |
+
{%- endif %}
|
| 23 |
+
{%- if do_vision_count %}
|
| 24 |
+
{%- set video_count.value = video_count.value + 1 %}
|
| 25 |
+
{%- endif %}
|
| 26 |
+
{%- if add_vision_id %}
|
| 27 |
+
{{- 'Video ' ~ video_count.value ~ ': ' }}
|
| 28 |
+
{%- endif %}
|
| 29 |
+
{{- '<|vision_start|><|video_pad|><|vision_end|>' }}
|
| 30 |
+
{%- elif 'text' in item %}
|
| 31 |
+
{{- item.text }}
|
| 32 |
+
{%- else %}
|
| 33 |
+
{{- raise_exception('Unexpected item type in content.') }}
|
| 34 |
+
{%- endif %}
|
| 35 |
+
{%- endfor %}
|
| 36 |
+
{%- elif content is none or content is undefined %}
|
| 37 |
+
{{- '' }}
|
| 38 |
+
{%- else %}
|
| 39 |
+
{{- raise_exception('Unexpected content type.') }}
|
| 40 |
+
{%- endif %}
|
| 41 |
+
{%- endmacro %}
|
| 42 |
+
{%- if not messages %}
|
| 43 |
+
{{- raise_exception('No messages provided.') }}
|
| 44 |
+
{%- endif %}
|
| 45 |
+
{%- if tools and tools is iterable and tools is not mapping %}
|
| 46 |
+
{{- '<|im_start|>system\n' }}
|
| 47 |
+
{{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
|
| 48 |
+
{%- for tool in tools %}
|
| 49 |
+
{{- "\n" }}
|
| 50 |
+
{{- tool | tojson }}
|
| 51 |
+
{%- endfor %}
|
| 52 |
+
{{- "\n</tools>" }}
|
| 53 |
+
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
| 54 |
+
{%- if messages[0].role == 'system' %}
|
| 55 |
+
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
| 56 |
+
{%- if content %}
|
| 57 |
+
{{- '\n\n' + content }}
|
| 58 |
+
{%- endif %}
|
| 59 |
+
{%- endif %}
|
| 60 |
+
{{- '<|im_end|>\n' }}
|
| 61 |
+
{%- else %}
|
| 62 |
+
{%- if messages[0].role == 'system' %}
|
| 63 |
+
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
| 64 |
+
{{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
|
| 65 |
+
{%- endif %}
|
| 66 |
+
{%- endif %}
|
| 67 |
+
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
| 68 |
+
{%- for message in messages[::-1] %}
|
| 69 |
+
{%- set index = (messages|length - 1) - loop.index0 %}
|
| 70 |
+
{%- if ns.multi_step_tool and message.role == "user" %}
|
| 71 |
+
{%- set content = render_content(message.content, false)|trim %}
|
| 72 |
+
{%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
|
| 73 |
+
{%- set ns.multi_step_tool = false %}
|
| 74 |
+
{%- set ns.last_query_index = index %}
|
| 75 |
+
{%- endif %}
|
| 76 |
+
{%- endif %}
|
| 77 |
+
{%- endfor %}
|
| 78 |
+
{%- if ns.multi_step_tool %}
|
| 79 |
+
{{- raise_exception('No user query found in messages.') }}
|
| 80 |
+
{%- endif %}
|
| 81 |
+
{%- for message in messages %}
|
| 82 |
+
{%- set content = render_content(message.content, true)|trim %}
|
| 83 |
+
{%- if message.role == "system" %}
|
| 84 |
+
{%- if not loop.first %}
|
| 85 |
+
{{- raise_exception('System message must be at the beginning.') }}
|
| 86 |
+
{%- endif %}
|
| 87 |
+
{%- elif message.role == "user" %}
|
| 88 |
+
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
| 89 |
+
{%- elif message.role == "assistant" %}
|
| 90 |
+
{%- set reasoning_content = '' %}
|
| 91 |
+
{%- if message.reasoning_content is string %}
|
| 92 |
+
{%- set reasoning_content = message.reasoning_content %}
|
| 93 |
+
{%- else %}
|
| 94 |
+
{%- if '</think>' in content %}
|
| 95 |
+
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
| 96 |
+
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
| 97 |
+
{%- endif %}
|
| 98 |
+
{%- endif %}
|
| 99 |
+
{%- set reasoning_content = reasoning_content|trim %}
|
| 100 |
+
{%- if loop.index0 > ns.last_query_index %}
|
| 101 |
+
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
|
| 102 |
+
{%- else %}
|
| 103 |
+
{{- '<|im_start|>' + message.role + '\n' + content }}
|
| 104 |
+
{%- endif %}
|
| 105 |
+
{%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
|
| 106 |
+
{%- for tool_call in message.tool_calls %}
|
| 107 |
+
{%- if tool_call.function is defined %}
|
| 108 |
+
{%- set tool_call = tool_call.function %}
|
| 109 |
+
{%- endif %}
|
| 110 |
+
{%- if loop.first %}
|
| 111 |
+
{%- if content|trim %}
|
| 112 |
+
{{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 113 |
+
{%- else %}
|
| 114 |
+
{{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 115 |
+
{%- endif %}
|
| 116 |
+
{%- else %}
|
| 117 |
+
{{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 118 |
+
{%- endif %}
|
| 119 |
+
{%- if tool_call.arguments is defined %}
|
| 120 |
+
{%- for args_name, args_value in tool_call.arguments|items %}
|
| 121 |
+
{{- '<parameter=' + args_name + '>\n' }}
|
| 122 |
+
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
| 123 |
+
{{- args_value }}
|
| 124 |
+
{{- '\n</parameter>\n' }}
|
| 125 |
+
{%- endfor %}
|
| 126 |
+
{%- endif %}
|
| 127 |
+
{{- '</function>\n</tool_call>' }}
|
| 128 |
+
{%- endfor %}
|
| 129 |
+
{%- endif %}
|
| 130 |
+
{{- '<|im_end|>\n' }}
|
| 131 |
+
{%- elif message.role == "tool" %}
|
| 132 |
+
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
| 133 |
+
{{- '<|im_start|>user' }}
|
| 134 |
+
{%- endif %}
|
| 135 |
+
{{- '\n<tool_response>\n' }}
|
| 136 |
+
{{- content }}
|
| 137 |
+
{{- '\n</tool_response>' }}
|
| 138 |
+
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
| 139 |
+
{{- '<|im_end|>\n' }}
|
| 140 |
+
{%- elif loop.last %}
|
| 141 |
+
{{- '<|im_end|>\n' }}
|
| 142 |
+
{%- endif %}
|
| 143 |
+
{%- else %}
|
| 144 |
+
{{- raise_exception('Unexpected message role.') }}
|
| 145 |
+
{%- endif %}
|
| 146 |
+
{%- endfor %}
|
| 147 |
+
{%- if add_generation_prompt %}
|
| 148 |
+
{{- '<|im_start|>assistant\n' }}
|
| 149 |
+
{%- if enable_thinking is defined and enable_thinking is false %}
|
| 150 |
+
{{- '<think>\n\n</think>\n\n' }}
|
| 151 |
+
{%- else %}
|
| 152 |
+
{{- '<think>\n' }}
|
| 153 |
+
{%- endif %}
|
| 154 |
+
{%- endif %}
|
backend/llama.cpp/models/templates/README.md
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
These templates can be updated with the following commands:
|
| 2 |
+
|
| 3 |
+
```bash
|
| 4 |
+
./scripts/get_chat_template.py CohereForAI/c4ai-command-r-plus tool_use > models/templates/CohereForAI-c4ai-command-r-plus-tool_use.jinja
|
| 5 |
+
./scripts/get_chat_template.py CohereForAI/c4ai-command-r7b-12-2024 default > models/templates/CohereForAI-c4ai-command-r7b-12-2024-default.jinja
|
| 6 |
+
./scripts/get_chat_template.py CohereForAI/c4ai-command-r7b-12-2024 rag > models/templates/CohereForAI-c4ai-command-r7b-12-2024-rag.jinja
|
| 7 |
+
./scripts/get_chat_template.py CohereForAI/c4ai-command-r7b-12-2024 tool_use > models/templates/CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja
|
| 8 |
+
./scripts/get_chat_template.py deepseek-ai/DeepSeek-R1-Distill-Llama-8B > models/templates/deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja
|
| 9 |
+
./scripts/get_chat_template.py deepseek-ai/DeepSeek-R1-Distill-Qwen-32B > models/templates/deepseek-ai-DeepSeek-R1-Distill-Qwen-32B.jinja
|
| 10 |
+
./scripts/get_chat_template.py fireworks-ai/llama-3-firefunction-v2 > models/templates/fireworks-ai-llama-3-firefunction-v2.jinja
|
| 11 |
+
./scripts/get_chat_template.py google/gemma-2-2b-it > models/templates/google-gemma-2-2b-it.jinja
|
| 12 |
+
./scripts/get_chat_template.py meetkai/functionary-medium-v3.1 > models/templates/meetkai-functionary-medium-v3.1.jinja
|
| 13 |
+
./scripts/get_chat_template.py meetkai/functionary-medium-v3.2 > models/templates/meetkai-functionary-medium-v3.2.jinja
|
| 14 |
+
./scripts/get_chat_template.py meta-llama/Llama-3.1-8B-Instruct > models/templates/meta-llama-Llama-3.1-8B-Instruct.jinja
|
| 15 |
+
./scripts/get_chat_template.py meta-llama/Llama-3.2-3B-Instruct > models/templates/meta-llama-Llama-3.2-3B-Instruct.jinja
|
| 16 |
+
./scripts/get_chat_template.py meta-llama/Llama-3.3-70B-Instruct > models/templates/meta-llama-Llama-3.3-70B-Instruct.jinja
|
| 17 |
+
./scripts/get_chat_template.py microsoft/Phi-3.5-mini-instruct > models/templates/microsoft-Phi-3.5-mini-instruct.jinja
|
| 18 |
+
./scripts/get_chat_template.py mistralai/Mistral-Nemo-Instruct-2407 > models/templates/mistralai-Mistral-Nemo-Instruct-2407.jinja
|
| 19 |
+
./scripts/get_chat_template.py NousResearch/Hermes-2-Pro-Llama-3-8B tool_use > models/templates/NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja
|
| 20 |
+
./scripts/get_chat_template.py NousResearch/Hermes-3-Llama-3.1-8B tool_use > models/templates/NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja
|
| 21 |
+
./scripts/get_chat_template.py Qwen/Qwen2.5-7B-Instruct > models/templates/Qwen-Qwen2.5-7B-Instruct.jinja
|
| 22 |
+
./scripts/get_chat_template.py Qwen/QwQ-32B > models/templates/Qwen-QwQ-32B.jinja
|
| 23 |
+
./scripts/get_chat_template.py Qwen/Qwen3-0.6B > models/templates/Qwen-Qwen3-0.6B.jinja
|
| 24 |
+
./scripts/get_chat_template.py zai-org/GLM-4.5 > models/templates/zai-org-GLM-4.5.jinja
|
| 25 |
+
./scripts/get_chat_template.py deepseek-ai/DeepSeek-V3.1 > models/templates/deepseek-ai-DeepSeek-V3.1.jinja
|
| 26 |
+
```
|
backend/llama.cpp/models/templates/Reka-Edge.jinja
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- macro render_content(content, num_img_tokens, num_video_frames) -%}
|
| 2 |
+
{%- if content is string -%}
|
| 3 |
+
{{- content -}}
|
| 4 |
+
{%- elif content is sequence -%}
|
| 5 |
+
{%- set ns = namespace(out="", prev_was_text=false) -%}
|
| 6 |
+
{%- for item in content -%}
|
| 7 |
+
{%- set item_type = item.get("type") -%}
|
| 8 |
+
{%- if item_type == "text" or item.get("text") is not none -%}
|
| 9 |
+
{%- set text = item.get("text", "") -%}
|
| 10 |
+
{%- if text -%}
|
| 11 |
+
{%- if ns.prev_was_text -%}
|
| 12 |
+
{%- set ns.out = ns.out ~ " " -%}
|
| 13 |
+
{%- endif -%}
|
| 14 |
+
{%- set ns.out = ns.out ~ text -%}
|
| 15 |
+
{%- endif -%}
|
| 16 |
+
{%- set ns.prev_was_text = text != "" -%}
|
| 17 |
+
{%- elif item_type in ["image", "image_url"] or item.get("image") is not none or item.get("image_url") is not none -%}
|
| 18 |
+
{%- set ns.out = ns.out ~ "<image>" ~ ("<REKA_IMG_TOKEN>" * num_img_tokens) ~ "</image>" -%}
|
| 19 |
+
{%- set ns.prev_was_text = false -%}
|
| 20 |
+
{%- elif item_type in ["video", "video_url"] or item.get("video") is not none or item.get("video_url") is not none -%}
|
| 21 |
+
{%- set repeat_tokens = num_img_tokens * num_video_frames -%}
|
| 22 |
+
{%- set ns.out = ns.out ~ "<video>" ~ ("<REKA_IMG_TOKEN>" * repeat_tokens) ~ "</video>" -%}
|
| 23 |
+
{%- set ns.prev_was_text = false -%}
|
| 24 |
+
{%- endif -%}
|
| 25 |
+
{%- endfor -%}
|
| 26 |
+
{{- ns.out -}}
|
| 27 |
+
{%- endif -%}
|
| 28 |
+
{%- endmacro -%}
|
| 29 |
+
{%- set ns = namespace(out="", last_query_index=messages|length - 1) -%}
|
| 30 |
+
{%- for msg in messages[::-1] -%}
|
| 31 |
+
{%- set idx = messages|length - 1 - loop.index0 -%}
|
| 32 |
+
{%- if msg.get("role") == "user" -%}
|
| 33 |
+
{%- set content = msg.get("content", "") -%}
|
| 34 |
+
{%- if not (content is string and content.startswith("<tool_response>") and content.endswith("</tool_response>")) -%}
|
| 35 |
+
{%- set ns.last_query_index = idx -%}
|
| 36 |
+
{%- break -%}
|
| 37 |
+
{%- endif -%}
|
| 38 |
+
{%- endif -%}
|
| 39 |
+
{%- endfor -%}
|
| 40 |
+
{%- set last_query_index = ns.last_query_index -%}
|
| 41 |
+
{%- set num_img_tokens = num_img_tokens | default(64, true) | int -%}
|
| 42 |
+
{%- set num_video_frames = num_video_frames | default(6, true) | int -%}
|
| 43 |
+
{%- set start_idx = 0 -%}
|
| 44 |
+
{%- set system_text = "" -%}
|
| 45 |
+
{%- if messages|length > 0 and messages[0].get("role") in ["system", "developer"] -%}
|
| 46 |
+
{%- set system_text = render_content(messages[0].get("content", ""), num_img_tokens, num_video_frames) -%}
|
| 47 |
+
{%- set start_idx = 1 -%}
|
| 48 |
+
{%- endif -%}
|
| 49 |
+
{%- if tools or system_text -%}
|
| 50 |
+
{%- set preamble_ns = namespace(text="") -%}
|
| 51 |
+
{%- if system_text -%}
|
| 52 |
+
{%- set preamble_ns.text = "system: " ~ system_text -%}
|
| 53 |
+
{%- endif -%}
|
| 54 |
+
{%- if tools -%}
|
| 55 |
+
{%- if preamble_ns.text -%}
|
| 56 |
+
{%- set preamble_ns.text = preamble_ns.text ~ "\n\n" -%}
|
| 57 |
+
{%- else -%}
|
| 58 |
+
{%- set preamble_ns.text = "system: " -%}
|
| 59 |
+
{%- endif -%}
|
| 60 |
+
{%- set preamble_ns.text = preamble_ns.text
|
| 61 |
+
~ "# Tools\n\n"
|
| 62 |
+
~ "You may call one or more functions to assist with the user query.\n\n"
|
| 63 |
+
~ "You are provided with function signatures within <tools></tools> XML tags:\n"
|
| 64 |
+
~ "<tools>" -%}
|
| 65 |
+
{%- for tool in tools -%}
|
| 66 |
+
{%- set preamble_ns.text = preamble_ns.text ~ "\n" ~ (tool | tojson(ensure_ascii=True)) -%}
|
| 67 |
+
{%- endfor -%}
|
| 68 |
+
{%- set preamble_ns.text = preamble_ns.text
|
| 69 |
+
~ "\n</tools>\n\n"
|
| 70 |
+
~ "For each function call, return a json object with function name and arguments "
|
| 71 |
+
~ "within <tool_call></tool_call> XML tags:\n"
|
| 72 |
+
~ "<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call>" -%}
|
| 73 |
+
{%- endif -%}
|
| 74 |
+
{%- set ns.out = ns.out ~ preamble_ns.text ~ "\n\n<sep>" -%}
|
| 75 |
+
{%- endif -%}
|
| 76 |
+
{%- for idx in range(start_idx, messages|length) -%}
|
| 77 |
+
{%- set message = messages[idx] -%}
|
| 78 |
+
{%- set role = message.get("role") -%}
|
| 79 |
+
{%- set content = message.get("content") -%}
|
| 80 |
+
{%- if role == "user" -%}
|
| 81 |
+
{%- set prefix_ns = namespace(value="human: ") -%}
|
| 82 |
+
{%- if content is sequence and content is not string -%}
|
| 83 |
+
{%- for item in content -%}
|
| 84 |
+
{%- if item.get("type") == "text" or item.get("text") is not none -%}
|
| 85 |
+
{%- set text = item.get("text", "") -%}
|
| 86 |
+
{%- if text -%}
|
| 87 |
+
{%- break -%}
|
| 88 |
+
{%- endif -%}
|
| 89 |
+
{%- elif item.get("type") in ["image", "image_url", "video", "video_url"] -%}
|
| 90 |
+
{%- set prefix_ns.value = "human:" -%}
|
| 91 |
+
{%- break -%}
|
| 92 |
+
{%- endif -%}
|
| 93 |
+
{%- endfor -%}
|
| 94 |
+
{%- endif -%}
|
| 95 |
+
{%- set ns.out = ns.out ~ prefix_ns.value ~ render_content(content, num_img_tokens, num_video_frames) ~ "<sep>" -%}
|
| 96 |
+
{%- elif role == "assistant" -%}
|
| 97 |
+
{%- set tool_calls = message.get("tool_calls") -%}
|
| 98 |
+
{%- set content_text = render_content(content, num_img_tokens, num_video_frames) -%}
|
| 99 |
+
{%- set reasoning_text = "" -%}
|
| 100 |
+
{%- if message.get("reasoning_content") is string -%}
|
| 101 |
+
{%- set reasoning_text = message.get("reasoning_content") -%}
|
| 102 |
+
{%- elif "</think>" in content_text -%}
|
| 103 |
+
{%- set reasoning_text = content_text.split("</think>", 1)[0].rstrip("\n").split("<think>")[-1].lstrip("\n") -%}
|
| 104 |
+
{%- set content_text = content_text.split("</think>", 1)[1].lstrip("\n") -%}
|
| 105 |
+
{%- endif -%}
|
| 106 |
+
{%- set ns.out = ns.out ~ "assistant: " -%}
|
| 107 |
+
{%- set include_thinking = enable_thinking is true
|
| 108 |
+
and idx > last_query_index
|
| 109 |
+
and (idx == messages|length - 1 or reasoning_text)
|
| 110 |
+
-%}
|
| 111 |
+
{%- if include_thinking -%}
|
| 112 |
+
{%- set ns.out = ns.out ~ "<think>\n" ~ (reasoning_text.strip() ) ~ "\n</think>\n\n" -%}
|
| 113 |
+
{%- endif -%}
|
| 114 |
+
{%- set ns.out = ns.out ~ content_text -%}
|
| 115 |
+
{%- if tool_calls -%}
|
| 116 |
+
{%- if content_text and not ns.out.endswith("\n") -%}
|
| 117 |
+
{%- set ns.out = ns.out ~ "\n" -%}
|
| 118 |
+
{%- endif -%}
|
| 119 |
+
{%- for tool_call in tool_calls -%}
|
| 120 |
+
{%- if tool_call.get("function") is not none -%}
|
| 121 |
+
{%- set tool_call = tool_call.get("function") -%}
|
| 122 |
+
{%- endif -%}
|
| 123 |
+
{%- set arguments = tool_call.get("arguments", {}) -%}
|
| 124 |
+
{%- if arguments is string -%}
|
| 125 |
+
{%- set arguments_json = arguments -%}
|
| 126 |
+
{%- elif arguments is mapping -%}
|
| 127 |
+
{%- set arguments_json = arguments | tojson(ensure_ascii=True) -%}
|
| 128 |
+
{%- else -%}
|
| 129 |
+
{%- set arguments_json = arguments | tojson(ensure_ascii=True) -%}
|
| 130 |
+
{%- endif -%}
|
| 131 |
+
{%- set ns.out = ns.out
|
| 132 |
+
~ "<tool_call>\n"
|
| 133 |
+
~ "{\"name\": \"" ~ tool_call.get("name", "") ~ "\", \"arguments\": "
|
| 134 |
+
~ arguments_json
|
| 135 |
+
~ "}\n</tool_call>" -%}
|
| 136 |
+
{%- endfor -%}
|
| 137 |
+
{%- endif -%}
|
| 138 |
+
{%- if not (continue_final_message and idx == messages|length - 1) -%}
|
| 139 |
+
{%- set ns.out = ns.out ~ "\n\n<sep>" -%}
|
| 140 |
+
{%- endif -%}
|
| 141 |
+
{%- elif role == "tool" -%}
|
| 142 |
+
{%- if idx == start_idx or messages[idx - 1].get("role") != "tool" -%}
|
| 143 |
+
{%- set ns.out = ns.out ~ "human: " -%}
|
| 144 |
+
{%- endif -%}
|
| 145 |
+
{%- set response_text = render_content(content, num_img_tokens, num_video_frames) -%}
|
| 146 |
+
{%- set ns.out = ns.out ~ "<tool_response>\n" ~ response_text ~ "\n</tool_response>" -%}
|
| 147 |
+
{%- if idx == messages|length - 1 or messages[idx + 1].get("role") != "tool" -%}
|
| 148 |
+
{%- set ns.out = ns.out ~ "<sep>" -%}
|
| 149 |
+
{%- endif -%}
|
| 150 |
+
{%- endif -%}
|
| 151 |
+
{%- endfor -%}
|
| 152 |
+
{%- if add_generation_prompt
|
| 153 |
+
and (messages|length == 0 or messages[-1].get("role") != "assistant")
|
| 154 |
+
-%}
|
| 155 |
+
{%- if enable_thinking is true -%}
|
| 156 |
+
{%- set ns.out = ns.out ~ "assistant: <think>\n" -%}
|
| 157 |
+
{%- else -%}
|
| 158 |
+
{%- set ns.out = ns.out ~ "assistant:" -%}
|
| 159 |
+
{%- endif -%}
|
| 160 |
+
{%- endif -%}
|
| 161 |
+
{{- ns.out -}}
|
backend/llama.cpp/models/templates/StepFun3.5-Flash.jinja
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{% macro render_content(content) %}{% if content is none %}{{- '' }}{% elif content is string %}{{- content }}{% elif content is mapping %}{{- content['value'] if 'value' in content else content['text'] }}{% elif content is iterable %}{% for item in content %}{% if item.type == 'text' %}{{- item['value'] if 'value' in item else item['text'] }}{% elif item.type == 'image' %}<im_patch>{% endif %}{% endfor %}{% endif %}{% endmacro %}
|
| 2 |
+
{{bos_token}}{%- if tools %}
|
| 3 |
+
{{- '<|im_start|>system\n' }}
|
| 4 |
+
{%- if messages[0].role == 'system' %}
|
| 5 |
+
{{- render_content(messages[0].content) + '\n\n' }}
|
| 6 |
+
{%- endif %}
|
| 7 |
+
{{- "# Tools\n\nYou have access to the following functions in JSONSchema format:\n\n<tools>" }}
|
| 8 |
+
{%- for tool in tools %}
|
| 9 |
+
{{- "\n" }}
|
| 10 |
+
{{- tool | tojson(ensure_ascii=False) }}
|
| 11 |
+
{%- endfor %}
|
| 12 |
+
{{- "\n</tools>\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...>\n...\n</function> block must be nested within <tool_call>\n...\n</tool_call> XML tags\n- Required parameters MUST be specified\n</IMPORTANT><|im_end|>\n" }}
|
| 13 |
+
{%- else %}
|
| 14 |
+
{%- if messages[0].role == 'system' %}
|
| 15 |
+
{{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
|
| 16 |
+
{%- endif %}
|
| 17 |
+
{%- endif %}
|
| 18 |
+
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
| 19 |
+
{%- for message in messages[::-1] %}
|
| 20 |
+
{%- set index = (messages|length - 1) - loop.index0 %}
|
| 21 |
+
{%- if ns.multi_step_tool and message.role == "user" and render_content(message.content) is string and not(render_content(message.content).startswith('<tool_response>') and render_content(message.content).endswith('</tool_response>')) %}
|
| 22 |
+
{%- set ns.multi_step_tool = false %}
|
| 23 |
+
{%- set ns.last_query_index = index %}
|
| 24 |
+
{%- endif %}
|
| 25 |
+
{%- endfor %}
|
| 26 |
+
{%- for message in messages %}
|
| 27 |
+
{%- set content = render_content(message.content) %}
|
| 28 |
+
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
|
| 29 |
+
{%- set role_name = 'observation' if (message.role == "system" and not loop.first and message.name == 'observation') else message.role %}
|
| 30 |
+
{{- '<|im_start|>' + role_name + '\n' + content + '<|im_end|>' + '\n' }}
|
| 31 |
+
{%- elif message.role == "assistant" %}
|
| 32 |
+
{%- if message.reasoning_content is string %}
|
| 33 |
+
{%- set reasoning_content = render_content(message.reasoning_content) %}
|
| 34 |
+
{%- else %}
|
| 35 |
+
{%- if '</think>' in content %}
|
| 36 |
+
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
| 37 |
+
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
| 38 |
+
{%- else %}
|
| 39 |
+
{%- set reasoning_content = '' %}
|
| 40 |
+
{%- endif %}
|
| 41 |
+
{%- endif %}
|
| 42 |
+
{%- if loop.index0 > ns.last_query_index %}
|
| 43 |
+
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n' + content }}
|
| 44 |
+
{%- else %}
|
| 45 |
+
{{- '<|im_start|>' + message.role + '\n' + content }}
|
| 46 |
+
{%- endif %}
|
| 47 |
+
{%- if message.tool_calls %}
|
| 48 |
+
{%- for tool_call in message.tool_calls %}
|
| 49 |
+
{%- if tool_call.function is defined %}
|
| 50 |
+
{%- set tool_call = tool_call.function %}
|
| 51 |
+
{%- endif %}
|
| 52 |
+
{{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 53 |
+
{%- if tool_call.arguments is defined %}
|
| 54 |
+
{%- set arguments = tool_call.arguments %}
|
| 55 |
+
{%- for args_name, args_value in arguments|items %}
|
| 56 |
+
{{- '<parameter=' + args_name + '>\n' }}
|
| 57 |
+
{%- set args_value = args_value | tojson(ensure_ascii=False) | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
| 58 |
+
{{- args_value }}
|
| 59 |
+
{{- '\n</parameter>\n' }}
|
| 60 |
+
{%- endfor %}
|
| 61 |
+
{%- endif %}
|
| 62 |
+
{{- '</function>\n</tool_call>' }}
|
| 63 |
+
{%- endfor %}
|
| 64 |
+
{%- endif %}
|
| 65 |
+
{{- '<|im_end|>\n' }}
|
| 66 |
+
{%- elif message.role == "tool" %}
|
| 67 |
+
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
| 68 |
+
{{- '<|im_start|>tool_response\n' }}
|
| 69 |
+
{%- endif %}
|
| 70 |
+
{{- '<tool_response>' }}
|
| 71 |
+
{{- content }}
|
| 72 |
+
{{- '</tool_response>' }}
|
| 73 |
+
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
| 74 |
+
{{- '<|im_end|>\n' }}
|
| 75 |
+
{%- endif %}
|
| 76 |
+
{%- endif %}
|
| 77 |
+
{%- endfor %}
|
| 78 |
+
{%- if add_generation_prompt %}
|
| 79 |
+
{{- '<|im_start|>assistant\n<think>\n' }}
|
| 80 |
+
{%- endif %}
|
backend/llama.cpp/models/templates/moonshotai-Kimi-K2.jinja
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- if tools -%}
|
| 2 |
+
<|im_system|>tool_declare<|im_middle|>{{ tools | tojson }}<|im_end|>
|
| 3 |
+
{%- endif -%}
|
| 4 |
+
{%- for message in messages -%}
|
| 5 |
+
{%- if loop.first and messages[0]['role'] != 'system' -%}
|
| 6 |
+
<|im_system|>system<|im_middle|>You are a helpful assistant<|im_end|>
|
| 7 |
+
{%- endif -%}
|
| 8 |
+
{%- if message['role'] == 'system' -%}
|
| 9 |
+
<|im_system|>system<|im_middle|>
|
| 10 |
+
{%- elif message['role'] == 'user' -%}
|
| 11 |
+
<|im_user|>user<|im_middle|>
|
| 12 |
+
{%- elif message['role'] == 'assistant' -%}
|
| 13 |
+
<|im_assistant|>assistant<|im_middle|>
|
| 14 |
+
{%- elif message['role'] == 'tool' -%}
|
| 15 |
+
<|im_system|>tool<|im_middle|>
|
| 16 |
+
{%- endif -%}
|
| 17 |
+
{%- if message['role'] == 'assistant' and message.get('tool_calls') -%}
|
| 18 |
+
{%- if message['content'] -%}{{ message['content'] }}{%- endif -%}
|
| 19 |
+
<|tool_calls_section_begin|>
|
| 20 |
+
{%- for tool_call in message['tool_calls'] -%}
|
| 21 |
+
{%- set func_name = tool_call['function']['name'] -%}
|
| 22 |
+
{%- set formatted_id = 'functions.' + func_name + ':' + loop.index0|string -%}
|
| 23 |
+
<|tool_call_begin|>{{ formatted_id }}<|tool_call_argument_begin|>{{ tool_call['function']['arguments'] | tojson}}<|tool_call_end|>
|
| 24 |
+
{%- endfor -%}
|
| 25 |
+
<|tool_calls_section_end|>
|
| 26 |
+
{%- elif message['role'] == 'tool' -%}
|
| 27 |
+
## Return of {{ message.tool_call_id }}\n{{ message['content'] }}
|
| 28 |
+
{%- elif message['content'] is string -%}
|
| 29 |
+
{{ message['content'] }}
|
| 30 |
+
{%- elif message['content'] is not none -%}
|
| 31 |
+
{% for content in message['content'] -%}
|
| 32 |
+
{% if content['type'] == 'image' or 'image' in content or 'image_url' in content -%}
|
| 33 |
+
<|media_start|>image<|media_content|><|media_pad|><|media_end|>
|
| 34 |
+
{% else -%}
|
| 35 |
+
{{ content['text'] }}
|
| 36 |
+
{%- endif -%}
|
| 37 |
+
{%- endfor -%}
|
| 38 |
+
{%- endif -%}
|
| 39 |
+
<|im_end|>
|
| 40 |
+
{%- endfor -%}
|
| 41 |
+
{%- if add_generation_prompt -%}
|
| 42 |
+
<|im_assistant|>assistant<|im_middle|>
|
| 43 |
+
{%- endif -%}
|
backend/llama.cpp/models/templates/openai-gpt-oss-120b.jinja
ADDED
|
@@ -0,0 +1,331 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{#-
|
| 2 |
+
In addition to the normal inputs of `messages` and `tools`, this template also accepts the
|
| 3 |
+
following kwargs:
|
| 4 |
+
- "builtin_tools": A list, can contain "browser" and/or "python".
|
| 5 |
+
- "model_identity": A string that optionally describes the model identity.
|
| 6 |
+
- "reasoning_effort": A string that describes the reasoning effort, defaults to "medium".
|
| 7 |
+
#}
|
| 8 |
+
|
| 9 |
+
{#- Tool Definition Rendering ============================================== #}
|
| 10 |
+
{%- macro render_typescript_type(param_spec, required_params, is_nullable=false) -%}
|
| 11 |
+
{%- if param_spec.type == "array" -%}
|
| 12 |
+
{%- if param_spec['items'] -%}
|
| 13 |
+
{%- if param_spec['items']['type'] == "string" -%}
|
| 14 |
+
{{- "string[]" }}
|
| 15 |
+
{%- elif param_spec['items']['type'] == "number" -%}
|
| 16 |
+
{{- "number[]" }}
|
| 17 |
+
{%- elif param_spec['items']['type'] == "integer" -%}
|
| 18 |
+
{{- "number[]" }}
|
| 19 |
+
{%- elif param_spec['items']['type'] == "boolean" -%}
|
| 20 |
+
{{- "boolean[]" }}
|
| 21 |
+
{%- else -%}
|
| 22 |
+
{%- set inner_type = render_typescript_type(param_spec['items'], required_params) -%}
|
| 23 |
+
{%- if inner_type == "object | object" or inner_type|length > 50 -%}
|
| 24 |
+
{{- "any[]" }}
|
| 25 |
+
{%- else -%}
|
| 26 |
+
{{- inner_type + "[]" }}
|
| 27 |
+
{%- endif -%}
|
| 28 |
+
{%- endif -%}
|
| 29 |
+
{%- if param_spec.nullable -%}
|
| 30 |
+
{{- " | null" }}
|
| 31 |
+
{%- endif -%}
|
| 32 |
+
{%- else -%}
|
| 33 |
+
{{- "any[]" }}
|
| 34 |
+
{%- if param_spec.nullable -%}
|
| 35 |
+
{{- " | null" }}
|
| 36 |
+
{%- endif -%}
|
| 37 |
+
{%- endif -%}
|
| 38 |
+
{%- elif param_spec.type is defined and param_spec.type is iterable and param_spec.type is not string and param_spec.type is not mapping and param_spec.type[0] is defined -%}
|
| 39 |
+
{#- Handle array of types like ["object", "object"] from Union[dict, list] #}
|
| 40 |
+
{%- if param_spec.type | length > 1 -%}
|
| 41 |
+
{{- param_spec.type | join(" | ") }}
|
| 42 |
+
{%- else -%}
|
| 43 |
+
{{- param_spec.type[0] }}
|
| 44 |
+
{%- endif -%}
|
| 45 |
+
{%- elif param_spec.oneOf -%}
|
| 46 |
+
{#- Handle oneOf schemas - check for complex unions and fallback to any #}
|
| 47 |
+
{%- set has_object_variants = false -%}
|
| 48 |
+
{%- for variant in param_spec.oneOf -%}
|
| 49 |
+
{%- if variant.type == "object" -%}
|
| 50 |
+
{%- set has_object_variants = true -%}
|
| 51 |
+
{%- endif -%}
|
| 52 |
+
{%- endfor -%}
|
| 53 |
+
{%- if has_object_variants and param_spec.oneOf|length > 1 -%}
|
| 54 |
+
{{- "any" }}
|
| 55 |
+
{%- else -%}
|
| 56 |
+
{%- for variant in param_spec.oneOf -%}
|
| 57 |
+
{{- render_typescript_type(variant, required_params) -}}
|
| 58 |
+
{%- if variant.description %}
|
| 59 |
+
{{- "// " + variant.description }}
|
| 60 |
+
{%- endif -%}
|
| 61 |
+
{%- if variant.default is defined %}
|
| 62 |
+
{{ "// default: " + variant.default|tojson }}
|
| 63 |
+
{%- endif -%}
|
| 64 |
+
{%- if not loop.last %}
|
| 65 |
+
{{- " | " }}
|
| 66 |
+
{% endif -%}
|
| 67 |
+
{%- endfor -%}
|
| 68 |
+
{%- endif -%}
|
| 69 |
+
{%- elif param_spec.type == "string" -%}
|
| 70 |
+
{%- if param_spec.enum -%}
|
| 71 |
+
{{- '"' + param_spec.enum|join('" | "') + '"' -}}
|
| 72 |
+
{%- else -%}
|
| 73 |
+
{{- "string" }}
|
| 74 |
+
{%- if param_spec.nullable %}
|
| 75 |
+
{{- " | null" }}
|
| 76 |
+
{%- endif -%}
|
| 77 |
+
{%- endif -%}
|
| 78 |
+
{%- elif param_spec.type == "number" -%}
|
| 79 |
+
{{- "number" }}
|
| 80 |
+
{%- elif param_spec.type == "integer" -%}
|
| 81 |
+
{{- "number" }}
|
| 82 |
+
{%- elif param_spec.type == "boolean" -%}
|
| 83 |
+
{{- "boolean" }}
|
| 84 |
+
|
| 85 |
+
{%- elif param_spec.type == "object" -%}
|
| 86 |
+
{%- if param_spec.properties -%}
|
| 87 |
+
{{- "{\n" }}
|
| 88 |
+
{%- for prop_name, prop_spec in param_spec.properties.items() -%}
|
| 89 |
+
{{- prop_name -}}
|
| 90 |
+
{%- if prop_name not in (param_spec.required or []) -%}
|
| 91 |
+
{{- "?" }}
|
| 92 |
+
{%- endif -%}
|
| 93 |
+
{{- ": " }}
|
| 94 |
+
{{ render_typescript_type(prop_spec, param_spec.required or []) }}
|
| 95 |
+
{%- if not loop.last -%}
|
| 96 |
+
{{-", " }}
|
| 97 |
+
{%- endif -%}
|
| 98 |
+
{%- endfor -%}
|
| 99 |
+
{{- "}" }}
|
| 100 |
+
{%- else -%}
|
| 101 |
+
{{- "object" }}
|
| 102 |
+
{%- endif -%}
|
| 103 |
+
{%- else -%}
|
| 104 |
+
{{- "any" }}
|
| 105 |
+
{%- endif -%}
|
| 106 |
+
{%- endmacro -%}
|
| 107 |
+
|
| 108 |
+
{%- macro render_tool_namespace(namespace_name, tools) -%}
|
| 109 |
+
{{- "## " + namespace_name + "\n\n" }}
|
| 110 |
+
{{- "namespace " + namespace_name + " {\n\n" }}
|
| 111 |
+
{%- for tool in tools %}
|
| 112 |
+
{%- set tool = tool.function %}
|
| 113 |
+
{{- "// " + tool.description + "\n" }}
|
| 114 |
+
{{- "type "+ tool.name + " = " }}
|
| 115 |
+
{%- if tool.parameters and tool.parameters.properties %}
|
| 116 |
+
{{- "(_: {\n" }}
|
| 117 |
+
{%- for param_name, param_spec in tool.parameters.properties.items() %}
|
| 118 |
+
{%- if param_spec.description %}
|
| 119 |
+
{{- "// " + param_spec.description + "\n" }}
|
| 120 |
+
{%- endif %}
|
| 121 |
+
{{- param_name }}
|
| 122 |
+
{%- if param_name not in (tool.parameters.required or []) -%}
|
| 123 |
+
{{- "?" }}
|
| 124 |
+
{%- endif -%}
|
| 125 |
+
{{- ": " }}
|
| 126 |
+
{{- render_typescript_type(param_spec, tool.parameters.required or []) }}
|
| 127 |
+
{%- if param_spec.default is defined -%}
|
| 128 |
+
{%- if param_spec.enum %}
|
| 129 |
+
{{- ", // default: " + param_spec.default }}
|
| 130 |
+
{%- elif param_spec.oneOf %}
|
| 131 |
+
{{- "// default: " + param_spec.default }}
|
| 132 |
+
{%- else %}
|
| 133 |
+
{{- ", // default: " + param_spec.default|tojson }}
|
| 134 |
+
{%- endif -%}
|
| 135 |
+
{%- endif -%}
|
| 136 |
+
{%- if not loop.last %}
|
| 137 |
+
{{- ",\n" }}
|
| 138 |
+
{%- else %}
|
| 139 |
+
{{- ",\n" }}
|
| 140 |
+
{%- endif -%}
|
| 141 |
+
{%- endfor %}
|
| 142 |
+
{{- "}) => any;\n\n" }}
|
| 143 |
+
{%- else -%}
|
| 144 |
+
{{- "() => any;\n\n" }}
|
| 145 |
+
{%- endif -%}
|
| 146 |
+
{%- endfor %}
|
| 147 |
+
{{- "} // namespace " + namespace_name }}
|
| 148 |
+
{%- endmacro -%}
|
| 149 |
+
|
| 150 |
+
{%- macro render_builtin_tools(browser_tool, python_tool) -%}
|
| 151 |
+
{%- if browser_tool %}
|
| 152 |
+
{{- "## browser\n\n" }}
|
| 153 |
+
{{- "// Tool for browsing.\n" }}
|
| 154 |
+
{{- "// The `cursor` appears in brackets before each browsing display: `[{cursor}]`.\n" }}
|
| 155 |
+
{{- "// Cite information from the tool using the following format:\n" }}
|
| 156 |
+
{{- "// `【{cursor}†L{line_start}(-L{line_end})?】`, for example: `【6†L9-L11】` or `【8†L3】`.\n" }}
|
| 157 |
+
{{- "// Do not quote more than 10 words directly from the tool output.\n" }}
|
| 158 |
+
{{- "// sources=web (default: web)\n" }}
|
| 159 |
+
{{- "namespace browser {\n\n" }}
|
| 160 |
+
{{- "// Searches for information related to `query` and displays `topn` results.\n" }}
|
| 161 |
+
{{- "type search = (_: {\n" }}
|
| 162 |
+
{{- "query: string,\n" }}
|
| 163 |
+
{{- "topn?: number, // default: 10\n" }}
|
| 164 |
+
{{- "source?: string,\n" }}
|
| 165 |
+
{{- "}) => any;\n\n" }}
|
| 166 |
+
{{- "// Opens the link `id` from the page indicated by `cursor` starting at line number `loc`, showing `num_lines` lines.\n" }}
|
| 167 |
+
{{- "// Valid link ids are displayed with the formatting: `【{id}†.*】`.\n" }}
|
| 168 |
+
{{- "// If `cursor` is not provided, the most recent page is implied.\n" }}
|
| 169 |
+
{{- "// If `id` is a string, it is treated as a fully qualified URL associated with `source`.\n" }}
|
| 170 |
+
{{- "// If `loc` is not provided, the viewport will be positioned at the beginning of the document or centered on the most relevant passage, if available.\n" }}
|
| 171 |
+
{{- "// Use this function without `id` to scroll to a new location of an opened page.\n" }}
|
| 172 |
+
{{- "type open = (_: {\n" }}
|
| 173 |
+
{{- "id?: number | string, // default: -1\n" }}
|
| 174 |
+
{{- "cursor?: number, // default: -1\n" }}
|
| 175 |
+
{{- "loc?: number, // default: -1\n" }}
|
| 176 |
+
{{- "num_lines?: number, // default: -1\n" }}
|
| 177 |
+
{{- "view_source?: boolean, // default: false\n" }}
|
| 178 |
+
{{- "source?: string,\n" }}
|
| 179 |
+
{{- "}) => any;\n\n" }}
|
| 180 |
+
{{- "// Finds exact matches of `pattern` in the current page, or the page given by `cursor`.\n" }}
|
| 181 |
+
{{- "type find = (_: {\n" }}
|
| 182 |
+
{{- "pattern: string,\n" }}
|
| 183 |
+
{{- "cursor?: number, // default: -1\n" }}
|
| 184 |
+
{{- "}) => any;\n\n" }}
|
| 185 |
+
{{- "} // namespace browser\n\n" }}
|
| 186 |
+
{%- endif -%}
|
| 187 |
+
|
| 188 |
+
{%- if python_tool %}
|
| 189 |
+
{{- "## python\n\n" }}
|
| 190 |
+
{{- "Use this tool to execute Python code in your chain of thought. The code will not be shown to the user. This tool should be used for internal reasoning, but not for code that is intended to be visible to the user (e.g. when creating plots, tables, or files).\n\n" }}
|
| 191 |
+
{{- "When you send a message containing Python code to python, it will be executed in a stateful Jupyter notebook environment. python will respond with the output of the execution or time out after 120.0 seconds. The drive at '/mnt/data' can be used to save and persist user files. Internet access for this session is UNKNOWN. Depends on the cluster.\n\n" }}
|
| 192 |
+
{%- endif -%}
|
| 193 |
+
{%- endmacro -%}
|
| 194 |
+
|
| 195 |
+
{#- System Message Construction ============================================ #}
|
| 196 |
+
{%- macro build_system_message() -%}
|
| 197 |
+
{%- if model_identity is not defined %}
|
| 198 |
+
{%- set model_identity = "You are ChatGPT, a large language model trained by OpenAI." %}
|
| 199 |
+
{%- endif %}
|
| 200 |
+
{{- model_identity + "\n" }}
|
| 201 |
+
{{- "Knowledge cutoff: 2024-06\n" }}
|
| 202 |
+
{{- "Current date: " + strftime_now("%Y-%m-%d") + "\n\n" }}
|
| 203 |
+
{%- if reasoning_effort is not defined %}
|
| 204 |
+
{%- set reasoning_effort = "medium" %}
|
| 205 |
+
{%- endif %}
|
| 206 |
+
{{- "Reasoning: " + reasoning_effort + "\n\n" }}
|
| 207 |
+
{%- if builtin_tools %}
|
| 208 |
+
{{- "# Tools\n\n" }}
|
| 209 |
+
{%- set available_builtin_tools = namespace(browser=false, python=false) %}
|
| 210 |
+
{%- for tool in builtin_tools %}
|
| 211 |
+
{%- if tool == "browser" %}
|
| 212 |
+
{%- set available_builtin_tools.browser = true %}
|
| 213 |
+
{%- elif tool == "python" %}
|
| 214 |
+
{%- set available_builtin_tools.python = true %}
|
| 215 |
+
{%- endif %}
|
| 216 |
+
{%- endfor %}
|
| 217 |
+
{{- render_builtin_tools(available_builtin_tools.browser, available_builtin_tools.python) }}
|
| 218 |
+
{%- endif -%}
|
| 219 |
+
{{- "# Valid channels: analysis, commentary, final. Channel must be included for every message." }}
|
| 220 |
+
{%- if tools -%}
|
| 221 |
+
{{- "\nCalls to these tools must go to the commentary channel: 'functions'." }}
|
| 222 |
+
{%- endif -%}
|
| 223 |
+
{%- endmacro -%}
|
| 224 |
+
|
| 225 |
+
{#- Main Template Logic ================================================= #}
|
| 226 |
+
{#- Set defaults #}
|
| 227 |
+
|
| 228 |
+
{#- Render system message #}
|
| 229 |
+
{{- "<|start|>system<|message|>" }}
|
| 230 |
+
{{- build_system_message() }}
|
| 231 |
+
{{- "<|end|>" }}
|
| 232 |
+
|
| 233 |
+
{#- Extract developer message #}
|
| 234 |
+
{%- if messages[0].role == "developer" or messages[0].role == "system" %}
|
| 235 |
+
{%- set developer_message = messages[0].content %}
|
| 236 |
+
{%- set loop_messages = messages[1:] %}
|
| 237 |
+
{%- else %}
|
| 238 |
+
{%- set developer_message = "" %}
|
| 239 |
+
{%- set loop_messages = messages %}
|
| 240 |
+
{%- endif %}
|
| 241 |
+
|
| 242 |
+
{#- Render developer message #}
|
| 243 |
+
{%- if developer_message or tools %}
|
| 244 |
+
{{- "<|start|>developer<|message|>" }}
|
| 245 |
+
{%- if developer_message %}
|
| 246 |
+
{{- "# Instructions\n\n" }}
|
| 247 |
+
{{- developer_message }}
|
| 248 |
+
{{- "\n\n" }}
|
| 249 |
+
{%- endif %}
|
| 250 |
+
{%- if tools -%}
|
| 251 |
+
{{- "# Tools\n\n" }}
|
| 252 |
+
{{- render_tool_namespace("functions", tools) }}
|
| 253 |
+
{%- endif -%}
|
| 254 |
+
{{- "<|end|>" }}
|
| 255 |
+
{%- endif %}
|
| 256 |
+
|
| 257 |
+
{#- Render messages #}
|
| 258 |
+
{%- set last_tool_call = namespace(name=none) %}
|
| 259 |
+
{%- for message in loop_messages -%}
|
| 260 |
+
{#- At this point only assistant/user/tool messages should remain #}
|
| 261 |
+
{%- if message.role == 'assistant' -%}
|
| 262 |
+
{#- Checks to ensure the messages are being passed in the format we expect #}
|
| 263 |
+
{%- if "content" in message %}
|
| 264 |
+
{%- if "<|channel|>analysis<|message|>" in message.content or "<|channel|>final<|message|>" in message.content %}
|
| 265 |
+
{{- raise_exception("You have passed a message containing <|channel|> tags in the content field. Instead of doing this, you should pass analysis messages (the string between '<|message|>' and '<|end|>') in the 'thinking' field, and final messages (the string between '<|message|>' and '<|end|>') in the 'content' field.") }}
|
| 266 |
+
{%- endif %}
|
| 267 |
+
{%- endif %}
|
| 268 |
+
{%- if "thinking" in message %}
|
| 269 |
+
{%- if "<|channel|>analysis<|message|>" in message.thinking or "<|channel|>final<|message|>" in message.thinking %}
|
| 270 |
+
{{- raise_exception("You have passed a message containing <|channel|> tags in the thinking field. Instead of doing this, you should pass analysis messages (the string between '<|message|>' and '<|end|>') in the 'thinking' field, and final messages (the string between '<|message|>' and '<|end|>') in the 'content' field.") }}
|
| 271 |
+
{%- endif %}
|
| 272 |
+
{%- endif %}
|
| 273 |
+
{%- if "tool_calls" in message %}
|
| 274 |
+
{#- We need very careful handling here - we want to drop the tool call analysis message if the model #}
|
| 275 |
+
{#- has output a later <|final|> message, but otherwise we want to retain it. This is the only case #}
|
| 276 |
+
{#- when we render CoT/analysis messages in inference. #}
|
| 277 |
+
{%- set future_final_message = namespace(found=false) %}
|
| 278 |
+
{%- for future_message in loop_messages[loop.index:] %}
|
| 279 |
+
{%- if future_message.role == 'assistant' and "tool_calls" not in future_message %}
|
| 280 |
+
{%- set future_final_message.found = true %}
|
| 281 |
+
{%- endif %}
|
| 282 |
+
{%- endfor %}
|
| 283 |
+
{#- We assume max 1 tool call per message, and so we infer the tool call name #}
|
| 284 |
+
{#- in "tool" messages from the most recent assistant tool call name #}
|
| 285 |
+
{%- set tool_call = message.tool_calls[0] %}
|
| 286 |
+
{%- if tool_call.function %}
|
| 287 |
+
{%- set tool_call = tool_call.function %}
|
| 288 |
+
{%- endif %}
|
| 289 |
+
{%- if message.content and message.thinking %}
|
| 290 |
+
{{- raise_exception("Cannot pass both content and thinking in an assistant message with tool calls! Put the analysis message in one or the other, but not both.") }}
|
| 291 |
+
{%- elif message.content and not future_final_message.found %}
|
| 292 |
+
{{- "<|start|>assistant<|channel|>analysis<|message|>" + message.content + "<|end|>" }}
|
| 293 |
+
{%- elif message.thinking and not future_final_message.found %}
|
| 294 |
+
{{- "<|start|>assistant<|channel|>analysis<|message|>" + message.thinking + "<|end|>" }}
|
| 295 |
+
{%- endif %}
|
| 296 |
+
{{- "<|start|>assistant to=" }}
|
| 297 |
+
{{- "functions." + tool_call.name + "<|channel|>commentary " }}
|
| 298 |
+
{{- (tool_call.content_type if tool_call.content_type is defined else "json") + "<|message|>" }}
|
| 299 |
+
{{- tool_call.arguments|tojson }}
|
| 300 |
+
{{- "<|call|>" }}
|
| 301 |
+
{%- set last_tool_call.name = tool_call.name %}
|
| 302 |
+
{%- elif loop.last and not add_generation_prompt %}
|
| 303 |
+
{#- Only render the CoT if the final turn is an assistant turn and add_generation_prompt is false #}
|
| 304 |
+
{#- This is a situation that should only occur in training, never in inference. #}
|
| 305 |
+
{%- if "thinking" in message %}
|
| 306 |
+
{{- "<|start|>assistant<|channel|>analysis<|message|>" + message.thinking + "<|end|>" }}
|
| 307 |
+
{%- endif %}
|
| 308 |
+
{#- <|return|> indicates the end of generation, but <|end|> does not #}
|
| 309 |
+
{#- <|return|> should never be an input to the model, but we include it as the final token #}
|
| 310 |
+
{#- when training, so the model learns to emit it. #}
|
| 311 |
+
{{- "<|start|>assistant<|channel|>final<|message|>" + message.content + "<|return|>" }}
|
| 312 |
+
{%- else %}
|
| 313 |
+
{#- CoT is dropped during all previous turns, so we never render it for inference #}
|
| 314 |
+
{{- "<|start|>assistant<|channel|>final<|message|>" + message.content + "<|end|>" }}
|
| 315 |
+
{%- set last_tool_call.name = none %}
|
| 316 |
+
{%- endif %}
|
| 317 |
+
{%- elif message.role == 'tool' -%}
|
| 318 |
+
{%- if last_tool_call.name is none %}
|
| 319 |
+
{{- raise_exception("Message has tool role, but there was no previous assistant message with a tool call!") }}
|
| 320 |
+
{%- endif %}
|
| 321 |
+
{{- "<|start|>functions." + last_tool_call.name }}
|
| 322 |
+
{{- " to=assistant<|channel|>commentary<|message|>" + message.content|tojson + "<|end|>" }}
|
| 323 |
+
{%- elif message.role == 'user' -%}
|
| 324 |
+
{{- "<|start|>user<|message|>" + message.content + "<|end|>" }}
|
| 325 |
+
{%- endif -%}
|
| 326 |
+
{%- endfor -%}
|
| 327 |
+
|
| 328 |
+
{#- Generation prompt #}
|
| 329 |
+
{%- if add_generation_prompt -%}
|
| 330 |
+
<|start|>assistant
|
| 331 |
+
{%- endif -%}
|
backend/llama.cpp/models/templates/openbmb-MiniCPM5-1B.jinja
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{{- bos_token }}{%- if tools %}
|
| 2 |
+
{%- set tool_definitions %}
|
| 3 |
+
{{- "# Tools\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
| 4 |
+
{%- for tool in tools %}
|
| 5 |
+
{{- "\n" }}
|
| 6 |
+
{{- tool | tojson(ensure_ascii=False) }}
|
| 7 |
+
{%- endfor %}
|
| 8 |
+
{{- '\n</tools>\n\nTool usage guidelines:\n- You may call zero or more functions. If no function calls are needed, just answer normally and do not include any <function ... </function>.\n- When calling a function, return an XML object within <function ... </function> using:\n<function name="function-name"><param name="param-name">param-value</param></function>\n- param-value may be multi-line. If it contains <, & or newline characters, wrap it in a CDATA block: <param name="param-name"><![CDATA[...multi-line value...]]></param>' }}
|
| 9 |
+
{%- endset %}
|
| 10 |
+
|
| 11 |
+
{{- '<|im_start|>system\n' }}
|
| 12 |
+
{%- if messages[0].role == 'system' %}
|
| 13 |
+
{%- if '<tool_def_sep>' in messages[0].content %}
|
| 14 |
+
{{- messages[0].content.replace('<tool_def_sep>', tool_definitions) }}
|
| 15 |
+
{%- else %}
|
| 16 |
+
{{- messages[0].content + '\n\n' + tool_definitions }}
|
| 17 |
+
{%- endif %}
|
| 18 |
+
{%- else %}
|
| 19 |
+
{{- tool_definitions.lstrip() }}
|
| 20 |
+
{%- endif %}
|
| 21 |
+
{{- '<|im_end|>\n' }}
|
| 22 |
+
{%- else %}
|
| 23 |
+
{%- if messages[0].role == 'system' %}
|
| 24 |
+
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
|
| 25 |
+
{%- endif %}
|
| 26 |
+
{%- endif %}
|
| 27 |
+
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
| 28 |
+
{%- for message in messages[::-1] %}
|
| 29 |
+
{%- set index = (messages|length - 1) - loop.index0 %}
|
| 30 |
+
{%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
|
| 31 |
+
{%- set ns.multi_step_tool = false %}
|
| 32 |
+
{%- set ns.last_query_index = index %}
|
| 33 |
+
{%- endif %}
|
| 34 |
+
{%- endfor %}
|
| 35 |
+
{%- for message in messages %}
|
| 36 |
+
{%- if message.content is string %}
|
| 37 |
+
{%- set content = message.content %}
|
| 38 |
+
{%- else %}
|
| 39 |
+
{%- set content = '' %}
|
| 40 |
+
{%- endif %}
|
| 41 |
+
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
|
| 42 |
+
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
| 43 |
+
{%- elif message.role == "assistant" %}
|
| 44 |
+
{%- set reasoning_content = '' %}
|
| 45 |
+
{%- if message.reasoning_content is string %}
|
| 46 |
+
{%- set reasoning_content = message.reasoning_content %}
|
| 47 |
+
{%- else %}
|
| 48 |
+
{%- if '</think>' in content %}
|
| 49 |
+
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
| 50 |
+
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
| 51 |
+
{%- endif %}
|
| 52 |
+
{%- endif %}
|
| 53 |
+
|
| 54 |
+
{%- if message.tool_calls %}
|
| 55 |
+
{%- set content_parts = content.split('<tool_sep>') %}
|
| 56 |
+
{%- set processed_content = content_parts[0] %}
|
| 57 |
+
{%- set tool_calls_count = message.tool_calls|length %}
|
| 58 |
+
{%- set tool_sep_count = content_parts|length - 1 %}
|
| 59 |
+
{%- set min_count = [tool_calls_count, tool_sep_count]|min %}
|
| 60 |
+
|
| 61 |
+
{%- for i in range(1, content_parts|length) %}
|
| 62 |
+
{%- set tool_index = i - 1 %}
|
| 63 |
+
{%- if tool_index < tool_calls_count %}
|
| 64 |
+
{%- set tool_call = message.tool_calls[tool_index] %}
|
| 65 |
+
{%- if tool_call.function %}
|
| 66 |
+
{%- set tool_call = tool_call.function %}
|
| 67 |
+
{%- endif %}
|
| 68 |
+
{%- set single_tool_xml %}
|
| 69 |
+
{{- '<function name="' ~ tool_call.name ~ '">' }}
|
| 70 |
+
{%- if tool_call.arguments %}
|
| 71 |
+
{%- set args_dict = tool_call.arguments %}
|
| 72 |
+
{%- for param_name, param_value in args_dict.items() %}
|
| 73 |
+
{{- '<param name="' ~ param_name ~ '">' }}
|
| 74 |
+
{%- if param_value is string and ('<' in param_value or '&' in param_value or '\n' in param_value) %}
|
| 75 |
+
{{- '<![CDATA[' + param_value + ']]>' }}
|
| 76 |
+
{%- else %}
|
| 77 |
+
{{- param_value }}
|
| 78 |
+
{%- endif %}
|
| 79 |
+
{{- '</param>' }}
|
| 80 |
+
{%- endfor %}
|
| 81 |
+
{%- endif %}
|
| 82 |
+
{{- '</function>' }}
|
| 83 |
+
{%- endset %}
|
| 84 |
+
{%- set processed_content = processed_content + single_tool_xml + content_parts[i] %}
|
| 85 |
+
{%- else %}
|
| 86 |
+
{%- set processed_content = processed_content + content_parts[i] %}
|
| 87 |
+
{%- endif %}
|
| 88 |
+
{%- endfor %}
|
| 89 |
+
|
| 90 |
+
{%- if tool_calls_count > tool_sep_count %}
|
| 91 |
+
{%- for remaining_index in range(tool_sep_count, tool_calls_count) %}
|
| 92 |
+
{%- set tool_call = message.tool_calls[remaining_index] %}
|
| 93 |
+
{%- if tool_call.function %}
|
| 94 |
+
{%- set tool_call = tool_call.function %}
|
| 95 |
+
{%- endif %}
|
| 96 |
+
{%- set remaining_tool_xml %}
|
| 97 |
+
{{- '<function name="' ~ tool_call.name ~ '">' }}
|
| 98 |
+
{%- if tool_call.arguments %}
|
| 99 |
+
{%- set args_dict = tool_call.arguments %}
|
| 100 |
+
{%- for param_name, param_value in args_dict.items() %}
|
| 101 |
+
{{- '<param name="' ~ param_name ~ '">' }}
|
| 102 |
+
{%- if param_value is string and ('<' in param_value or '&' in param_value or '\n' in param_value) %}
|
| 103 |
+
{{- '<![CDATA[' + param_value + ']]>' }}
|
| 104 |
+
{%- else %}
|
| 105 |
+
{{- param_value }}
|
| 106 |
+
{%- endif %}
|
| 107 |
+
{{- '</param>' }}
|
| 108 |
+
{%- endfor %}
|
| 109 |
+
{%- endif %}
|
| 110 |
+
{{- '</function>' }}
|
| 111 |
+
{%- endset %}
|
| 112 |
+
{%- set processed_content = processed_content + remaining_tool_xml %}
|
| 113 |
+
{%- endfor %}
|
| 114 |
+
{%- endif %}
|
| 115 |
+
|
| 116 |
+
{%- set content = processed_content %}
|
| 117 |
+
{%- endif %}
|
| 118 |
+
|
| 119 |
+
{%- if loop.index0 > ns.last_query_index %}
|
| 120 |
+
{%- if reasoning_content %}
|
| 121 |
+
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
|
| 122 |
+
{%- else %}
|
| 123 |
+
{{- '<|im_start|>' + message.role + '\n' + content }}
|
| 124 |
+
{%- endif %}
|
| 125 |
+
{%- else %}
|
| 126 |
+
{{- '<|im_start|>' + message.role + '\n' + content }}
|
| 127 |
+
{%- endif %}
|
| 128 |
+
|
| 129 |
+
{%- if message.tool_calls and not has_tool_sep %}
|
| 130 |
+
{%- for tool_call in message.tool_calls %}
|
| 131 |
+
{%- if (loop.first and content) or (not loop.first) %}
|
| 132 |
+
{{- '\n' }}
|
| 133 |
+
{%- endif %}
|
| 134 |
+
{%- if tool_call.function %}
|
| 135 |
+
{%- set tool_call = tool_call.function %}
|
| 136 |
+
{%- endif %}
|
| 137 |
+
{{- '<function name="' ~ tool_call.name ~ '">' }}
|
| 138 |
+
{%- if tool_call.arguments %}
|
| 139 |
+
{%- set args_dict = tool_call.arguments %}
|
| 140 |
+
{%- for param_name, param_value in args_dict.items() %}
|
| 141 |
+
{{- '<param name="' ~ param_name ~ '">' }}
|
| 142 |
+
{%- if param_value is string and ('<' in param_value or '&' in param_value or '\n' in param_value) %}
|
| 143 |
+
{{- '<![CDATA[' + param_value + ']]>' }}
|
| 144 |
+
{%- else %}
|
| 145 |
+
{{- param_value }}
|
| 146 |
+
{%- endif %}
|
| 147 |
+
{{- '</param>' }}
|
| 148 |
+
{%- endfor %}
|
| 149 |
+
{%- endif %}
|
| 150 |
+
{{- '</function>' }}
|
| 151 |
+
{%- endfor %}
|
| 152 |
+
{%- endif %}
|
| 153 |
+
{{- '<|im_end|>\n' }}
|
| 154 |
+
{%- elif message.role == "tool" %}
|
| 155 |
+
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
| 156 |
+
{{- '<|im_start|>user' }}
|
| 157 |
+
{%- endif %}
|
| 158 |
+
{{- '\n<tool_response>\n' }}
|
| 159 |
+
{%- if message.content is string %}
|
| 160 |
+
{{- content }}
|
| 161 |
+
{%- else %}
|
| 162 |
+
{{- message.content | tojson(ensure_ascii=False) }}
|
| 163 |
+
{%- endif %}
|
| 164 |
+
{{- '\n</tool_response>' }}
|
| 165 |
+
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
| 166 |
+
{{- '<|im_end|>\n' }}
|
| 167 |
+
{%- endif %}
|
| 168 |
+
{%- endif %}
|
| 169 |
+
{%- endfor %}
|
| 170 |
+
{%- if add_generation_prompt %}
|
| 171 |
+
{{- '<|im_start|>assistant\n' }}
|
| 172 |
+
{%- if enable_thinking is defined %}
|
| 173 |
+
{%- if enable_thinking is false %}
|
| 174 |
+
{{- '<think>\n\n</think>\n\n' }}
|
| 175 |
+
{%- elif enable_thinking is true %}
|
| 176 |
+
{{- '<think>\n' }}
|
| 177 |
+
{%- endif %}
|
| 178 |
+
{%- endif %}
|
| 179 |
+
{%- endif %}
|
backend/llama.cpp/models/templates/unsloth-Apriel-1.5.jinja
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{# Unsloth template fixes #}
|
| 2 |
+
{%- set available_tools_string = '' -%}
|
| 3 |
+
{%- set thought_instructions = '' -%}
|
| 4 |
+
{%- set add_tool_id = true -%}
|
| 5 |
+
{%- set tool_output_format = "default" -%}
|
| 6 |
+
{%- if tools is not none and tools|length > 0 -%}
|
| 7 |
+
{%- set available_tools_string -%}
|
| 8 |
+
You are provided with function signatures within <available_tools></available_tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about the arguments. You should infer the argument values from previous user responses and the system message. Here are the available tools:
|
| 9 |
+
<available_tools>
|
| 10 |
+
{% for tool in tools %}
|
| 11 |
+
{{ tool|string }}
|
| 12 |
+
{% endfor %}
|
| 13 |
+
</available_tools>
|
| 14 |
+
{%- endset -%}
|
| 15 |
+
{%- endif -%}
|
| 16 |
+
{%- if tool_output_format is none or tool_output_format == "default" -%}
|
| 17 |
+
{%- set tool_output_instructions -%}
|
| 18 |
+
Return all function calls as a list of json objects within <tool_call></tool_call> XML tags. Each json object should contain a function name and arguments as follows:
|
| 19 |
+
<tool_calls>[{"name": <function-name-1>, "arguments": <args-dict-1>}, {"name": <function-name-2>, "arguments": <args-dict-2>},...]</tool_calls>
|
| 20 |
+
{%- endset -%}
|
| 21 |
+
{%- elif tool_output_format == "yaml" -%}
|
| 22 |
+
{%- set tool_output_instructions -%}
|
| 23 |
+
Return all function calls as a list of yaml objects within <tool_call></tool_call> XML tags. Each yaml object should contain a function name and arguments as follows:
|
| 24 |
+
<tool_calls>
|
| 25 |
+
- name: <function-name-1>
|
| 26 |
+
arguments: <args-dict-1>
|
| 27 |
+
- name: <function-name-2>
|
| 28 |
+
arguments: <args-dict-2>
|
| 29 |
+
...
|
| 30 |
+
</tool_calls>
|
| 31 |
+
{%- endset -%}
|
| 32 |
+
{%- endif -%}
|
| 33 |
+
{%- if add_thoughts -%}
|
| 34 |
+
{%- set thought_instructions -%}
|
| 35 |
+
Prior to generating the function calls, you should generate the reasoning for why you're calling the function. Please generate these reasoning thoughts between <thinking> and </thinking> XML tags.
|
| 36 |
+
{%- endset -%}
|
| 37 |
+
{%- endif -%}
|
| 38 |
+
{{- bos_token -}}
|
| 39 |
+
{%- set reasoning_prompt='You are a thoughtful and systematic AI assistant built by ServiceNow Language Models (SLAM) lab. Before providing an answer, analyze the problem carefully and present your reasoning step by step. After explaining your thought process, provide the final solution in the following format: [BEGIN FINAL RESPONSE] ... [END FINAL RESPONSE].' -%}
|
| 40 |
+
{%- if messages[0]['role'] != 'system' and tools is not none and tools|length > 0 -%}
|
| 41 |
+
{{- '<|system|>\n' + reasoning_prompt + available_tools_string + "\n" + tool_output_instructions + '\n<|end|>\n' -}}
|
| 42 |
+
{%- endif -%}
|
| 43 |
+
{%- if messages|selectattr('role', 'equalto', 'system')|list|length == 0 -%}
|
| 44 |
+
{{- '<|system|>\n' + reasoning_prompt + '\n<|end|>\n' -}}
|
| 45 |
+
{%- endif -%}
|
| 46 |
+
{%- for message in messages -%}
|
| 47 |
+
{%- if message['role'] == 'user' -%}
|
| 48 |
+
{{- '<|user|>\n' }}
|
| 49 |
+
{%- if message['content'] is not string %}
|
| 50 |
+
{%- for chunk in message['content'] %}
|
| 51 |
+
{%- if chunk['type'] == 'text' %}
|
| 52 |
+
{{- chunk['text'] }}
|
| 53 |
+
{%- elif chunk['type'] == 'image' or chunk['type'] == 'image_url'%}
|
| 54 |
+
{{- '[IMG]' }}
|
| 55 |
+
{%- else %}
|
| 56 |
+
{{- raise_exception('Unrecognized content type!') }}
|
| 57 |
+
{%- endif %}
|
| 58 |
+
{%- endfor %}
|
| 59 |
+
{%- else %}
|
| 60 |
+
{{- message['content'] }}
|
| 61 |
+
{%- endif %}
|
| 62 |
+
{{- '\n<|end|>\n' }}
|
| 63 |
+
{%- elif message['role'] == 'content' -%}
|
| 64 |
+
{%- if message['content'] is not string %}
|
| 65 |
+
{{- '<|content|>\n' + message['content'][0]['text'] + '\n<|end|>\n' -}}
|
| 66 |
+
{%- else %}
|
| 67 |
+
{{- '<|content|>\n' + message['content'] + '\n<|end|>\n' -}}
|
| 68 |
+
{%- endif -%}
|
| 69 |
+
{%- elif message['role'] == 'system' -%}
|
| 70 |
+
{%- if message['content'] is not none and message['content']|length > 0 %}
|
| 71 |
+
{%- if message['content'] is string %}
|
| 72 |
+
{%- set system_message = message['content'] %}
|
| 73 |
+
{%- else %}
|
| 74 |
+
{%- set system_message = message['content'][0]['text'] %}
|
| 75 |
+
{%- endif %}
|
| 76 |
+
{%- else %}
|
| 77 |
+
{%- set system_message = '' %}
|
| 78 |
+
{%- endif %}
|
| 79 |
+
{%- if tools is not none and tools|length > 0 -%}
|
| 80 |
+
{{- '<|system|>\n' + reasoning_prompt + system_message + '\n' + available_tools_string + '\n<|end|>\n' -}}
|
| 81 |
+
{%- else -%}
|
| 82 |
+
{{- '<|system|>\n' + reasoning_prompt + system_message + '\n<|end|>\n' -}}
|
| 83 |
+
{%- endif -%}
|
| 84 |
+
{%- elif message['role'] == 'assistant' -%}
|
| 85 |
+
{%- if loop.last -%}
|
| 86 |
+
{%- set add_tool_id = false -%}
|
| 87 |
+
{%- endif -%}
|
| 88 |
+
{{- '<|assistant|>\n' -}}
|
| 89 |
+
{%- if message['content'] is defined and message['content'] is not none and message['content']|length > 0 -%}
|
| 90 |
+
{%- if message['content'] is not string and message['content'][0]['text'] is not none %}
|
| 91 |
+
{{- message['content'][0]['text'] }}
|
| 92 |
+
{%- else %}
|
| 93 |
+
{{- message['content'] -}}
|
| 94 |
+
{%- endif -%}
|
| 95 |
+
{%- elif message['chosen'] is defined and message['chosen'] is not none and message['chosen']|length > 0 -%}
|
| 96 |
+
{{- message['chosen'][0] -}}
|
| 97 |
+
{%- endif -%}
|
| 98 |
+
{%- if add_thoughts and 'thought' in message and message['thought'] is not none -%}
|
| 99 |
+
{{- '<thinking>' + message['thought'] + '</thinking>' -}}
|
| 100 |
+
{%- endif -%}
|
| 101 |
+
{%- if message['tool_calls'] is defined and message['tool_calls'] is not none and message['tool_calls']|length > 0 -%}
|
| 102 |
+
{{- '\n<tool_calls>[' -}}
|
| 103 |
+
{%- for tool_call in message["tool_calls"] -%}
|
| 104 |
+
{{- '{"name": "' + tool_call['function']['name'] + '", "arguments": ' + tool_call['function']['arguments']|tojson -}}
|
| 105 |
+
{%- if add_tool_id == true -%}
|
| 106 |
+
{{- ', "id": "' + tool_call['id'] + '"' -}}
|
| 107 |
+
{%- endif -%}
|
| 108 |
+
{{- '}' -}}
|
| 109 |
+
{%- if not loop.last -%}{{- ', ' -}}{%- endif -%}
|
| 110 |
+
{%- endfor -%}
|
| 111 |
+
{{- ']</tool_calls>' -}}
|
| 112 |
+
{%- endif -%}
|
| 113 |
+
{{- '\n<|end|>\n' + eos_token -}}
|
| 114 |
+
{%- elif message['role'] == 'tool' -%}
|
| 115 |
+
{%- if message['content'] is string %}
|
| 116 |
+
{%- set tool_message = message['content'] %}
|
| 117 |
+
{%- else %}
|
| 118 |
+
{%- set tool_message = message['content'][0]['text'] %}
|
| 119 |
+
{%- endif -%}
|
| 120 |
+
{{- '<|tool_result|>\n' + tool_message|string + '\n<|end|>\n' -}}
|
| 121 |
+
{%- endif -%}
|
| 122 |
+
{%- if loop.last and add_generation_prompt and message['role'] != 'assistant' -%}
|
| 123 |
+
{{- '<|assistant|>\n' -}}
|
| 124 |
+
{%- endif -%}
|
| 125 |
+
{%- endfor -%}
|
| 126 |
+
{# Copyright 2025-present Unsloth. Apache 2.0 License. #}
|
backend/llama.cpp/models/templates/unsloth-mistral-Devstral-Small-2507.jinja
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{#- Copyright 2025-present the Unsloth team. All rights reserved. #}
|
| 2 |
+
{#- Licensed under the Apache License, Version 2.0 (the "License") #}
|
| 3 |
+
{#- Edits made by Unsloth #}
|
| 4 |
+
{%- set default_system_message = 'You are Devstral, a helpful agentic model trained by Mistral AI and using the OpenHands scaffold. You can interact with a computer to solve tasks.\n\n<ROLE>\nYour primary role is to assist users by executing commands, modifying code, and solving technical problems effectively. You should be thorough, methodical, and prioritize quality over speed.\n* If the user asks a question, like \"why is X happening\", don\'t try to fix the problem. Just give an answer to the question.\n</ROLE>\n\n<EFFICIENCY>\n* Each action you take is somewhat expensive. Wherever possible, combine multiple actions into a single action, e.g. combine multiple bash commands into one, using sed and grep to edit/view multiple files at once.\n* When exploring the codebase, use efficient tools like find, grep, and git commands with appropriate filters to minimize unnecessary operations.\n</EFFICIENCY>\n\n<FILE_SYSTEM_GUIDELINES>\n* When a user provides a file path, do NOT assume it\'s relative to the current working directory. First explore the file system to locate the file before working on it.\n* If asked to edit a file, edit the file directly, rather than creating a new file with a different filename.\n* For global search-and-replace operations, consider using `sed` instead of opening file editors multiple times.\n</FILE_SYSTEM_GUIDELINES>\n\n<CODE_QUALITY>\n* Write clean, efficient code with minimal comments. Avoid redundancy in comments: Do not repeat information that can be easily inferred from the code itself.\n* When implementing solutions, focus on making the minimal changes needed to solve the problem.\n* Before implementing any changes, first thoroughly understand the codebase through exploration.\n* If you are adding a lot of code to a function or file, consider splitting the function or file into smaller pieces when appropriate.\n</CODE_QUALITY>\n\n<VERSION_CONTROL>\n* When configuring git credentials, use \"openhands\" as the user.name and \"openhands@all-hands.dev\" as the user.email by default, unless explicitly instructed otherwise.\n* Exercise caution with git operations. Do NOT make potentially dangerous changes (e.g., pushing to main, deleting repositories) unless explicitly asked to do so.\n* When committing changes, use `git status` to see all modified files, and stage all files necessary for the commit. Use `git commit -a` whenever possible.\n* Do NOT commit files that typically shouldn\'t go into version control (e.g., node_modules/, .env files, build directories, cache files, large binaries) unless explicitly instructed by the user.\n* If unsure about committing certain files, check for the presence of .gitignore files or ask the user for clarification.\n</VERSION_CONTROL>\n\n<PULL_REQUESTS>\n* When creating pull requests, create only ONE per session/issue unless explicitly instructed otherwise.\n* When working with an existing PR, update it with new commits rather than creating additional PRs for the same issue.\n* When updating a PR, preserve the original PR title and purpose, updating description only when necessary.\n</PULL_REQUESTS>\n\n<PROBLEM_SOLVING_WORKFLOW>\n1. EXPLORATION: Thoroughly explore relevant files and understand the context before proposing solutions\n2. ANALYSIS: Consider multiple approaches and select the most promising one\n3. TESTING:\n * For bug fixes: Create tests to verify issues before implementing fixes\n * For new features: Consider test-driven development when appropriate\n * If the repository lacks testing infrastructure and implementing tests would require extensive setup, consult with the user before investing time in building testing infrastructure\n * If the environment is not set up to run tests, consult with the user first before investing time to install all dependencies\n4. IMPLEMENTATION: Make focused, minimal changes to address the problem\n5. VERIFICATION: If the environment is set up to run tests, test your implementation thoroughly, including edge cases. If the environment is not set up to run tests, consult with the user first before investing time to run tests.\n</PROBLEM_SOLVING_WORKFLOW>\n\n<SECURITY>\n* Only use GITHUB_TOKEN and other credentials in ways the user has explicitly requested and would expect.\n* Use APIs to work with GitHub or other platforms, unless the user asks otherwise or your task requires browsing.\n</SECURITY>\n\n<ENVIRONMENT_SETUP>\n* When user asks you to run an application, don\'t stop if the application is not installed. Instead, please install the application and run the command again.\n* If you encounter missing dependencies:\n 1. First, look around in the repository for existing dependency files (requirements.txt, pyproject.toml, package.json, Gemfile, etc.)\n 2. If dependency files exist, use them to install all dependencies at once (e.g., `pip install -r requirements.txt`, `npm install`, etc.)\n 3. Only install individual packages directly if no dependency files are found or if only specific packages are needed\n* Similarly, if you encounter missing dependencies for essential tools requested by the user, install them when possible.\n</ENVIRONMENT_SETUP>\n\n<TROUBLESHOOTING>\n* If you\'ve made repeated attempts to solve a problem but tests still fail or the user reports it\'s still broken:\n 1. Step back and reflect on 5-7 different possible sources of the problem\n 2. Assess the likelihood of each possible cause\n 3. Methodically address the most likely causes, starting with the highest probability\n 4. Document your reasoning process\n* When you run into any major issue while executing a plan from the user, please don\'t try to directly work around it. Instead, propose a new plan and confirm with the user before proceeding.\n</TROUBLESHOOTING>' %}
|
| 5 |
+
|
| 6 |
+
{{- bos_token }}
|
| 7 |
+
|
| 8 |
+
{%- if messages[0]['role'] == 'system' %}
|
| 9 |
+
{%- if messages[0]['content'] is string %}
|
| 10 |
+
{%- set system_message = messages[0]['content'] %}
|
| 11 |
+
{%- else %}
|
| 12 |
+
{%- set system_message = messages[0]['content'][0]['text'] %}
|
| 13 |
+
{%- endif %}
|
| 14 |
+
{%- set loop_messages = messages[1:] %}
|
| 15 |
+
{%- else %}
|
| 16 |
+
{%- set system_message = default_system_message %}
|
| 17 |
+
{%- set loop_messages = messages %}
|
| 18 |
+
{%- endif %}
|
| 19 |
+
{{- '[SYSTEM_PROMPT]' + system_message + '[/SYSTEM_PROMPT]' }}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
{#- Tool description appended ONLY to last user message. Edits made by Unsloth #}
|
| 23 |
+
{#- Tool description appended also if last message is tool. Edits made by Unsloth #}
|
| 24 |
+
{%- set tools_description = "" %}
|
| 25 |
+
{%- set has_tools = false %}
|
| 26 |
+
|
| 27 |
+
{%- if tools is defined and tools is not none and tools|length > 0 %}
|
| 28 |
+
|
| 29 |
+
{%- set has_tools = true %}
|
| 30 |
+
{%- set tools_description = "[AVAILABLE_TOOLS]" + (tools | tojson) + "[/AVAILABLE_TOOLS]" %}
|
| 31 |
+
|
| 32 |
+
{{- tools_description }}
|
| 33 |
+
|
| 34 |
+
{%- endif %}
|
| 35 |
+
|
| 36 |
+
{%- for message in loop_messages %}
|
| 37 |
+
{%- if message['role'] == 'user' %}
|
| 38 |
+
|
| 39 |
+
{%- if message['content'] is string %}
|
| 40 |
+
{{- '[INST]' + message['content'] + '[/INST]' }}
|
| 41 |
+
{%- else %}
|
| 42 |
+
{{- '[INST]' }}
|
| 43 |
+
{%- for block in message['content'] %}
|
| 44 |
+
{%- if block['type'] == 'text' %}
|
| 45 |
+
|
| 46 |
+
{#- Original did not have content which is weird. Added by Un-sloth. #}
|
| 47 |
+
{%- if block['text'] is defined %}
|
| 48 |
+
{{- block['text'] }}
|
| 49 |
+
{%- else %}
|
| 50 |
+
{{- block['content'] }}
|
| 51 |
+
{%- endif %}
|
| 52 |
+
|
| 53 |
+
{%- elif block['type'] in ['image', 'image_url'] %}
|
| 54 |
+
{{- '[IMG]' }}
|
| 55 |
+
{%- else %}
|
| 56 |
+
{{- raise_exception('Only text and image blocks are supported in message content!') }}
|
| 57 |
+
{%- endif %}
|
| 58 |
+
{%- endfor %}
|
| 59 |
+
{{- '[/INST]' }}
|
| 60 |
+
{%- endif %}
|
| 61 |
+
|
| 62 |
+
{%- elif message['role'] == 'system' %}
|
| 63 |
+
{%- if message['content'] is string %}
|
| 64 |
+
{{- '[SYSTEM_PROMPT]' + message['content'] + '[/SYSTEM_PROMPT]' }}
|
| 65 |
+
{%- else %}
|
| 66 |
+
{{- '[SYSTEM_PROMPT]' + message['content'][0]['text'] + '[/SYSTEM_PROMPT]' }}
|
| 67 |
+
{%- endif %}
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
{%- elif message['role'] == 'assistant' %}
|
| 71 |
+
{%- if message['content'] is string %}
|
| 72 |
+
{{- message['content'] }}
|
| 73 |
+
{%- else %}
|
| 74 |
+
{{- message['content'][0]['text'] }}
|
| 75 |
+
{%- endif %}
|
| 76 |
+
|
| 77 |
+
{#- If User,Assistant,Tool,Tool we also need to append tools_description. Edits made by Unsloth #}
|
| 78 |
+
|
| 79 |
+
{%- if message['tool_calls'] is defined and message['tool_calls'] is not none %}
|
| 80 |
+
{%- for tool in message['tool_calls'] %}
|
| 81 |
+
{%- set arguments = tool['function']['arguments'] %}
|
| 82 |
+
{%- if arguments is not string %}
|
| 83 |
+
{%- set arguments = arguments|tojson %}
|
| 84 |
+
{%- endif %}
|
| 85 |
+
{#- Must list tool calls AFTER assistant. Edits made by Un-sloth #}
|
| 86 |
+
{{- "[TOOL_CALLS]" + tool['function']['name'] + "[ARGS]" + arguments }}
|
| 87 |
+
{%- endfor %}
|
| 88 |
+
{%- endif %}
|
| 89 |
+
|
| 90 |
+
{{- eos_token }}
|
| 91 |
+
|
| 92 |
+
{%- elif message["role"] == "tool_results" or message["role"] == "tool" %}
|
| 93 |
+
{%- if message.content is defined and message.content.content is defined %}
|
| 94 |
+
{%- set content = message.content.content %}
|
| 95 |
+
{%- else %}
|
| 96 |
+
{%- set content = message.content %}
|
| 97 |
+
{%- endif %}
|
| 98 |
+
{{- "[TOOL_RESULTS]" + content|string + "[/TOOL_RESULTS]" }}
|
| 99 |
+
|
| 100 |
+
{%- else %}
|
| 101 |
+
{{- raise_exception('Only user, systemm assistant and tool roles are supported in the custom template made by Unsloth!') }}
|
| 102 |
+
{%- endif %}
|
| 103 |
+
{%- endfor %}
|
| 104 |
+
{#- Copyright 2025-present the Unsloth team. All rights reserved. #}
|
| 105 |
+
{#- Licensed under the Apache License, Version 2.0 (the "License") #}
|
backend/llama.cpp/models/templates/upstage-Solar-Open-100B.jinja
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{#- ======== Template Parameters ======== #}
|
| 2 |
+
{%- set add_generation_prompt = add_generation_prompt if add_generation_prompt is defined else true %}
|
| 3 |
+
{%- set default_system_prompt = default_system_prompt if default_system_prompt is defined else true %}
|
| 4 |
+
{%- set reasoning_effort = reasoning_effort if reasoning_effort is defined else "high" %}
|
| 5 |
+
{%- set think_render_option = think_render_option if think_render_option is defined else "lastthink" %}
|
| 6 |
+
|
| 7 |
+
{#- ======== System Block State ======== #}
|
| 8 |
+
{%- set sys_ns = namespace(is_first_block=true) -%}
|
| 9 |
+
|
| 10 |
+
{#- ======== Find last user message index ======== #}
|
| 11 |
+
{%- set last_user_idx = namespace(value=-1) -%}
|
| 12 |
+
{%- for message in messages -%}
|
| 13 |
+
{%- if message.role == 'user' -%}
|
| 14 |
+
{%- set last_user_idx.value = loop.index0 -%}
|
| 15 |
+
{%- endif -%}
|
| 16 |
+
{%- endfor -%}
|
| 17 |
+
|
| 18 |
+
{#- ======== System messages renderers ======== #}
|
| 19 |
+
{%- macro render_system_message(user_system_messages) %}
|
| 20 |
+
{%- if default_system_prompt %}
|
| 21 |
+
{%- if not sys_ns.is_first_block %}{{- "\n\n" }}{%- endif %}
|
| 22 |
+
{%- set sys_ns.is_first_block = false %}
|
| 23 |
+
{{- "## Provider System Prompt\n\nYou are Solar Open 100B, a large language model trained by Upstage AI, a Korean startup. Your knowledge cutoff is 2025-07. The current date is " + strftime_now("%Y-%m-%d") + "." }}
|
| 24 |
+
{%- endif -%}
|
| 25 |
+
{%- if user_system_messages %}
|
| 26 |
+
{%- if not sys_ns.is_first_block %}{{- "\n\n" }}{%- endif %}
|
| 27 |
+
{%- set sys_ns.is_first_block = false %}
|
| 28 |
+
{{- "## System Prompt" }}
|
| 29 |
+
{%- for system_message in user_system_messages %}
|
| 30 |
+
{{- "\n\n" }}
|
| 31 |
+
{{- system_message }}
|
| 32 |
+
{%- endfor %}
|
| 33 |
+
{%- endif -%}
|
| 34 |
+
{%- endmacro %}
|
| 35 |
+
|
| 36 |
+
{%- macro render_tool_instruction(tools) %}
|
| 37 |
+
{%- if not sys_ns.is_first_block %}{{- "\n\n" }}{%- endif %}
|
| 38 |
+
{%- set sys_ns.is_first_block = false %}
|
| 39 |
+
{{- "## Tools\n\n### Tool Call Instruction" }}
|
| 40 |
+
{{- "\nYou may invoke one or more tools to assist with the user's query. Available tools are provided in JSON Schema format: <|tools:begin|><|tool:begin|><tools-json-object><|tool:end|>...<|tools:end|>\n" }}
|
| 41 |
+
{{- "\n### Available Tools\n" }}
|
| 42 |
+
{{- "<|tools:begin|>" }}
|
| 43 |
+
{%- for tool in tools %}
|
| 44 |
+
{{- "<|tool:begin|>" }}
|
| 45 |
+
{{- tool.function | tojson }}
|
| 46 |
+
{{- "<|tool:end|>" }}
|
| 47 |
+
{%- endfor %}
|
| 48 |
+
{{- "<|tools:end|>\n" }}
|
| 49 |
+
{{- "\n### Tool Call Format\n" }}
|
| 50 |
+
{{- "For each tool call, return a JSON object with the following structure, enclosed within <|tool_call:begin|> and <|tool_call:end|> tags: \n<|tool_call:begin|><tool-call-id><|tool_call:name|><tool-name><|tool_call:args|><args-json-object><|tool_call:end|>\n" }}
|
| 51 |
+
{{- "- The <tool-call-id> must be a randomly generated string consisting of 10 lowercase letters (a-z) and/or digits (0-9) (e.g., a1b2c3d4e5)\n" }}
|
| 52 |
+
{{- "\n### Tool Response Format\n" }}
|
| 53 |
+
{{- "Each tool is responded by `tool` with the following structure:\n<|tool_response:id|><tool-call-id><|tool_response:name|><tool-name><|tool_response:result|><results><|tool_response:end|>\n" }}
|
| 54 |
+
{{- "- Ensure the <tool-call-id> matches the corresponding tool call" -}}
|
| 55 |
+
{%- endmacro %}
|
| 56 |
+
|
| 57 |
+
{%- macro render_json_response_format_instruction(response_format) %}
|
| 58 |
+
{%- if not sys_ns.is_first_block %}{{- "\n\n" }}{%- endif %}
|
| 59 |
+
{%- set sys_ns.is_first_block = false %}
|
| 60 |
+
{{- "## Output Format Constraint" }}
|
| 61 |
+
{{- "\n\nYour final response should follow the JSON schema: \n[Start of schema]" }}
|
| 62 |
+
{{- response_format }}
|
| 63 |
+
{{- "\n[End of schema]\nPlease ensure your answers adhere to this format and do not contain any unnecessary text." }}
|
| 64 |
+
{%- endmacro %}
|
| 65 |
+
|
| 66 |
+
{%- macro get_tool_name(messages, tool_call_id) %}
|
| 67 |
+
{%- for msg in messages -%}
|
| 68 |
+
{%- if msg.role == 'assistant' and msg.tool_calls -%}
|
| 69 |
+
{%- for tool_call in msg.tool_calls -%}
|
| 70 |
+
{%- if tool_call.id == tool_call_id -%}
|
| 71 |
+
{{- tool_call.function.name }}
|
| 72 |
+
{%- endif -%}
|
| 73 |
+
{%- endfor -%}
|
| 74 |
+
{%- endif -%}
|
| 75 |
+
{%- endfor -%}
|
| 76 |
+
{%- endmacro %}
|
| 77 |
+
|
| 78 |
+
{%- macro render_tool_arguments(tool_arguments) %}
|
| 79 |
+
{%- if tool_arguments is mapping -%}
|
| 80 |
+
{{- tool_arguments | tojson }}
|
| 81 |
+
{%- else -%}
|
| 82 |
+
{{- tool_arguments }}
|
| 83 |
+
{%- endif -%}
|
| 84 |
+
{%- endmacro %}
|
| 85 |
+
|
| 86 |
+
{#- ======== Render system message ======== #}
|
| 87 |
+
{%- set ns = namespace(system_messages=[]) -%}
|
| 88 |
+
{%- for message in messages -%}
|
| 89 |
+
{%- if message.role == 'system' -%}
|
| 90 |
+
{%- set ns.system_messages = ns.system_messages + [message.content] -%}
|
| 91 |
+
{%- endif -%}
|
| 92 |
+
{%- endfor -%}
|
| 93 |
+
|
| 94 |
+
{%- if ns.system_messages or default_system_prompt or tools or response_format -%}
|
| 95 |
+
{{- "<|begin|>system<|content|>" }}
|
| 96 |
+
{{- render_system_message(ns.system_messages) }}
|
| 97 |
+
{%- if tools -%}
|
| 98 |
+
{{- render_tool_instruction(tools) }}
|
| 99 |
+
{%- endif %}
|
| 100 |
+
{%- if response_format -%}
|
| 101 |
+
{{- render_json_response_format_instruction(response_format) }}
|
| 102 |
+
{%- endif %}
|
| 103 |
+
{{- "<|end|>" }}
|
| 104 |
+
{%- endif -%}
|
| 105 |
+
|
| 106 |
+
{#- ======== Render main messages ======== #}
|
| 107 |
+
{%- for message in messages -%}
|
| 108 |
+
{%- if message.role == 'user' -%}
|
| 109 |
+
{{- "<|begin|>user<|content|>" + message.content + "<|end|>" }}
|
| 110 |
+
{%- elif message.role == 'tool' -%}
|
| 111 |
+
{%- set prev_is_tool = loop.index0 > 0 and messages[loop.index0 - 1].role == 'tool' -%}
|
| 112 |
+
{%- set next_is_tool = loop.index0 < (messages | length - 1) and messages[loop.index0 + 1].role == 'tool' -%}
|
| 113 |
+
{%- if not prev_is_tool -%}
|
| 114 |
+
{{- "<|begin|>tool<|tool_response|>" }}
|
| 115 |
+
{%- endif -%}
|
| 116 |
+
{{- "<|tool_response:begin|>" + message.tool_call_id + "<|tool_response:name|>" }}
|
| 117 |
+
{{- get_tool_name(messages, message.tool_call_id) }}
|
| 118 |
+
{{- "<|tool_response:result|>" }}
|
| 119 |
+
{{- message.content }}
|
| 120 |
+
{{- "<|tool_response:end|>" }}
|
| 121 |
+
{%- if not next_is_tool -%}
|
| 122 |
+
{{- "<|end|>" }}
|
| 123 |
+
{%- endif -%}
|
| 124 |
+
{%- elif message.role == 'assistant' -%}
|
| 125 |
+
{#- ======== Assistant Thinking ======== #}
|
| 126 |
+
{%- if think_render_option == "all" -%}
|
| 127 |
+
{%- if message.reasoning -%}
|
| 128 |
+
{{- "<|begin|>assistant<|think|>" + message.reasoning + "<|end|>" }}
|
| 129 |
+
{%- endif -%}
|
| 130 |
+
{%- elif think_render_option == "lastthink" -%}
|
| 131 |
+
{%- if message.reasoning and loop.index0 > last_user_idx.value -%}
|
| 132 |
+
{{- "<|begin|>assistant<|think|>" + message.reasoning + "<|end|>" }}
|
| 133 |
+
{%- endif -%}
|
| 134 |
+
{%- endif -%}
|
| 135 |
+
|
| 136 |
+
{#- ======== Assistant Messages ======== #}
|
| 137 |
+
{%- if message.tool_calls -%}
|
| 138 |
+
{{- "<|begin|>assistant<|tool_calls|>" }}
|
| 139 |
+
{%- for tool_call in message.tool_calls -%}
|
| 140 |
+
{{- "<|tool_call:begin|>" + tool_call.id +"<|tool_call:name|>" + tool_call.function.name + "<|tool_call:args|>" }}
|
| 141 |
+
{{- render_tool_arguments(tool_call.function.arguments) }}
|
| 142 |
+
{{- "<|tool_call:end|>" }}
|
| 143 |
+
{%- endfor -%}
|
| 144 |
+
{{- "<|calls|>" }}
|
| 145 |
+
{%- else -%}
|
| 146 |
+
{{- "<|begin|>assistant<|content|>" + message.content + "<|end|>" }}
|
| 147 |
+
{%- endif -%}
|
| 148 |
+
{%- endif -%}
|
| 149 |
+
{%- endfor -%}
|
| 150 |
+
|
| 151 |
+
{%- if add_generation_prompt -%}
|
| 152 |
+
{%- if reasoning_effort in ["low", "minimal"] -%}
|
| 153 |
+
{{- "<|begin|>assistant<|think|><|end|>" }}
|
| 154 |
+
{%- endif -%}
|
| 155 |
+
{{- "<|begin|>assistant" }}
|
| 156 |
+
{%- endif -%}
|
backend/llama.cpp/mypy.ini
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[mypy]
|
| 2 |
+
strict = true
|
| 3 |
+
allow_untyped_calls = true
|
| 4 |
+
allow_untyped_defs = true
|
| 5 |
+
allow_incomplete_defs = true
|
| 6 |
+
disable_error_code = import-untyped
|
| 7 |
+
warn_return_any = false
|
backend/llama.cpp/pocs/CMakeLists.txt
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# dependencies
|
| 2 |
+
|
| 3 |
+
find_package(Threads REQUIRED)
|
| 4 |
+
|
| 5 |
+
# third-party
|
| 6 |
+
|
| 7 |
+
include_directories(${CMAKE_CURRENT_SOURCE_DIR})
|
| 8 |
+
|
| 9 |
+
if (EMSCRIPTEN)
|
| 10 |
+
else()
|
| 11 |
+
if (NOT GGML_BACKEND_DL)
|
| 12 |
+
add_subdirectory(vdot)
|
| 13 |
+
endif()
|
| 14 |
+
endif()
|
backend/llama.cpp/pocs/vdot/CMakeLists.txt
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
set(TARGET llama-vdot)
|
| 2 |
+
add_executable(${TARGET} vdot.cpp)
|
| 3 |
+
target_link_libraries(${TARGET} PRIVATE llama-common llama ${CMAKE_THREAD_LIBS_INIT})
|
| 4 |
+
target_compile_features(${TARGET} PRIVATE cxx_std_17)
|
| 5 |
+
|
| 6 |
+
set(TARGET llama-q8dot)
|
| 7 |
+
add_executable(${TARGET} q8dot.cpp)
|
| 8 |
+
target_link_libraries(${TARGET} PRIVATE llama-common llama ${CMAKE_THREAD_LIBS_INIT})
|
| 9 |
+
target_compile_features(${TARGET} PRIVATE cxx_std_17)
|
backend/llama.cpp/pocs/vdot/q8dot.cpp
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include <cstdio>
|
| 2 |
+
#include <type_traits>
|
| 3 |
+
#include <vector>
|
| 4 |
+
#include <random>
|
| 5 |
+
#include <chrono>
|
| 6 |
+
#include <cstdlib>
|
| 7 |
+
#include <cmath>
|
| 8 |
+
#include <cassert>
|
| 9 |
+
#include <cstring>
|
| 10 |
+
#include <array>
|
| 11 |
+
#include <type_traits>
|
| 12 |
+
|
| 13 |
+
#include <ggml.h>
|
| 14 |
+
#include <ggml-cpu.h>
|
| 15 |
+
|
| 16 |
+
constexpr int kVecSize = 1 << 16;
|
| 17 |
+
|
| 18 |
+
// Copy-pasted from ggml.c
|
| 19 |
+
#define QK4_0 32
|
| 20 |
+
typedef struct {
|
| 21 |
+
float d; // delta
|
| 22 |
+
uint8_t qs[QK4_0 / 2]; // nibbles / quants
|
| 23 |
+
} block_q4_0;
|
| 24 |
+
static_assert(sizeof(block_q4_0) == sizeof(float) + QK4_0 / 2, "wrong q4_0 block size/padding");
|
| 25 |
+
|
| 26 |
+
#define QK4_1 32
|
| 27 |
+
typedef struct {
|
| 28 |
+
float d; // delta
|
| 29 |
+
float m; // min
|
| 30 |
+
uint8_t qs[QK4_1 / 2]; // nibbles / quants
|
| 31 |
+
} block_q4_1;
|
| 32 |
+
static_assert(sizeof(block_q4_1) == sizeof(float) * 2 + QK4_1 / 2, "wrong q4_1 block size/padding");
|
| 33 |
+
|
| 34 |
+
// Copy-pasted from ggml.c
|
| 35 |
+
#define QK8_0 32
|
| 36 |
+
typedef struct {
|
| 37 |
+
float d; // delta
|
| 38 |
+
float s; // d * sum(qs[i])
|
| 39 |
+
int8_t qs[QK8_0]; // quants
|
| 40 |
+
} block_q8_0;
|
| 41 |
+
static_assert(sizeof(block_q8_0) == 2*sizeof(float) + QK8_0, "wrong q8_0 block size/padding");
|
| 42 |
+
|
| 43 |
+
static_assert(QK4_1 == QK8_0, "QK4_1 and QK8_0 must be the same");
|
| 44 |
+
static_assert(QK4_0 == QK8_0, "QK4_0 and QK8_0 must be the same");
|
| 45 |
+
|
| 46 |
+
template <typename T>
|
| 47 |
+
static void fillQ4blocks(std::vector<T>& blocks, std::mt19937& rndm) {
|
| 48 |
+
for (auto& b : blocks) {
|
| 49 |
+
b.d = 1;
|
| 50 |
+
for (int i=0; i<QK4_1/2; ++i) {
|
| 51 |
+
uint8_t v1 = rndm() >> 28;
|
| 52 |
+
uint8_t v2 = rndm() >> 28;
|
| 53 |
+
b.qs[i] = v1 | (v2 << 4);
|
| 54 |
+
}
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
static void fillQ80blocks(std::vector<block_q8_0>& blocks, std::mt19937& rndm) {
|
| 59 |
+
for (auto& b : blocks) {
|
| 60 |
+
b.d = 1;
|
| 61 |
+
int sum = 0;
|
| 62 |
+
for (int i=0; i<QK8_0; ++i) {
|
| 63 |
+
b.qs[i] = (rndm() >> 24) - 128;
|
| 64 |
+
sum += b.qs[i];
|
| 65 |
+
}
|
| 66 |
+
b.s = b.d * sum;
|
| 67 |
+
}
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
static float simpleDot(const block_q4_0& x, const block_q8_0& y) {
|
| 71 |
+
int s1 = 0; //, s2 = 0;
|
| 72 |
+
for (int i=0; i<QK4_1/2; i+=2) {
|
| 73 |
+
int v1 = x.qs[i+0] & 0xf;
|
| 74 |
+
int v2 = x.qs[i+0] >> 4;
|
| 75 |
+
int v3 = x.qs[i+1] & 0xf;
|
| 76 |
+
int v4 = x.qs[i+1] >> 4;
|
| 77 |
+
int j = 2*i;
|
| 78 |
+
s1 += v1*y.qs[j] + v2*y.qs[j+1] + v3*y.qs[j+2] + v4*y.qs[j+3];
|
| 79 |
+
//s2 += y.qs[j] + y.qs[j+1] + y.qs[j+2] + y.qs[j+3];
|
| 80 |
+
}
|
| 81 |
+
return y.d * x.d * s1 - 8 * x.d * y.s;
|
| 82 |
+
//return y.d * x.d * (s1 - 8 * s2);
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
static float simpleDot(const block_q4_1& x, const block_q8_0& y) {
|
| 86 |
+
int s1 = 0; //, s2 = 0;
|
| 87 |
+
for (int i=0; i<QK4_1/2; i+=2) {
|
| 88 |
+
int v1 = x.qs[i+0] & 0xf;
|
| 89 |
+
int v2 = x.qs[i+0] >> 4;
|
| 90 |
+
int v3 = x.qs[i+1] & 0xf;
|
| 91 |
+
int v4 = x.qs[i+1] >> 4;
|
| 92 |
+
int j = 2*i;
|
| 93 |
+
s1 += v1*y.qs[j] + v2*y.qs[j+1] + v3*y.qs[j+2] + v4*y.qs[j+3];
|
| 94 |
+
//s2 += y.qs[j] + y.qs[j+1] + y.qs[j+2] + y.qs[j+3];
|
| 95 |
+
}
|
| 96 |
+
return y.d * x.d * s1 + y.s * x.m;
|
| 97 |
+
//return y.d * (x.d * s1 + x.m * s2);
|
| 98 |
+
}
|
| 99 |
+
|
| 100 |
+
struct Stat {
|
| 101 |
+
double sum = 0, sumt = 0, sumt2 = 0, maxt = 0;
|
| 102 |
+
int nloop = 0;
|
| 103 |
+
void addResult(double s, double t) {
|
| 104 |
+
sum += s;
|
| 105 |
+
sumt += t; sumt2 += t*t; maxt = std::max(maxt, t);
|
| 106 |
+
++nloop;
|
| 107 |
+
}
|
| 108 |
+
void reportResult(const char* title) const {
|
| 109 |
+
if (nloop < 1) {
|
| 110 |
+
printf("%s(%s): no result\n",__func__,title);
|
| 111 |
+
return;
|
| 112 |
+
}
|
| 113 |
+
printf("============ %s\n",title);
|
| 114 |
+
printf("<dot> = %g\n",sum/nloop);
|
| 115 |
+
auto t = sumt/nloop, dt = sumt2/nloop - t*t;
|
| 116 |
+
if (dt > 0) dt = sqrt(dt);
|
| 117 |
+
printf("<time> = %g +/- %g us. Max. time = %g us.\n",t,dt,maxt);
|
| 118 |
+
}
|
| 119 |
+
};
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
int main(int argc, char** argv) {
|
| 123 |
+
|
| 124 |
+
int nloop = argc > 1 ? atoi(argv[1]) : 10;
|
| 125 |
+
int type = argc > 2 ? atoi(argv[2]) : 1;
|
| 126 |
+
|
| 127 |
+
std::mt19937 rndm(1234);
|
| 128 |
+
|
| 129 |
+
std::vector<block_q4_1> x41;
|
| 130 |
+
std::vector<block_q4_0> x40;
|
| 131 |
+
std::vector<block_q8_0> y(kVecSize);
|
| 132 |
+
if (type == 0) x40.resize(kVecSize);
|
| 133 |
+
else {
|
| 134 |
+
x41.resize(kVecSize);
|
| 135 |
+
for (auto& b : x41) b.m = 1;
|
| 136 |
+
}
|
| 137 |
+
|
| 138 |
+
auto ggml_type = type == 0 ? GGML_TYPE_Q4_0 : GGML_TYPE_Q4_1;
|
| 139 |
+
|
| 140 |
+
const auto * funcs = ggml_get_type_traits_cpu(ggml_type);
|
| 141 |
+
|
| 142 |
+
Stat simple, ggml;
|
| 143 |
+
|
| 144 |
+
for (int iloop=0; iloop<nloop; ++iloop) {
|
| 145 |
+
|
| 146 |
+
if (type == 0) fillQ4blocks(x40, rndm);
|
| 147 |
+
else fillQ4blocks(x41, rndm);
|
| 148 |
+
fillQ80blocks(y, rndm);
|
| 149 |
+
|
| 150 |
+
auto t1 = std::chrono::high_resolution_clock::now();
|
| 151 |
+
double s = 0;
|
| 152 |
+
if (type == 0) for (int i=0; i<kVecSize; ++i) s += simpleDot(x40[i], y[i]);
|
| 153 |
+
else for (int i=0; i<kVecSize; ++i) s += simpleDot(x41[i], y[i]);
|
| 154 |
+
auto t2 = std::chrono::high_resolution_clock::now();
|
| 155 |
+
auto t = 1e-3*std::chrono::duration_cast<std::chrono::nanoseconds>(t2-t1).count();
|
| 156 |
+
if (iloop > 3) simple.addResult(s, t);
|
| 157 |
+
|
| 158 |
+
t1 = std::chrono::high_resolution_clock::now();
|
| 159 |
+
float fs;
|
| 160 |
+
if (type == 0) funcs->vec_dot(kVecSize * QK4_1, &fs, 0, x40.data(), 0, y.data(), 0, 1);
|
| 161 |
+
else funcs->vec_dot(kVecSize * QK4_1, &fs, 0, x41.data(), 0, y.data(), 0, 1);
|
| 162 |
+
t2 = std::chrono::high_resolution_clock::now();
|
| 163 |
+
t = 1e-3*std::chrono::duration_cast<std::chrono::nanoseconds>(t2-t1).count();
|
| 164 |
+
if (iloop > 3) ggml.addResult(fs, t);
|
| 165 |
+
|
| 166 |
+
}
|
| 167 |
+
|
| 168 |
+
// Report the time (and the average of the dot products so the compiler does not come up with the idea
|
| 169 |
+
// of optimizing away the function calls after figuring that the result is not used).
|
| 170 |
+
simple.reportResult("Simple");
|
| 171 |
+
ggml.reportResult("ggml");
|
| 172 |
+
return 0;
|
| 173 |
+
}
|
backend/llama.cpp/pocs/vdot/vdot.cpp
ADDED
|
@@ -0,0 +1,311 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include <cstdio>
|
| 2 |
+
#include <vector>
|
| 3 |
+
#include <random>
|
| 4 |
+
#include <chrono>
|
| 5 |
+
#include <cstdlib>
|
| 6 |
+
#include <cmath>
|
| 7 |
+
#include <cassert>
|
| 8 |
+
#include <cstring>
|
| 9 |
+
#include <array>
|
| 10 |
+
|
| 11 |
+
#include <ggml.h>
|
| 12 |
+
#include <ggml-cpu.h>
|
| 13 |
+
|
| 14 |
+
#if defined(_MSC_VER)
|
| 15 |
+
#pragma warning(disable: 4244 4267) // possible loss of data
|
| 16 |
+
#endif
|
| 17 |
+
|
| 18 |
+
constexpr int kVecSize = 1 << 18;
|
| 19 |
+
|
| 20 |
+
static float drawFromGaussianPdf(std::mt19937& rndm) {
|
| 21 |
+
constexpr double kScale = 1./(1. + std::mt19937::max());
|
| 22 |
+
constexpr double kTwoPiTimesScale = 6.28318530717958647692*kScale;
|
| 23 |
+
static float lastX;
|
| 24 |
+
static bool haveX = false;
|
| 25 |
+
if (haveX) { haveX = false; return lastX; }
|
| 26 |
+
auto r = sqrt(-2*log(1 - kScale*rndm()));
|
| 27 |
+
auto phi = kTwoPiTimesScale * rndm();
|
| 28 |
+
lastX = r*sin(phi);
|
| 29 |
+
haveX = true;
|
| 30 |
+
return r*cos(phi);
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
static void fillRandomGaussianFloats(std::vector<float>& values, std::mt19937& rndm, float mean = 0) {
|
| 34 |
+
for (auto& v : values) v = mean + drawFromGaussianPdf(rndm);
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
// Copy-pasted from ggml.c
|
| 38 |
+
#define QK4_0 32
|
| 39 |
+
typedef struct {
|
| 40 |
+
float d; // delta
|
| 41 |
+
uint8_t qs[QK4_0 / 2]; // nibbles / quants
|
| 42 |
+
} block_q4_0;
|
| 43 |
+
static_assert(sizeof(block_q4_0) == sizeof(float) + QK4_0 / 2, "wrong q4_0 block size/padding");
|
| 44 |
+
|
| 45 |
+
#define QK4_1 32
|
| 46 |
+
typedef struct {
|
| 47 |
+
float d; // delta
|
| 48 |
+
float m; // min
|
| 49 |
+
uint8_t qs[QK4_1 / 2]; // nibbles / quants
|
| 50 |
+
} block_q4_1;
|
| 51 |
+
static_assert(sizeof(block_q4_1) == sizeof(float) * 2 + QK4_1 / 2, "wrong q4_1 block size/padding");
|
| 52 |
+
|
| 53 |
+
// Copy-pasted from ggml.c
|
| 54 |
+
#define QK8_0 32
|
| 55 |
+
typedef struct {
|
| 56 |
+
float d; // delta
|
| 57 |
+
int8_t qs[QK8_0]; // quants
|
| 58 |
+
} block_q8_0;
|
| 59 |
+
static_assert(sizeof(block_q8_0) == sizeof(float) + QK8_0, "wrong q8_0 block size/padding");
|
| 60 |
+
|
| 61 |
+
// "Scalar" dot product between the quantized vector x and float vector y
|
| 62 |
+
inline double dot(int n, const block_q4_0* x, const float* y) {
|
| 63 |
+
const static float kValues[16] = {-8.f, -7.f, -6.f, -5.f, -4.f, -3.f, -2.f, -1.f, 0.f, 1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f};
|
| 64 |
+
constexpr uint32_t kMask1 = 0x0f0f0f0f;
|
| 65 |
+
uint32_t u1, u2;
|
| 66 |
+
auto q1 = (const uint8_t*)&u1;
|
| 67 |
+
auto q2 = (const uint8_t*)&u2;
|
| 68 |
+
double sum = 0;
|
| 69 |
+
for (int i=0; i<n; ++i) {
|
| 70 |
+
float d = x->d;
|
| 71 |
+
auto u = (const uint32_t*)x->qs;
|
| 72 |
+
float s = 0;
|
| 73 |
+
for (int k=0; k<4; ++k) {
|
| 74 |
+
u1 = u[k] & kMask1;
|
| 75 |
+
u2 = (u[k] >> 4) & kMask1;
|
| 76 |
+
s += y[0]*kValues[q1[0]] + y[1]*kValues[q2[0]] +
|
| 77 |
+
y[2]*kValues[q1[1]] + y[3]*kValues[q2[1]] +
|
| 78 |
+
y[4]*kValues[q1[2]] + y[5]*kValues[q2[2]] +
|
| 79 |
+
y[6]*kValues[q1[3]] + y[7]*kValues[q2[3]];
|
| 80 |
+
y += 8;
|
| 81 |
+
}
|
| 82 |
+
sum += s*d;
|
| 83 |
+
++x;
|
| 84 |
+
}
|
| 85 |
+
return sum;
|
| 86 |
+
}
|
| 87 |
+
// Alternative version of the above. Faster on my Mac (~45 us vs ~55 us per dot product),
|
| 88 |
+
// but about the same on X86_64 (Ryzen 7950X CPU).
|
| 89 |
+
inline double dot3(int n, const block_q4_0* x, const float* y) {
|
| 90 |
+
const static std::pair<float,float> kValues[256] = {
|
| 91 |
+
{-8.f, -8.f}, {-7.f, -8.f}, {-6.f, -8.f}, {-5.f, -8.f}, {-4.f, -8.f}, {-3.f, -8.f}, {-2.f, -8.f}, {-1.f, -8.f},
|
| 92 |
+
{ 0.f, -8.f}, { 1.f, -8.f}, { 2.f, -8.f}, { 3.f, -8.f}, { 4.f, -8.f}, { 5.f, -8.f}, { 6.f, -8.f}, { 7.f, -8.f},
|
| 93 |
+
{-8.f, -7.f}, {-7.f, -7.f}, {-6.f, -7.f}, {-5.f, -7.f}, {-4.f, -7.f}, {-3.f, -7.f}, {-2.f, -7.f}, {-1.f, -7.f},
|
| 94 |
+
{ 0.f, -7.f}, { 1.f, -7.f}, { 2.f, -7.f}, { 3.f, -7.f}, { 4.f, -7.f}, { 5.f, -7.f}, { 6.f, -7.f}, { 7.f, -7.f},
|
| 95 |
+
{-8.f, -6.f}, {-7.f, -6.f}, {-6.f, -6.f}, {-5.f, -6.f}, {-4.f, -6.f}, {-3.f, -6.f}, {-2.f, -6.f}, {-1.f, -6.f},
|
| 96 |
+
{ 0.f, -6.f}, { 1.f, -6.f}, { 2.f, -6.f}, { 3.f, -6.f}, { 4.f, -6.f}, { 5.f, -6.f}, { 6.f, -6.f}, { 7.f, -6.f},
|
| 97 |
+
{-8.f, -5.f}, {-7.f, -5.f}, {-6.f, -5.f}, {-5.f, -5.f}, {-4.f, -5.f}, {-3.f, -5.f}, {-2.f, -5.f}, {-1.f, -5.f},
|
| 98 |
+
{ 0.f, -5.f}, { 1.f, -5.f}, { 2.f, -5.f}, { 3.f, -5.f}, { 4.f, -5.f}, { 5.f, -5.f}, { 6.f, -5.f}, { 7.f, -5.f},
|
| 99 |
+
{-8.f, -4.f}, {-7.f, -4.f}, {-6.f, -4.f}, {-5.f, -4.f}, {-4.f, -4.f}, {-3.f, -4.f}, {-2.f, -4.f}, {-1.f, -4.f},
|
| 100 |
+
{ 0.f, -4.f}, { 1.f, -4.f}, { 2.f, -4.f}, { 3.f, -4.f}, { 4.f, -4.f}, { 5.f, -4.f}, { 6.f, -4.f}, { 7.f, -4.f},
|
| 101 |
+
{-8.f, -3.f}, {-7.f, -3.f}, {-6.f, -3.f}, {-5.f, -3.f}, {-4.f, -3.f}, {-3.f, -3.f}, {-2.f, -3.f}, {-1.f, -3.f},
|
| 102 |
+
{ 0.f, -3.f}, { 1.f, -3.f}, { 2.f, -3.f}, { 3.f, -3.f}, { 4.f, -3.f}, { 5.f, -3.f}, { 6.f, -3.f}, { 7.f, -3.f},
|
| 103 |
+
{-8.f, -2.f}, {-7.f, -2.f}, {-6.f, -2.f}, {-5.f, -2.f}, {-4.f, -2.f}, {-3.f, -2.f}, {-2.f, -2.f}, {-1.f, -2.f},
|
| 104 |
+
{ 0.f, -2.f}, { 1.f, -2.f}, { 2.f, -2.f}, { 3.f, -2.f}, { 4.f, -2.f}, { 5.f, -2.f}, { 6.f, -2.f}, { 7.f, -2.f},
|
| 105 |
+
{-8.f, -1.f}, {-7.f, -1.f}, {-6.f, -1.f}, {-5.f, -1.f}, {-4.f, -1.f}, {-3.f, -1.f}, {-2.f, -1.f}, {-1.f, -1.f},
|
| 106 |
+
{ 0.f, -1.f}, { 1.f, -1.f}, { 2.f, -1.f}, { 3.f, -1.f}, { 4.f, -1.f}, { 5.f, -1.f}, { 6.f, -1.f}, { 7.f, -1.f},
|
| 107 |
+
{-8.f, 0.f}, {-7.f, 0.f}, {-6.f, 0.f}, {-5.f, 0.f}, {-4.f, 0.f}, {-3.f, 0.f}, {-2.f, 0.f}, {-1.f, 0.f},
|
| 108 |
+
{ 0.f, 0.f}, { 1.f, 0.f}, { 2.f, 0.f}, { 3.f, 0.f}, { 4.f, 0.f}, { 5.f, 0.f}, { 6.f, 0.f}, { 7.f, 0.f},
|
| 109 |
+
{-8.f, 1.f}, {-7.f, 1.f}, {-6.f, 1.f}, {-5.f, 1.f}, {-4.f, 1.f}, {-3.f, 1.f}, {-2.f, 1.f}, {-1.f, 1.f},
|
| 110 |
+
{ 0.f, 1.f}, { 1.f, 1.f}, { 2.f, 1.f}, { 3.f, 1.f}, { 4.f, 1.f}, { 5.f, 1.f}, { 6.f, 1.f}, { 7.f, 1.f},
|
| 111 |
+
{-8.f, 2.f}, {-7.f, 2.f}, {-6.f, 2.f}, {-5.f, 2.f}, {-4.f, 2.f}, {-3.f, 2.f}, {-2.f, 2.f}, {-1.f, 2.f},
|
| 112 |
+
{ 0.f, 2.f}, { 1.f, 2.f}, { 2.f, 2.f}, { 3.f, 2.f}, { 4.f, 2.f}, { 5.f, 2.f}, { 6.f, 2.f}, { 7.f, 2.f},
|
| 113 |
+
{-8.f, 3.f}, {-7.f, 3.f}, {-6.f, 3.f}, {-5.f, 3.f}, {-4.f, 3.f}, {-3.f, 3.f}, {-2.f, 3.f}, {-1.f, 3.f},
|
| 114 |
+
{ 0.f, 3.f}, { 1.f, 3.f}, { 2.f, 3.f}, { 3.f, 3.f}, { 4.f, 3.f}, { 5.f, 3.f}, { 6.f, 3.f}, { 7.f, 3.f},
|
| 115 |
+
{-8.f, 4.f}, {-7.f, 4.f}, {-6.f, 4.f}, {-5.f, 4.f}, {-4.f, 4.f}, {-3.f, 4.f}, {-2.f, 4.f}, {-1.f, 4.f},
|
| 116 |
+
{ 0.f, 4.f}, { 1.f, 4.f}, { 2.f, 4.f}, { 3.f, 4.f}, { 4.f, 4.f}, { 5.f, 4.f}, { 6.f, 4.f}, { 7.f, 4.f},
|
| 117 |
+
{-8.f, 5.f}, {-7.f, 5.f}, {-6.f, 5.f}, {-5.f, 5.f}, {-4.f, 5.f}, {-3.f, 5.f}, {-2.f, 5.f}, {-1.f, 5.f},
|
| 118 |
+
{ 0.f, 5.f}, { 1.f, 5.f}, { 2.f, 5.f}, { 3.f, 5.f}, { 4.f, 5.f}, { 5.f, 5.f}, { 6.f, 5.f}, { 7.f, 5.f},
|
| 119 |
+
{-8.f, 6.f}, {-7.f, 6.f}, {-6.f, 6.f}, {-5.f, 6.f}, {-4.f, 6.f}, {-3.f, 6.f}, {-2.f, 6.f}, {-1.f, 6.f},
|
| 120 |
+
{ 0.f, 6.f}, { 1.f, 6.f}, { 2.f, 6.f}, { 3.f, 6.f}, { 4.f, 6.f}, { 5.f, 6.f}, { 6.f, 6.f}, { 7.f, 6.f},
|
| 121 |
+
{-8.f, 7.f}, {-7.f, 7.f}, {-6.f, 7.f}, {-5.f, 7.f}, {-4.f, 7.f}, {-3.f, 7.f}, {-2.f, 7.f}, {-1.f, 7.f},
|
| 122 |
+
{ 0.f, 7.f}, { 1.f, 7.f}, { 2.f, 7.f}, { 3.f, 7.f}, { 4.f, 7.f}, { 5.f, 7.f}, { 6.f, 7.f}, { 7.f, 7.f}
|
| 123 |
+
};
|
| 124 |
+
double sum = 0;
|
| 125 |
+
for (int i=0; i<n; ++i) {
|
| 126 |
+
float d = x->d;
|
| 127 |
+
auto q = x->qs;
|
| 128 |
+
float s = 0;
|
| 129 |
+
for (int k=0; k<4; ++k) {
|
| 130 |
+
s += y[0]*kValues[q[0]].first + y[1]*kValues[q[0]].second +
|
| 131 |
+
y[2]*kValues[q[1]].first + y[3]*kValues[q[1]].second +
|
| 132 |
+
y[4]*kValues[q[2]].first + y[5]*kValues[q[2]].second +
|
| 133 |
+
y[6]*kValues[q[3]].first + y[7]*kValues[q[3]].second;
|
| 134 |
+
y += 8; q += 4;
|
| 135 |
+
}
|
| 136 |
+
sum += s*d;
|
| 137 |
+
++x;
|
| 138 |
+
}
|
| 139 |
+
return sum;
|
| 140 |
+
}
|
| 141 |
+
|
| 142 |
+
inline double dot41(int n, const block_q4_1* x, const float* y) {
|
| 143 |
+
const static float kValues[16] = {0.f, 1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.f, 9.f, 10.f, 11.f, 12.f, 13.f, 14.f, 15.f};
|
| 144 |
+
constexpr uint32_t kMask1 = 0x0f0f0f0f;
|
| 145 |
+
uint32_t u1, u2;
|
| 146 |
+
auto q1 = (const uint8_t*)&u1;
|
| 147 |
+
auto q2 = (const uint8_t*)&u2;
|
| 148 |
+
double sum = 0;
|
| 149 |
+
for (int i=0; i<n; ++i) {
|
| 150 |
+
auto u = (const uint32_t*)x->qs;
|
| 151 |
+
float s = 0, s1 = 0;
|
| 152 |
+
for (int k=0; k<4; ++k) {
|
| 153 |
+
u1 = u[k] & kMask1;
|
| 154 |
+
u2 = (u[k] >> 4) & kMask1;
|
| 155 |
+
s += y[0]*kValues[q1[0]] + y[1]*kValues[q2[0]] +
|
| 156 |
+
y[2]*kValues[q1[1]] + y[3]*kValues[q2[1]] +
|
| 157 |
+
y[4]*kValues[q1[2]] + y[5]*kValues[q2[2]] +
|
| 158 |
+
y[6]*kValues[q1[3]] + y[7]*kValues[q2[3]];
|
| 159 |
+
s1 += y[0] + y[1] + y[2] + y[3] + y[4] + y[5] + y[6] + y[7];
|
| 160 |
+
y += 8;
|
| 161 |
+
}
|
| 162 |
+
sum += s*x->d + s1*x->m;
|
| 163 |
+
++x;
|
| 164 |
+
}
|
| 165 |
+
return sum;
|
| 166 |
+
}
|
| 167 |
+
|
| 168 |
+
// Copy-pasted from ggml.c
|
| 169 |
+
static void quantize_row_q8_0_reference(const float *x, block_q8_0 *y, int k) {
|
| 170 |
+
assert(k % QK8_0 == 0);
|
| 171 |
+
const int nb = k / QK8_0;
|
| 172 |
+
|
| 173 |
+
for (int i = 0; i < nb; i++) {
|
| 174 |
+
float amax = 0.0f; // absolute max
|
| 175 |
+
|
| 176 |
+
for (int l = 0; l < QK8_0; l++) {
|
| 177 |
+
const float v = x[i*QK8_0 + l];
|
| 178 |
+
amax = std::max(amax, fabsf(v));
|
| 179 |
+
}
|
| 180 |
+
|
| 181 |
+
const float d = amax / ((1 << 7) - 1);
|
| 182 |
+
const float id = d ? 1.0f/d : 0.0f;
|
| 183 |
+
|
| 184 |
+
y[i].d = d;
|
| 185 |
+
|
| 186 |
+
for (int l = 0; l < QK8_0; ++l) {
|
| 187 |
+
const float v = x[i*QK8_0 + l]*id;
|
| 188 |
+
y[i].qs[l] = roundf(v);
|
| 189 |
+
}
|
| 190 |
+
}
|
| 191 |
+
}
|
| 192 |
+
|
| 193 |
+
// Copy-pasted from ggml.c
|
| 194 |
+
static void dot_q4_q8(const int n, float* s, const void* vx, const void* vy) {
|
| 195 |
+
const int nb = n / QK8_0;
|
| 196 |
+
const block_q4_0* x = (const block_q4_0*)vx;
|
| 197 |
+
const block_q8_0* y = (const block_q8_0*)vy;
|
| 198 |
+
float sumf = 0;
|
| 199 |
+
for (int i = 0; i < nb; i++) {
|
| 200 |
+
const float d0 = x[i].d;
|
| 201 |
+
const float d1 = y[i].d;
|
| 202 |
+
|
| 203 |
+
const uint8_t * p0 = x[i].qs;
|
| 204 |
+
const int8_t * p1 = y[i].qs;
|
| 205 |
+
|
| 206 |
+
int sumi = 0;
|
| 207 |
+
for (int j = 0; j < QK8_0/2; j++) {
|
| 208 |
+
const uint8_t v0 = p0[j];
|
| 209 |
+
|
| 210 |
+
const int i0 = (int8_t) (v0 & 0xf) - 8;
|
| 211 |
+
const int i1 = (int8_t) (v0 >> 4) - 8;
|
| 212 |
+
|
| 213 |
+
const int i2 = p1[2*j + 0];
|
| 214 |
+
const int i3 = p1[2*j + 1];
|
| 215 |
+
|
| 216 |
+
sumi += i0*i2 + i1*i3;
|
| 217 |
+
}
|
| 218 |
+
sumf += d0*d1*sumi;
|
| 219 |
+
}
|
| 220 |
+
*s = sumf;
|
| 221 |
+
}
|
| 222 |
+
|
| 223 |
+
int main(int argc, char** argv) {
|
| 224 |
+
|
| 225 |
+
int nloop = argc > 1 ? atoi(argv[1]) : 10;
|
| 226 |
+
bool scalar = argc > 2 ? atoi(argv[2]) : false;
|
| 227 |
+
bool useQ4_1 = argc > 3 ? atoi(argv[3]) : false;
|
| 228 |
+
|
| 229 |
+
if (scalar && useQ4_1) {
|
| 230 |
+
printf("It is not possible to use Q4_1 quantization and scalar implementations\n");
|
| 231 |
+
return 1;
|
| 232 |
+
}
|
| 233 |
+
|
| 234 |
+
std::mt19937 rndm(1234);
|
| 235 |
+
|
| 236 |
+
std::vector<float> x1(kVecSize), y1(kVecSize);
|
| 237 |
+
int n4 = useQ4_1 ? kVecSize / QK4_1 : kVecSize / QK4_0; n4 = 64*((n4 + 63)/64);
|
| 238 |
+
int n8 = kVecSize / QK8_0; n8 = 64*((n8 + 63)/64);
|
| 239 |
+
|
| 240 |
+
const auto * funcs_cpu = ggml_get_type_traits_cpu(useQ4_1 ? GGML_TYPE_Q4_1 : GGML_TYPE_Q4_0);
|
| 241 |
+
|
| 242 |
+
std::vector<block_q4_0> q40;
|
| 243 |
+
std::vector<block_q4_1> q41;
|
| 244 |
+
if (useQ4_1) q41.resize(n4);
|
| 245 |
+
else q40.resize(n4);
|
| 246 |
+
std::vector<block_q8_0> q8(n8);
|
| 247 |
+
double sumt = 0, sumt2 = 0, maxt = 0;
|
| 248 |
+
double sumqt = 0, sumqt2 = 0, maxqt = 0;
|
| 249 |
+
double sum = 0, sumq = 0, exactSum = 0;
|
| 250 |
+
for (int iloop=0; iloop<nloop; ++iloop) {
|
| 251 |
+
|
| 252 |
+
// Fill vector x with random numbers
|
| 253 |
+
fillRandomGaussianFloats(x1, rndm);
|
| 254 |
+
|
| 255 |
+
// Fill vector y with random numbers
|
| 256 |
+
fillRandomGaussianFloats(y1, rndm);
|
| 257 |
+
|
| 258 |
+
// Compute the exact dot product
|
| 259 |
+
for (int k=0; k<kVecSize; ++k) exactSum += x1[k]*y1[k];
|
| 260 |
+
|
| 261 |
+
// quantize x.
|
| 262 |
+
// Note, we do not include this in the timing as in practical application
|
| 263 |
+
// we already have the quantized model weights.
|
| 264 |
+
if (useQ4_1) {
|
| 265 |
+
funcs_cpu->from_float(x1.data(), q41.data(), kVecSize);
|
| 266 |
+
} else {
|
| 267 |
+
funcs_cpu->from_float(x1.data(), q40.data(), kVecSize);
|
| 268 |
+
}
|
| 269 |
+
|
| 270 |
+
// Now measure time the dot product needs using the "scalar" version above
|
| 271 |
+
auto t1 = std::chrono::high_resolution_clock::now();
|
| 272 |
+
if (useQ4_1) sum += dot41(kVecSize / QK4_1, q41.data(), y1.data());
|
| 273 |
+
else sum += dot(kVecSize / QK4_0, q40.data(), y1.data());
|
| 274 |
+
auto t2 = std::chrono::high_resolution_clock::now();
|
| 275 |
+
auto t = 1e-3*std::chrono::duration_cast<std::chrono::nanoseconds>(t2-t1).count();
|
| 276 |
+
sumt += t; sumt2 += t*t; maxt = std::max(maxt, t);
|
| 277 |
+
|
| 278 |
+
// And now measure the time needed to quantize y and perform the dot product with the quantized y
|
| 279 |
+
t1 = std::chrono::high_resolution_clock::now();
|
| 280 |
+
float result;
|
| 281 |
+
if (scalar) {
|
| 282 |
+
quantize_row_q8_0_reference(y1.data(), q8.data(), kVecSize);
|
| 283 |
+
dot_q4_q8(kVecSize, &result, q40.data(), q8.data());
|
| 284 |
+
}
|
| 285 |
+
else {
|
| 286 |
+
const auto * vdot = ggml_get_type_traits_cpu(funcs_cpu->vec_dot_type);
|
| 287 |
+
vdot->from_float(y1.data(), q8.data(), kVecSize);
|
| 288 |
+
if (useQ4_1) funcs_cpu->vec_dot(kVecSize, &result, 0, q41.data(), 0, q8.data(), 0, 1);
|
| 289 |
+
else funcs_cpu->vec_dot(kVecSize, &result, 0, q40.data(), 0, q8.data(), 0, 1);
|
| 290 |
+
}
|
| 291 |
+
sumq += result;
|
| 292 |
+
t2 = std::chrono::high_resolution_clock::now();
|
| 293 |
+
t = 1e-3*std::chrono::duration_cast<std::chrono::nanoseconds>(t2-t1).count();
|
| 294 |
+
sumqt += t; sumqt2 += t*t; maxqt = std::max(maxqt, t);
|
| 295 |
+
|
| 296 |
+
}
|
| 297 |
+
|
| 298 |
+
// Report the time (and the average of the dot products so the compiler does not come up with the idea
|
| 299 |
+
// of optimizing away the function calls after figuring that the result is not used).
|
| 300 |
+
sum /= nloop; sumq /= nloop;
|
| 301 |
+
exactSum /= nloop;
|
| 302 |
+
printf("Exact result: <dot> = %g\n",exactSum);
|
| 303 |
+
printf("<dot> = %g, %g\n",sum,sumq);
|
| 304 |
+
sumt /= nloop; sumt2 /= nloop; sumt2 -= sumt*sumt;
|
| 305 |
+
if (sumt2 > 0) sumt2 = sqrt(sumt2);
|
| 306 |
+
printf("time = %g +/- %g us. maxt = %g us\n",sumt,sumt2,maxt);
|
| 307 |
+
sumqt /= nloop; sumqt2 /= nloop; sumqt2 -= sumqt*sumqt;
|
| 308 |
+
if (sumqt2 > 0) sumqt2 = sqrt(sumqt2);
|
| 309 |
+
printf("timeq = %g +/- %g us. maxt = %g us\n",sumqt,sumqt2,maxqt);
|
| 310 |
+
return 0;
|
| 311 |
+
}
|
backend/llama.cpp/pyproject.toml
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[project]
|
| 2 |
+
name = "llama-cpp-scripts"
|
| 3 |
+
description = "Scripts that ship with llama.cpp"
|
| 4 |
+
keywords = ["ggml", "gguf", "llama.cpp"]
|
| 5 |
+
version = "0.0.0"
|
| 6 |
+
dynamic = ["classifiers"]
|
| 7 |
+
readme = "README.md"
|
| 8 |
+
authors = [{name = "GGML", email = "ggml@ggml.ai"}]
|
| 9 |
+
requires-python = '>=3.10,<3.15'
|
| 10 |
+
dependencies = [
|
| 11 |
+
'numpy (>=1.26.4,<3.0.0)',
|
| 12 |
+
'sentencepiece (>=0.1.98,<0.3.0)',
|
| 13 |
+
'transformers (==4.57.6)',
|
| 14 |
+
'protobuf (>=4.21.0,<5.0.0)',
|
| 15 |
+
'torch (>=2.6.0,<3.0.0)',
|
| 16 |
+
'gguf @ ./gguf-py',
|
| 17 |
+
]
|
| 18 |
+
classifiers = [
|
| 19 |
+
"Programming Language :: Python :: 3",
|
| 20 |
+
"License :: OSI Approved :: MIT License",
|
| 21 |
+
"Operating System :: OS Independent",
|
| 22 |
+
]
|
| 23 |
+
|
| 24 |
+
[project.urls]
|
| 25 |
+
homepage = "https://ggml.ai"
|
| 26 |
+
repository = "https://github.com/ggml-org/llama.cpp"
|
| 27 |
+
|
| 28 |
+
[project.scripts]
|
| 29 |
+
llama-convert-hf-to-gguf = "convert_hf_to_gguf:main"
|
| 30 |
+
llama-convert-lora-to-gguf = "convert_lora_to_gguf:main"
|
| 31 |
+
llama-convert-llama-ggml-to-gguf = "convert_llama_ggml_to_gguf:main"
|
| 32 |
+
llama-ggml-vk-generate-shaders = "ggml_vk_generate_shaders:main"
|
| 33 |
+
|
| 34 |
+
[tool.poetry]
|
| 35 |
+
packages = [
|
| 36 |
+
{ include = "*.py", from = "." },
|
| 37 |
+
{ include = "conversion", from = "." },
|
| 38 |
+
]
|
| 39 |
+
|
| 40 |
+
[tool.poetry.dependencies]
|
| 41 |
+
torch = [
|
| 42 |
+
{ version = "==2.11.0", source = "pypi", markers = "sys_platform == 'darwin'" },
|
| 43 |
+
{ version = "==2.11.0+cpu", source = "pytorch", markers = "sys_platform == 'linux'" },
|
| 44 |
+
{ version = "==2.11.0", source = "pypi", markers = "sys_platform == 'win32'" },
|
| 45 |
+
]
|
| 46 |
+
|
| 47 |
+
[tool.poetry.group.dev.dependencies]
|
| 48 |
+
pytest = "~=8.3.3"
|
| 49 |
+
|
| 50 |
+
# Force wheel + cpu
|
| 51 |
+
# For discussion and context see https://github.com/python-poetry/poetry#6409
|
| 52 |
+
[[tool.poetry.source]]
|
| 53 |
+
name = "pytorch"
|
| 54 |
+
url = "https://download.pytorch.org/whl/cpu"
|
| 55 |
+
priority = "explicit"
|
| 56 |
+
|
| 57 |
+
[tool.uv.sources]
|
| 58 |
+
torch = { index = "pytorch" }
|
| 59 |
+
|
| 60 |
+
[[tool.uv.index]]
|
| 61 |
+
name = "pytorch"
|
| 62 |
+
url = "https://download.pytorch.org/whl/cpu"
|
| 63 |
+
explicit = true
|
| 64 |
+
|
| 65 |
+
[build-system]
|
| 66 |
+
requires = ["poetry-core>=1.0.0"]
|
| 67 |
+
build-backend = "poetry.core.masonry.api"
|
backend/llama.cpp/pyrightconfig.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"extraPaths": ["gguf-py", "examples/model-conversion/scripts", "examples/model-conversion/scripts/utils"],
|
| 3 |
+
"pythonVersion": "3.9",
|
| 4 |
+
"pythonPlatform": "All",
|
| 5 |
+
"reportUnusedImport": "warning",
|
| 6 |
+
"reportDuplicateImport": "error",
|
| 7 |
+
"reportDeprecated": "warning",
|
| 8 |
+
"reportUnnecessaryTypeIgnoreComment": "information",
|
| 9 |
+
"disableBytesTypePromotions": false, // TODO: change once Python 3.12 is the minimum
|
| 10 |
+
"executionEnvironments": [
|
| 11 |
+
{
|
| 12 |
+
// TODO: make this version override work correctly
|
| 13 |
+
"root": "gguf-py",
|
| 14 |
+
"pythonVersion": "3.8",
|
| 15 |
+
},
|
| 16 |
+
{
|
| 17 |
+
// uses match expressions in steps.py
|
| 18 |
+
"root": "tools/server/tests",
|
| 19 |
+
"pythonVersion": "3.10",
|
| 20 |
+
},
|
| 21 |
+
],
|
| 22 |
+
}
|
backend/llama.cpp/requirements.txt
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# These requirements include all dependencies for all top-level python scripts
|
| 2 |
+
# for llama.cpp. Avoid adding packages here directly.
|
| 3 |
+
#
|
| 4 |
+
# Package versions must stay compatible across all top-level python scripts.
|
| 5 |
+
#
|
| 6 |
+
|
| 7 |
+
-r ./requirements/requirements-convert_legacy_llama.txt
|
| 8 |
+
|
| 9 |
+
-r ./requirements/requirements-convert_hf_to_gguf.txt
|
| 10 |
+
-r ./requirements/requirements-convert_hf_to_gguf_update.txt
|
| 11 |
+
-r ./requirements/requirements-convert_llama_ggml_to_gguf.txt
|
| 12 |
+
-r ./requirements/requirements-convert_lora_to_gguf.txt
|
| 13 |
+
-r ./requirements/requirements-tool_bench.txt
|
backend/llama.cpp/requirements/requirements-all.txt
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
-r ../tools/mtmd/requirements.txt
|
| 2 |
+
-r ../tools/server/bench/requirements.txt
|
| 3 |
+
-r ../tools/server/tests/requirements.txt
|
| 4 |
+
|
| 5 |
+
-r ./requirements-compare-llama-bench.txt
|
| 6 |
+
-r ./requirements-server-bench.txt
|
| 7 |
+
-r ./requirements-pydantic.txt
|
| 8 |
+
-r ./requirements-test-tokenizer-random.txt
|
| 9 |
+
|
| 10 |
+
-r ./requirements-convert_hf_to_gguf.txt
|
| 11 |
+
-r ./requirements-convert_hf_to_gguf_update.txt
|
| 12 |
+
-r ./requirements-convert_legacy_llama.txt
|
| 13 |
+
-r ./requirements-convert_llama_ggml_to_gguf.txt
|
| 14 |
+
-r ./requirements-tool_bench.txt
|
| 15 |
+
|
| 16 |
+
-r ./requirements-gguf_editor_gui.txt
|
| 17 |
+
|
| 18 |
+
-r ../examples/model-conversion/requirements.txt
|
backend/llama.cpp/requirements/requirements-compare-llama-bench.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
tabulate~=0.9.0
|
| 2 |
+
GitPython~=3.1.43
|
| 3 |
+
matplotlib~=3.10.0
|
backend/llama.cpp/requirements/requirements-convert_hf_to_gguf.txt
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
-r ./requirements-convert_legacy_llama.txt
|
| 2 |
+
--extra-index-url https://download.pytorch.org/whl/cpu
|
| 3 |
+
|
| 4 |
+
## Embedding Gemma requires PyTorch 2.6.0 or later, bumped to 2.11.0 for compatibility
|
| 5 |
+
torch==2.11.0; platform_machine != "s390x"
|
| 6 |
+
|
| 7 |
+
# torch s390x packages can only be found from nightly builds
|
| 8 |
+
--extra-index-url https://download.pytorch.org/whl/nightly
|
| 9 |
+
torch>=0.0.0.dev0; platform_machine == "s390x"
|
backend/llama.cpp/requirements/requirements-convert_hf_to_gguf_update.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
-r ./requirements-convert_legacy_llama.txt
|
backend/llama.cpp/requirements/requirements-convert_legacy_llama.txt
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
numpy~=1.26.4
|
| 2 |
+
sentencepiece>=0.1.98,<0.3.0
|
| 3 |
+
|
| 4 |
+
transformers==4.57.6
|
| 5 |
+
|
| 6 |
+
gguf>=0.1.0
|
| 7 |
+
protobuf>=4.21.0,<5.0.0
|
backend/llama.cpp/requirements/requirements-convert_llama_ggml_to_gguf.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
-r ./requirements-convert_legacy_llama.txt
|
backend/llama.cpp/requirements/requirements-convert_lora_to_gguf.txt
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
-r ./requirements-convert_hf_to_gguf.txt
|
| 2 |
+
--extra-index-url https://download.pytorch.org/whl/cpu
|
| 3 |
+
# torch s390x packages can only be found from nightly builds
|
| 4 |
+
--extra-index-url https://download.pytorch.org/whl/nightly
|
backend/llama.cpp/requirements/requirements-gguf_editor_gui.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
numpy~=1.26.4
|
| 2 |
+
PySide6~=6.9.0
|
| 3 |
+
gguf>=0.17.0
|
backend/llama.cpp/requirements/requirements-pydantic.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
docstring_parser~=0.15
|
| 2 |
+
pydantic~=2.11.7
|
| 3 |
+
requests~=2.32.3
|
backend/llama.cpp/requirements/requirements-server-bench.txt
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
datasets~=4.8.0
|
| 2 |
+
matplotlib~=3.10.0
|
| 3 |
+
numpy~=1.26.4
|
| 4 |
+
requests~=2.32.3
|
| 5 |
+
tqdm~=4.67.1
|
backend/llama.cpp/requirements/requirements-test-tokenizer-random.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
cffi~=1.16.0
|
backend/llama.cpp/requirements/requirements-tool_bench.txt
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
aiohttp~=3.9.3
|
| 2 |
+
pytest~=8.3.3
|
| 3 |
+
matplotlib~=3.10.0
|
| 4 |
+
numpy~=1.26.4
|
| 5 |
+
openai~=2.14.0
|
| 6 |
+
pandas~=2.2.3
|
| 7 |
+
prometheus-client~=0.20.0
|
| 8 |
+
requests~=2.32.3
|
| 9 |
+
wget~=3.2
|
| 10 |
+
typer~=0.24.1
|
| 11 |
+
seaborn~=0.13.2
|
backend/llama.cpp/scripts/apple/validate-apps.sh
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
./scripts/apple/validate-ios.sh
|
| 3 |
+
./scripts/apple/validate-macos.sh
|
| 4 |
+
./scripts/apple/validate-visionos.sh
|
| 5 |
+
./scripts/apple/validate-tvos.sh
|
backend/llama.cpp/scripts/apple/validate-ios.sh
ADDED
|
@@ -0,0 +1,820 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# validate-ios.sh - Validate iOS Application with embedded llama.xcframework using SwiftUI
|
| 3 |
+
|
| 4 |
+
# Authentication options (optional) (can be set via environment variables)
|
| 5 |
+
# To use: export APPLE_ID=your.email@example.com
|
| 6 |
+
# export APPLE_PASSWORD=your-app-specific-password
|
| 7 |
+
# ./validate-ios.sh
|
| 8 |
+
APPLE_ID=${APPLE_ID:-""}
|
| 9 |
+
APPLE_PASSWORD=${APPLE_PASSWORD:-""}
|
| 10 |
+
|
| 11 |
+
# Ensure the script exits on error
|
| 12 |
+
set -e
|
| 13 |
+
|
| 14 |
+
# Function to print usage instructions
|
| 15 |
+
print_usage() {
|
| 16 |
+
echo "Usage: ./validate-ios.sh [OPTIONS]"
|
| 17 |
+
echo ""
|
| 18 |
+
echo "Options:"
|
| 19 |
+
echo " --help Show this help message"
|
| 20 |
+
echo " --apple-id EMAIL Apple ID email for validation"
|
| 21 |
+
echo " --apple-password PWD App-specific password for Apple ID"
|
| 22 |
+
echo ""
|
| 23 |
+
echo "Environment variables:"
|
| 24 |
+
echo " APPLE_ID Apple ID email for validation"
|
| 25 |
+
echo " APPLE_PASSWORD App-specific password for Apple ID"
|
| 26 |
+
echo ""
|
| 27 |
+
echo "Notes:"
|
| 28 |
+
echo " - Command line options take precedence over environment variables"
|
| 29 |
+
echo " - Authentication is optional. If not provided, alternative validation will be performed"
|
| 30 |
+
echo " - For APPLE_PASSWORD, use an app-specific password generated at https://appleid.apple.com/account/manage"
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
# Parse command line arguments
|
| 34 |
+
while [[ $# -gt 0 ]]; do
|
| 35 |
+
case $1 in
|
| 36 |
+
--help)
|
| 37 |
+
print_usage
|
| 38 |
+
exit 0
|
| 39 |
+
;;
|
| 40 |
+
--apple-id)
|
| 41 |
+
APPLE_ID="$2"
|
| 42 |
+
shift 2
|
| 43 |
+
;;
|
| 44 |
+
--apple-password)
|
| 45 |
+
APPLE_PASSWORD="$2"
|
| 46 |
+
shift 2
|
| 47 |
+
;;
|
| 48 |
+
*)
|
| 49 |
+
echo "Unknown option: $1"
|
| 50 |
+
print_usage
|
| 51 |
+
exit 1
|
| 52 |
+
;;
|
| 53 |
+
esac
|
| 54 |
+
done
|
| 55 |
+
|
| 56 |
+
# Function to clean up in case of error
|
| 57 |
+
cleanup() {
|
| 58 |
+
# Don't clean up temp files on error to help with debugging
|
| 59 |
+
echo "===== iOS Validation Process Failed ====="
|
| 60 |
+
exit 1
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
# Set up trap to call cleanup function on error
|
| 64 |
+
trap cleanup ERR
|
| 65 |
+
|
| 66 |
+
set -e # Exit on any error
|
| 67 |
+
|
| 68 |
+
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )"
|
| 69 |
+
BUILD_DIR="${ROOT_DIR}/validation-builds/ios"
|
| 70 |
+
|
| 71 |
+
# Configuration
|
| 72 |
+
APP_NAME="iOSLlamaTest"
|
| 73 |
+
BUNDLE_ID="org.ggml.iOSLlamaTest"
|
| 74 |
+
XCFRAMEWORK_PATH="${ROOT_DIR}/build-apple/llama.xcframework"
|
| 75 |
+
TEMP_DIR="${BUILD_DIR}/temp"
|
| 76 |
+
ARCHIVE_PATH="${BUILD_DIR}/${APP_NAME}.xcarchive"
|
| 77 |
+
IPA_PATH="${BUILD_DIR}/${APP_NAME}.ipa"
|
| 78 |
+
VALIDATION_DIR="${BUILD_DIR}/validation"
|
| 79 |
+
|
| 80 |
+
# Create necessary directories
|
| 81 |
+
mkdir -p "${BUILD_DIR}"
|
| 82 |
+
mkdir -p "${TEMP_DIR}"
|
| 83 |
+
mkdir -p "${VALIDATION_DIR}"
|
| 84 |
+
|
| 85 |
+
echo "===== iOS Validation Process Started ====="
|
| 86 |
+
|
| 87 |
+
# 1. Create a simple test app project
|
| 88 |
+
echo "Creating test iOS app project..."
|
| 89 |
+
mkdir -p "${TEMP_DIR}/${APP_NAME}/${APP_NAME}"
|
| 90 |
+
cat > "${TEMP_DIR}/${APP_NAME}/${APP_NAME}/Info.plist" << EOF
|
| 91 |
+
<?xml version="1.0" encoding="UTF-8"?>
|
| 92 |
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
| 93 |
+
<plist version="1.0">
|
| 94 |
+
<dict>
|
| 95 |
+
<key>CFBundleDevelopmentRegion</key>
|
| 96 |
+
<string>en</string>
|
| 97 |
+
<key>CFBundleExecutable</key>
|
| 98 |
+
<string>${APP_NAME}</string>
|
| 99 |
+
<key>CFBundleIdentifier</key>
|
| 100 |
+
<string>${BUNDLE_ID}</string>
|
| 101 |
+
<key>CFBundleInfoDictionaryVersion</key>
|
| 102 |
+
<string>6.0</string>
|
| 103 |
+
<key>CFBundleName</key>
|
| 104 |
+
<string>${APP_NAME}</string>
|
| 105 |
+
<key>CFBundlePackageType</key>
|
| 106 |
+
<string>APPL</string>
|
| 107 |
+
<key>CFBundleShortVersionString</key>
|
| 108 |
+
<string>1.0</string>
|
| 109 |
+
<key>CFBundleVersion</key>
|
| 110 |
+
<string>1</string>
|
| 111 |
+
<key>LSRequiresIPhoneOS</key>
|
| 112 |
+
<true/>
|
| 113 |
+
<key>UILaunchScreen</key>
|
| 114 |
+
<dict/>
|
| 115 |
+
<key>UIRequiredDeviceCapabilities</key>
|
| 116 |
+
<array>
|
| 117 |
+
<string>armv7</string>
|
| 118 |
+
</array>
|
| 119 |
+
<key>UISupportedInterfaceOrientations</key>
|
| 120 |
+
<array>
|
| 121 |
+
<string>UIInterfaceOrientationPortrait</string>
|
| 122 |
+
</array>
|
| 123 |
+
</dict>
|
| 124 |
+
</plist>
|
| 125 |
+
EOF
|
| 126 |
+
|
| 127 |
+
# Create SwiftUI app files
|
| 128 |
+
mkdir -p "${TEMP_DIR}/${APP_NAME}/${APP_NAME}/Sources"
|
| 129 |
+
|
| 130 |
+
# Create App.swift
|
| 131 |
+
cat > "${TEMP_DIR}/${APP_NAME}/${APP_NAME}/Sources/App.swift" << EOF
|
| 132 |
+
import SwiftUI
|
| 133 |
+
import llama
|
| 134 |
+
|
| 135 |
+
@main
|
| 136 |
+
struct LlamaTestApp: App {
|
| 137 |
+
var body: some Scene {
|
| 138 |
+
WindowGroup {
|
| 139 |
+
ContentView()
|
| 140 |
+
}
|
| 141 |
+
}
|
| 142 |
+
}
|
| 143 |
+
EOF
|
| 144 |
+
|
| 145 |
+
# Create ContentView.swift
|
| 146 |
+
cat > "${TEMP_DIR}/${APP_NAME}/${APP_NAME}/Sources/ContentView.swift" << EOF
|
| 147 |
+
import SwiftUI
|
| 148 |
+
import llama
|
| 149 |
+
|
| 150 |
+
struct ContentView: View {
|
| 151 |
+
// Test that we can initialize a llama context params struct
|
| 152 |
+
let params = llama_context_default_params()
|
| 153 |
+
|
| 154 |
+
var body: some View {
|
| 155 |
+
VStack(spacing: 20) {
|
| 156 |
+
Text("Llama Framework Test")
|
| 157 |
+
.font(.largeTitle)
|
| 158 |
+
.padding()
|
| 159 |
+
|
| 160 |
+
Text("llama_context_default_params() created successfully")
|
| 161 |
+
.font(.headline)
|
| 162 |
+
.multilineTextAlignment(.center)
|
| 163 |
+
.padding()
|
| 164 |
+
|
| 165 |
+
// Display some param values to confirm the framework is working
|
| 166 |
+
Text("n_ctx: \(params.n_ctx)")
|
| 167 |
+
.font(.body)
|
| 168 |
+
|
| 169 |
+
Text("n_batch: \(params.n_batch)")
|
| 170 |
+
.font(.body)
|
| 171 |
+
|
| 172 |
+
Spacer()
|
| 173 |
+
}
|
| 174 |
+
.padding()
|
| 175 |
+
}
|
| 176 |
+
}
|
| 177 |
+
|
| 178 |
+
struct ContentView_Previews: PreviewProvider {
|
| 179 |
+
static var previews: some View {
|
| 180 |
+
ContentView()
|
| 181 |
+
}
|
| 182 |
+
}
|
| 183 |
+
EOF
|
| 184 |
+
|
| 185 |
+
# Create project.pbxproj, fixing the framework search paths issues
|
| 186 |
+
mkdir -p "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj"
|
| 187 |
+
cat > "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << 'EOF'
|
| 188 |
+
// !$*UTF8*$!
|
| 189 |
+
{
|
| 190 |
+
archiveVersion = 1;
|
| 191 |
+
classes = {
|
| 192 |
+
};
|
| 193 |
+
objectVersion = 54;
|
| 194 |
+
objects = {
|
| 195 |
+
|
| 196 |
+
/* Begin PBXBuildFile section */
|
| 197 |
+
11111111111111111111111 /* App.swift in Sources */ = {isa = PBXBuildFile; fileRef = 22222222222222222222222; };
|
| 198 |
+
33333333333333333333333 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 44444444444444444444444; };
|
| 199 |
+
55555555555555555555555 /* llama.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 66666666666666666666666; };
|
| 200 |
+
77777777777777777777777 /* llama.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 66666666666666666666666; };
|
| 201 |
+
/* End PBXBuildFile section */
|
| 202 |
+
|
| 203 |
+
/* Begin PBXCopyFilesBuildPhase section */
|
| 204 |
+
88888888888888888888888 /* Embed Frameworks */ = {
|
| 205 |
+
isa = PBXCopyFilesBuildPhase;
|
| 206 |
+
buildActionMask = 2147483647;
|
| 207 |
+
dstPath = "";
|
| 208 |
+
dstSubfolderSpec = 10;
|
| 209 |
+
files = (
|
| 210 |
+
77777777777777777777777 /* llama.xcframework in Embed Frameworks */,
|
| 211 |
+
);
|
| 212 |
+
name = "Embed Frameworks";
|
| 213 |
+
runOnlyForDeploymentPostprocessing = 0;
|
| 214 |
+
};
|
| 215 |
+
/* End PBXCopyFilesBuildPhase section */
|
| 216 |
+
|
| 217 |
+
/* Begin PBXFileReference section */
|
| 218 |
+
EOF
|
| 219 |
+
|
| 220 |
+
# Continue with the project.pbxproj file, using the APP_NAME variable appropriately
|
| 221 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << EOF
|
| 222 |
+
99999999999999999999999 /* ${APP_NAME}.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "${APP_NAME}.app"; sourceTree = BUILT_PRODUCTS_DIR; };
|
| 223 |
+
22222222222222222222222 /* App.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App.swift; sourceTree = "<group>"; };
|
| 224 |
+
44444444444444444444444 /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = "<group>"; };
|
| 225 |
+
AAAAAAAAAAAAAAAAAAAAAAA /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
|
| 226 |
+
66666666666666666666666 /* llama.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = llama.xcframework; sourceTree = "<group>"; };
|
| 227 |
+
/* End PBXFileReference section */
|
| 228 |
+
EOF
|
| 229 |
+
|
| 230 |
+
# Add the rest of the project file with fixed framework search paths
|
| 231 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << 'EOF'
|
| 232 |
+
/* Begin PBXFrameworksBuildPhase section */
|
| 233 |
+
BBBBBBBBBBBBBBBBBBBBBBBB /* Frameworks */ = {
|
| 234 |
+
isa = PBXFrameworksBuildPhase;
|
| 235 |
+
buildActionMask = 2147483647;
|
| 236 |
+
files = (
|
| 237 |
+
55555555555555555555555 /* llama.xcframework in Frameworks */,
|
| 238 |
+
);
|
| 239 |
+
runOnlyForDeploymentPostprocessing = 0;
|
| 240 |
+
};
|
| 241 |
+
/* End PBXFrameworksBuildPhase section */
|
| 242 |
+
|
| 243 |
+
/* Begin PBXGroup section */
|
| 244 |
+
EOF
|
| 245 |
+
|
| 246 |
+
# Continue with the project.pbxproj file, using the APP_NAME variable appropriately
|
| 247 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << EOF
|
| 248 |
+
CCCCCCCCCCCCCCCCCCCCCCCC /* Products */ = {
|
| 249 |
+
isa = PBXGroup;
|
| 250 |
+
children = (
|
| 251 |
+
99999999999999999999999 /* ${APP_NAME}.app */,
|
| 252 |
+
);
|
| 253 |
+
name = Products;
|
| 254 |
+
sourceTree = "<group>";
|
| 255 |
+
};
|
| 256 |
+
EOF
|
| 257 |
+
|
| 258 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << 'EOF'
|
| 259 |
+
DDDDDDDDDDDDDDDDDDDDDDDD /* Frameworks */ = {
|
| 260 |
+
isa = PBXGroup;
|
| 261 |
+
children = (
|
| 262 |
+
66666666666666666666666 /* llama.xcframework */,
|
| 263 |
+
);
|
| 264 |
+
name = Frameworks;
|
| 265 |
+
sourceTree = "<group>";
|
| 266 |
+
};
|
| 267 |
+
EEEEEEEEEEEEEEEEEEEEEEEE = {
|
| 268 |
+
isa = PBXGroup;
|
| 269 |
+
children = (
|
| 270 |
+
FFFFFFFFFFFFFFFFFFFFFFFF /* iOSLlamaTest */,
|
| 271 |
+
CCCCCCCCCCCCCCCCCCCCCCCC /* Products */,
|
| 272 |
+
DDDDDDDDDDDDDDDDDDDDDDDD /* Frameworks */,
|
| 273 |
+
);
|
| 274 |
+
sourceTree = "<group>";
|
| 275 |
+
};
|
| 276 |
+
FFFFFFFFFFFFFFFFFFFFFFFF /* iOSLlamaTest */ = {
|
| 277 |
+
isa = PBXGroup;
|
| 278 |
+
children = (
|
| 279 |
+
1111111111111111111111AA /* Sources */,
|
| 280 |
+
AAAAAAAAAAAAAAAAAAAAAAA /* Info.plist */,
|
| 281 |
+
);
|
| 282 |
+
path = "iOSLlamaTest";
|
| 283 |
+
sourceTree = "<group>";
|
| 284 |
+
};
|
| 285 |
+
1111111111111111111111AA /* Sources */ = {
|
| 286 |
+
isa = PBXGroup;
|
| 287 |
+
children = (
|
| 288 |
+
22222222222222222222222 /* App.swift */,
|
| 289 |
+
44444444444444444444444 /* ContentView.swift */,
|
| 290 |
+
);
|
| 291 |
+
path = Sources;
|
| 292 |
+
sourceTree = "<group>";
|
| 293 |
+
};
|
| 294 |
+
/* End PBXGroup section */
|
| 295 |
+
EOF
|
| 296 |
+
|
| 297 |
+
# Continue with the project.pbxproj file, using the APP_NAME variable appropriately
|
| 298 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << EOF
|
| 299 |
+
/* Begin PBXNativeTarget section */
|
| 300 |
+
3333333333333333333333AA /* ${APP_NAME} */ = {
|
| 301 |
+
isa = PBXNativeTarget;
|
| 302 |
+
buildConfigurationList = 4444444444444444444444AA /* Build configuration list for PBXNativeTarget "${APP_NAME}" */;
|
| 303 |
+
buildPhases = (
|
| 304 |
+
5555555555555555555555AA /* Sources */,
|
| 305 |
+
BBBBBBBBBBBBBBBBBBBBBBBB /* Frameworks */,
|
| 306 |
+
6666666666666666666666AA /* Resources */,
|
| 307 |
+
88888888888888888888888 /* Embed Frameworks */,
|
| 308 |
+
);
|
| 309 |
+
buildRules = (
|
| 310 |
+
);
|
| 311 |
+
dependencies = (
|
| 312 |
+
);
|
| 313 |
+
name = "${APP_NAME}";
|
| 314 |
+
productName = "${APP_NAME}";
|
| 315 |
+
productReference = 99999999999999999999999 /* ${APP_NAME}.app */;
|
| 316 |
+
productType = "com.apple.product-type.application";
|
| 317 |
+
};
|
| 318 |
+
/* End PBXNativeTarget section */
|
| 319 |
+
|
| 320 |
+
/* Begin PBXProject section */
|
| 321 |
+
7777777777777777777777AA /* Project object */ = {
|
| 322 |
+
isa = PBXProject;
|
| 323 |
+
attributes = {
|
| 324 |
+
LastSwiftUpdateCheck = 1240;
|
| 325 |
+
LastUpgradeCheck = 1240;
|
| 326 |
+
TargetAttributes = {
|
| 327 |
+
3333333333333333333333AA = {
|
| 328 |
+
CreatedOnToolsVersion = 12.4;
|
| 329 |
+
};
|
| 330 |
+
};
|
| 331 |
+
};
|
| 332 |
+
buildConfigurationList = 8888888888888888888888AA /* Build configuration list for PBXProject "${APP_NAME}" */;
|
| 333 |
+
compatibilityVersion = "Xcode 12.0";
|
| 334 |
+
developmentRegion = en;
|
| 335 |
+
hasScannedForEncodings = 0;
|
| 336 |
+
knownRegions = (
|
| 337 |
+
en,
|
| 338 |
+
Base,
|
| 339 |
+
);
|
| 340 |
+
mainGroup = EEEEEEEEEEEEEEEEEEEEEEEE;
|
| 341 |
+
productRefGroup = CCCCCCCCCCCCCCCCCCCCCCCC /* Products */;
|
| 342 |
+
projectDirPath = "";
|
| 343 |
+
projectRoot = "";
|
| 344 |
+
targets = (
|
| 345 |
+
3333333333333333333333AA /* ${APP_NAME} */,
|
| 346 |
+
);
|
| 347 |
+
};
|
| 348 |
+
/* End PBXProject section */
|
| 349 |
+
EOF
|
| 350 |
+
|
| 351 |
+
# Add the rest of the file with correct FRAMEWORK_SEARCH_PATHS
|
| 352 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << 'EOF'
|
| 353 |
+
/* Begin PBXResourcesBuildPhase section */
|
| 354 |
+
6666666666666666666666AA /* Resources */ = {
|
| 355 |
+
isa = PBXResourcesBuildPhase;
|
| 356 |
+
buildActionMask = 2147483647;
|
| 357 |
+
files = (
|
| 358 |
+
);
|
| 359 |
+
runOnlyForDeploymentPostprocessing = 0;
|
| 360 |
+
};
|
| 361 |
+
/* End PBXResourcesBuildPhase section */
|
| 362 |
+
|
| 363 |
+
/* Begin PBXSourcesBuildPhase section */
|
| 364 |
+
5555555555555555555555AA /* Sources */ = {
|
| 365 |
+
isa = PBXSourcesBuildPhase;
|
| 366 |
+
buildActionMask = 2147483647;
|
| 367 |
+
files = (
|
| 368 |
+
33333333333333333333333 /* ContentView.swift in Sources */,
|
| 369 |
+
11111111111111111111111 /* App.swift in Sources */,
|
| 370 |
+
);
|
| 371 |
+
runOnlyForDeploymentPostprocessing = 0;
|
| 372 |
+
};
|
| 373 |
+
/* End PBXSourcesBuildPhase section */
|
| 374 |
+
|
| 375 |
+
/* Begin XCBuildConfiguration section */
|
| 376 |
+
9999999999999999999999AA /* Debug */ = {
|
| 377 |
+
isa = XCBuildConfiguration;
|
| 378 |
+
buildSettings = {
|
| 379 |
+
ALWAYS_SEARCH_USER_PATHS = NO;
|
| 380 |
+
CLANG_ANALYZER_NONNULL = YES;
|
| 381 |
+
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
|
| 382 |
+
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
|
| 383 |
+
CLANG_CXX_LIBRARY = "libc++";
|
| 384 |
+
CLANG_ENABLE_MODULES = YES;
|
| 385 |
+
CLANG_ENABLE_OBJC_ARC = YES;
|
| 386 |
+
CLANG_ENABLE_OBJC_WEAK = YES;
|
| 387 |
+
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
|
| 388 |
+
CLANG_WARN_BOOL_CONVERSION = YES;
|
| 389 |
+
CLANG_WARN_COMMA = YES;
|
| 390 |
+
CLANG_WARN_CONSTANT_CONVERSION = YES;
|
| 391 |
+
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
|
| 392 |
+
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
|
| 393 |
+
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
|
| 394 |
+
CLANG_WARN_EMPTY_BODY = YES;
|
| 395 |
+
CLANG_WARN_ENUM_CONVERSION = YES;
|
| 396 |
+
CLANG_WARN_INFINITE_RECURSION = YES;
|
| 397 |
+
CLANG_WARN_INT_CONVERSION = YES;
|
| 398 |
+
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
|
| 399 |
+
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
|
| 400 |
+
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
|
| 401 |
+
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
|
| 402 |
+
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
|
| 403 |
+
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
|
| 404 |
+
CLANG_WARN_STRICT_PROTOTYPES = YES;
|
| 405 |
+
CLANG_WARN_SUSPICIOUS_MOVE = YES;
|
| 406 |
+
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
|
| 407 |
+
CLANG_WARN_UNREACHABLE_CODE = YES;
|
| 408 |
+
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
|
| 409 |
+
COPY_PHASE_STRIP = NO;
|
| 410 |
+
DEBUG_INFORMATION_FORMAT = dwarf;
|
| 411 |
+
ENABLE_STRICT_OBJC_MSGSEND = YES;
|
| 412 |
+
ENABLE_TESTABILITY = YES;
|
| 413 |
+
GCC_C_LANGUAGE_STANDARD = gnu11;
|
| 414 |
+
GCC_DYNAMIC_NO_PIC = NO;
|
| 415 |
+
GCC_NO_COMMON_BLOCKS = YES;
|
| 416 |
+
GCC_OPTIMIZATION_LEVEL = 0;
|
| 417 |
+
GCC_PREPROCESSOR_DEFINITIONS = (
|
| 418 |
+
"DEBUG=1",
|
| 419 |
+
"$(inherited)",
|
| 420 |
+
);
|
| 421 |
+
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
|
| 422 |
+
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
|
| 423 |
+
GCC_WARN_UNDECLARED_SELECTOR = YES;
|
| 424 |
+
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
|
| 425 |
+
GCC_WARN_UNUSED_FUNCTION = YES;
|
| 426 |
+
GCC_WARN_UNUSED_VARIABLE = YES;
|
| 427 |
+
IPHONEOS_DEPLOYMENT_TARGET = 16.4;
|
| 428 |
+
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
|
| 429 |
+
MTL_FAST_MATH = YES;
|
| 430 |
+
ONLY_ACTIVE_ARCH = YES;
|
| 431 |
+
SDKROOT = iphoneos;
|
| 432 |
+
SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
|
| 433 |
+
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
|
| 434 |
+
};
|
| 435 |
+
name = Debug;
|
| 436 |
+
};
|
| 437 |
+
AAAAAAAAAAAAAAAAAAAAABBB /* Release */ = {
|
| 438 |
+
isa = XCBuildConfiguration;
|
| 439 |
+
buildSettings = {
|
| 440 |
+
ALWAYS_SEARCH_USER_PATHS = NO;
|
| 441 |
+
CLANG_ANALYZER_NONNULL = YES;
|
| 442 |
+
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
|
| 443 |
+
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
|
| 444 |
+
CLANG_CXX_LIBRARY = "libc++";
|
| 445 |
+
CLANG_ENABLE_MODULES = YES;
|
| 446 |
+
CLANG_ENABLE_OBJC_ARC = YES;
|
| 447 |
+
CLANG_ENABLE_OBJC_WEAK = YES;
|
| 448 |
+
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
|
| 449 |
+
CLANG_WARN_BOOL_CONVERSION = YES;
|
| 450 |
+
CLANG_WARN_COMMA = YES;
|
| 451 |
+
CLANG_WARN_CONSTANT_CONVERSION = YES;
|
| 452 |
+
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
|
| 453 |
+
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
|
| 454 |
+
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
|
| 455 |
+
CLANG_WARN_EMPTY_BODY = YES;
|
| 456 |
+
CLANG_WARN_ENUM_CONVERSION = YES;
|
| 457 |
+
CLANG_WARN_INFINITE_RECURSION = YES;
|
| 458 |
+
CLANG_WARN_INT_CONVERSION = YES;
|
| 459 |
+
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
|
| 460 |
+
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
|
| 461 |
+
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
|
| 462 |
+
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
|
| 463 |
+
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
|
| 464 |
+
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
|
| 465 |
+
CLANG_WARN_STRICT_PROTOTYPES = YES;
|
| 466 |
+
CLANG_WARN_SUSPICIOUS_MOVE = YES;
|
| 467 |
+
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
|
| 468 |
+
CLANG_WARN_UNREACHABLE_CODE = YES;
|
| 469 |
+
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
|
| 470 |
+
COPY_PHASE_STRIP = NO;
|
| 471 |
+
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
|
| 472 |
+
ENABLE_NS_ASSERTIONS = NO;
|
| 473 |
+
ENABLE_STRICT_OBJC_MSGSEND = YES;
|
| 474 |
+
GCC_C_LANGUAGE_STANDARD = gnu11;
|
| 475 |
+
GCC_NO_COMMON_BLOCKS = YES;
|
| 476 |
+
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
|
| 477 |
+
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
|
| 478 |
+
GCC_WARN_UNDECLARED_SELECTOR = YES;
|
| 479 |
+
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
|
| 480 |
+
GCC_WARN_UNUSED_FUNCTION = YES;
|
| 481 |
+
GCC_WARN_UNUSED_VARIABLE = YES;
|
| 482 |
+
IPHONEOS_DEPLOYMENT_TARGET = 16.4;
|
| 483 |
+
MTL_ENABLE_DEBUG_INFO = NO;
|
| 484 |
+
MTL_FAST_MATH = YES;
|
| 485 |
+
SDKROOT = iphoneos;
|
| 486 |
+
SWIFT_COMPILATION_MODE = wholemodule;
|
| 487 |
+
SWIFT_OPTIMIZATION_LEVEL = "-O";
|
| 488 |
+
VALIDATE_PRODUCT = YES;
|
| 489 |
+
};
|
| 490 |
+
name = Release;
|
| 491 |
+
};
|
| 492 |
+
BBBBBBBBBBBBBBBBBBBBBBCCC /* Debug */ = {
|
| 493 |
+
isa = XCBuildConfiguration;
|
| 494 |
+
buildSettings = {
|
| 495 |
+
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
| 496 |
+
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
|
| 497 |
+
CODE_SIGN_STYLE = Manual;
|
| 498 |
+
DEVELOPMENT_TEAM = "";
|
| 499 |
+
ENABLE_PREVIEWS = YES;
|
| 500 |
+
FRAMEWORK_SEARCH_PATHS = "$(PROJECT_DIR)";
|
| 501 |
+
INFOPLIST_FILE = "iOSLlamaTest/Info.plist";
|
| 502 |
+
LD_RUNPATH_SEARCH_PATHS = (
|
| 503 |
+
"$(inherited)",
|
| 504 |
+
"@executable_path/Frameworks",
|
| 505 |
+
);
|
| 506 |
+
PRODUCT_BUNDLE_IDENTIFIER = "org.ggml.iOSLlamaTest";
|
| 507 |
+
PRODUCT_NAME = "$(TARGET_NAME)";
|
| 508 |
+
PROVISIONING_PROFILE_SPECIFIER = "";
|
| 509 |
+
SWIFT_VERSION = 5.0;
|
| 510 |
+
TARGETED_DEVICE_FAMILY = "1,2";
|
| 511 |
+
};
|
| 512 |
+
name = Debug;
|
| 513 |
+
};
|
| 514 |
+
CCCCCCCCCCCCCCCCCCCCCCDDD /* Release */ = {
|
| 515 |
+
isa = XCBuildConfiguration;
|
| 516 |
+
buildSettings = {
|
| 517 |
+
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
| 518 |
+
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
|
| 519 |
+
CODE_SIGN_STYLE = Manual;
|
| 520 |
+
DEVELOPMENT_TEAM = "";
|
| 521 |
+
ENABLE_PREVIEWS = YES;
|
| 522 |
+
FRAMEWORK_SEARCH_PATHS = (
|
| 523 |
+
"$(inherited)",
|
| 524 |
+
"$(PROJECT_DIR)",
|
| 525 |
+
);
|
| 526 |
+
INFOPLIST_FILE = "iOSLlamaTest/Info.plist";
|
| 527 |
+
LD_RUNPATH_SEARCH_PATHS = (
|
| 528 |
+
"$(inherited)",
|
| 529 |
+
"@executable_path/Frameworks",
|
| 530 |
+
);
|
| 531 |
+
PRODUCT_BUNDLE_IDENTIFIER = "org.ggml.iOSLlamaTest";
|
| 532 |
+
PRODUCT_NAME = "$(TARGET_NAME)";
|
| 533 |
+
PROVISIONING_PROFILE_SPECIFIER = "";
|
| 534 |
+
SWIFT_VERSION = 5.0;
|
| 535 |
+
TARGETED_DEVICE_FAMILY = "1,2";
|
| 536 |
+
};
|
| 537 |
+
name = Release;
|
| 538 |
+
};
|
| 539 |
+
/* End XCBuildConfiguration section */
|
| 540 |
+
EOF
|
| 541 |
+
|
| 542 |
+
# Finish the project.pbxproj file
|
| 543 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << EOF
|
| 544 |
+
/* Begin XCConfigurationList section */
|
| 545 |
+
8888888888888888888888AA /* Build configuration list for PBXProject "${APP_NAME}" */ = {
|
| 546 |
+
isa = XCConfigurationList;
|
| 547 |
+
buildConfigurations = (
|
| 548 |
+
9999999999999999999999AA /* Debug */,
|
| 549 |
+
AAAAAAAAAAAAAAAAAAAAABBB /* Release */,
|
| 550 |
+
);
|
| 551 |
+
defaultConfigurationIsVisible = 0;
|
| 552 |
+
defaultConfigurationName = Release;
|
| 553 |
+
};
|
| 554 |
+
4444444444444444444444AA /* Build configuration list for PBXNativeTarget "${APP_NAME}" */ = {
|
| 555 |
+
isa = XCConfigurationList;
|
| 556 |
+
buildConfigurations = (
|
| 557 |
+
BBBBBBBBBBBBBBBBBBBBBBCCC /* Debug */,
|
| 558 |
+
CCCCCCCCCCCCCCCCCCCCCCDDD /* Release */,
|
| 559 |
+
);
|
| 560 |
+
defaultConfigurationIsVisible = 0;
|
| 561 |
+
defaultConfigurationName = Release;
|
| 562 |
+
};
|
| 563 |
+
/* End XCConfigurationList section */
|
| 564 |
+
};
|
| 565 |
+
rootObject = 7777777777777777777777AA /* Project object */;
|
| 566 |
+
}
|
| 567 |
+
EOF
|
| 568 |
+
|
| 569 |
+
# 2. Copy XCFramework to test project
|
| 570 |
+
echo "Copying XCFramework to test project..."
|
| 571 |
+
cp -R "${XCFRAMEWORK_PATH}" "${TEMP_DIR}/${APP_NAME}/"
|
| 572 |
+
|
| 573 |
+
# 3. Build and archive the app
|
| 574 |
+
echo "Building and archiving test app..."
|
| 575 |
+
cd "${TEMP_DIR}/${APP_NAME}"
|
| 576 |
+
|
| 577 |
+
# Create a simple xcscheme file to avoid xcodebuild scheme issues
|
| 578 |
+
mkdir -p "${APP_NAME}.xcodeproj/xcshareddata/xcschemes"
|
| 579 |
+
cat > "${APP_NAME}.xcodeproj/xcshareddata/xcschemes/${APP_NAME}.xcscheme" << EOF
|
| 580 |
+
<?xml version="1.0" encoding="UTF-8"?>
|
| 581 |
+
<Scheme
|
| 582 |
+
LastUpgradeVersion = "1240"
|
| 583 |
+
version = "1.3">
|
| 584 |
+
<BuildAction
|
| 585 |
+
parallelizeBuildables = "YES"
|
| 586 |
+
buildImplicitDependencies = "YES">
|
| 587 |
+
<BuildActionEntries>
|
| 588 |
+
<BuildActionEntry
|
| 589 |
+
buildForTesting = "YES"
|
| 590 |
+
buildForRunning = "YES"
|
| 591 |
+
buildForProfiling = "YES"
|
| 592 |
+
buildForArchiving = "YES"
|
| 593 |
+
buildForAnalyzing = "YES">
|
| 594 |
+
<BuildableReference
|
| 595 |
+
BuildableIdentifier = "primary"
|
| 596 |
+
BlueprintIdentifier = "3333333333333333333333AA"
|
| 597 |
+
BuildableName = "${APP_NAME}.app"
|
| 598 |
+
BlueprintName = "${APP_NAME}"
|
| 599 |
+
ReferencedContainer = "container:${APP_NAME}.xcodeproj">
|
| 600 |
+
</BuildableReference>
|
| 601 |
+
</BuildActionEntry>
|
| 602 |
+
</BuildActionEntries>
|
| 603 |
+
</BuildAction>
|
| 604 |
+
<TestAction
|
| 605 |
+
buildConfiguration = "Debug"
|
| 606 |
+
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
|
| 607 |
+
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
|
| 608 |
+
shouldUseLaunchSchemeArgsEnv = "YES">
|
| 609 |
+
<Testables>
|
| 610 |
+
</Testables>
|
| 611 |
+
</TestAction>
|
| 612 |
+
<LaunchAction
|
| 613 |
+
buildConfiguration = "Debug"
|
| 614 |
+
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
|
| 615 |
+
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
|
| 616 |
+
launchStyle = "0"
|
| 617 |
+
useCustomWorkingDirectory = "NO"
|
| 618 |
+
ignoresPersistentStateOnLaunch = "NO"
|
| 619 |
+
debugDocumentVersioning = "YES"
|
| 620 |
+
debugServiceExtension = "internal"
|
| 621 |
+
allowLocationSimulation = "YES">
|
| 622 |
+
<BuildableProductRunnable
|
| 623 |
+
runnableDebuggingMode = "0">
|
| 624 |
+
<BuildableReference
|
| 625 |
+
BuildableIdentifier = "primary"
|
| 626 |
+
BlueprintIdentifier = "3333333333333333333333AA"
|
| 627 |
+
BuildableName = "${APP_NAME}.app"
|
| 628 |
+
BlueprintName = "${APP_NAME}"
|
| 629 |
+
ReferencedContainer = "container:${APP_NAME}.xcodeproj">
|
| 630 |
+
</BuildableReference>
|
| 631 |
+
</BuildableProductRunnable>
|
| 632 |
+
</LaunchAction>
|
| 633 |
+
<ProfileAction
|
| 634 |
+
buildConfiguration = "Release"
|
| 635 |
+
shouldUseLaunchSchemeArgsEnv = "YES"
|
| 636 |
+
savedToolIdentifier = ""
|
| 637 |
+
useCustomWorkingDirectory = "NO"
|
| 638 |
+
debugDocumentVersioning = "YES">
|
| 639 |
+
<BuildableProductRunnable
|
| 640 |
+
runnableDebuggingMode = "0">
|
| 641 |
+
<BuildableReference
|
| 642 |
+
BuildableIdentifier = "primary"
|
| 643 |
+
BlueprintIdentifier = "3333333333333333333333AA"
|
| 644 |
+
BuildableName = "${APP_NAME}.app"
|
| 645 |
+
BlueprintName = "${APP_NAME}"
|
| 646 |
+
ReferencedContainer = "container:${APP_NAME}.xcodeproj">
|
| 647 |
+
</BuildableReference>
|
| 648 |
+
</BuildableProductRunnable>
|
| 649 |
+
</ProfileAction>
|
| 650 |
+
<AnalyzeAction
|
| 651 |
+
buildConfiguration = "Debug">
|
| 652 |
+
</AnalyzeAction>
|
| 653 |
+
<ArchiveAction
|
| 654 |
+
buildConfiguration = "Release"
|
| 655 |
+
revealArchiveInOrganizer = "YES">
|
| 656 |
+
</ArchiveAction>
|
| 657 |
+
</Scheme>
|
| 658 |
+
EOF
|
| 659 |
+
|
| 660 |
+
# Now use xcodebuild with an explicitly defined product name
|
| 661 |
+
xcodebuild -project "${APP_NAME}.xcodeproj" -scheme "${APP_NAME}" -sdk iphoneos -configuration Release archive -archivePath "${ARCHIVE_PATH}" CODE_SIGN_IDENTITY="-" CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO PRODUCT_NAME="${APP_NAME}" SWIFT_OPTIMIZATION_LEVEL="-Onone" -quiet
|
| 662 |
+
|
| 663 |
+
# 4. Create IPA from archive
|
| 664 |
+
echo "Creating IPA from archive..."
|
| 665 |
+
mkdir -p "${TEMP_DIR}/Payload"
|
| 666 |
+
cp -R "${ARCHIVE_PATH}/Products/Applications/${APP_NAME}.app" "${TEMP_DIR}/Payload/"
|
| 667 |
+
|
| 668 |
+
# Check and log app structure before zipping
|
| 669 |
+
echo "App structure:"
|
| 670 |
+
ls -la "${TEMP_DIR}/Payload/${APP_NAME}.app/"
|
| 671 |
+
echo "Frameworks:"
|
| 672 |
+
ls -la "${TEMP_DIR}/Payload/${APP_NAME}.app/Frameworks/" 2>/dev/null || echo "No Frameworks directory found"
|
| 673 |
+
|
| 674 |
+
cd "${TEMP_DIR}"
|
| 675 |
+
zip -r "${IPA_PATH}" Payload
|
| 676 |
+
|
| 677 |
+
# Check embedded provisioning profile
|
| 678 |
+
echo "Checking provisioning profile (if any)..."
|
| 679 |
+
PROVISIONING_PROFILE=$(find "${ARCHIVE_PATH}/Products/Applications/${APP_NAME}.app" -name "embedded.mobileprovision" 2>/dev/null)
|
| 680 |
+
if [ -n "$PROVISIONING_PROFILE" ]; then
|
| 681 |
+
echo "Found embedded provisioning profile:"
|
| 682 |
+
security cms -D -i "$PROVISIONING_PROFILE" || echo "Unable to decode provisioning profile"
|
| 683 |
+
else
|
| 684 |
+
echo "No embedded provisioning profile found (expected for ad-hoc builds)"
|
| 685 |
+
fi
|
| 686 |
+
|
| 687 |
+
# 5. Validate the IPA
|
| 688 |
+
echo "Validating IPA..."
|
| 689 |
+
VALIDATION_OUTPUT="${VALIDATION_DIR}/validation_output.txt"
|
| 690 |
+
|
| 691 |
+
# Check if authentication credentials are provided
|
| 692 |
+
AUTH_ARGS=""
|
| 693 |
+
if [ -n "$APPLE_ID" ] && [ -n "$APPLE_PASSWORD" ]; then
|
| 694 |
+
echo "Using Apple ID authentication for validation..."
|
| 695 |
+
AUTH_ARGS="--username \"$APPLE_ID\" --password \"$APPLE_PASSWORD\""
|
| 696 |
+
else
|
| 697 |
+
echo "No authentication credentials provided. Will perform basic validation."
|
| 698 |
+
echo "To use your personal developer account, you can run the script with:"
|
| 699 |
+
echo " APPLE_ID='your.email@example.com' APPLE_PASSWORD='your-app-specific-password' ./validate-ios.sh"
|
| 700 |
+
echo "Note: You need to create an app-specific password at https://appleid.apple.com/account/manage"
|
| 701 |
+
fi
|
| 702 |
+
|
| 703 |
+
# Run validation with detailed output
|
| 704 |
+
echo "Running validation with altool..."
|
| 705 |
+
if [ -n "$AUTH_ARGS" ]; then
|
| 706 |
+
# Use eval to properly handle the quoted arguments
|
| 707 |
+
eval "xcrun altool --validate-app -f \"${IPA_PATH}\" --type ios --output-format xml $AUTH_ARGS" 2>&1 | tee "${VALIDATION_OUTPUT}"
|
| 708 |
+
else
|
| 709 |
+
xcrun altool --validate-app -f "${IPA_PATH}" --type ios --output-format xml 2>&1 | tee "${VALIDATION_OUTPUT}"
|
| 710 |
+
fi
|
| 711 |
+
VALIDATION_RESULT=$?
|
| 712 |
+
|
| 713 |
+
# Final validation result
|
| 714 |
+
FINAL_VALIDATION_RESULT=0
|
| 715 |
+
|
| 716 |
+
# Check if validation failed because the app isn't in App Store Connect
|
| 717 |
+
if grep -q "No suitable application records were found" "${VALIDATION_OUTPUT}"; then
|
| 718 |
+
echo "⚠️ App Store Connect Warning: The app bundle identifier is not found in App Store Connect"
|
| 719 |
+
echo "This is expected for apps that haven't been registered in App Store Connect yet."
|
| 720 |
+
echo "This doesn't indicate a problem with the build or framework."
|
| 721 |
+
|
| 722 |
+
# Perform alternative validation
|
| 723 |
+
echo "Performing alternative validation checks..."
|
| 724 |
+
|
| 725 |
+
# Check if IPA was created successfully
|
| 726 |
+
if [ -f "${IPA_PATH}" ] && [ -s "${IPA_PATH}" ]; then
|
| 727 |
+
echo "✅ IPA file created successfully"
|
| 728 |
+
else
|
| 729 |
+
echo "❌ IPA file not created or empty"
|
| 730 |
+
FINAL_VALIDATION_RESULT=1
|
| 731 |
+
fi
|
| 732 |
+
|
| 733 |
+
# Check if app binary exists and is executable
|
| 734 |
+
if [ -f "${TEMP_DIR}/Payload/${APP_NAME}.app/${APP_NAME}" ] && [ -x "${TEMP_DIR}/Payload/${APP_NAME}.app/${APP_NAME}" ]; then
|
| 735 |
+
echo "✅ App binary exists and is executable"
|
| 736 |
+
else
|
| 737 |
+
echo "❌ App binary missing or not executable"
|
| 738 |
+
FINAL_VALIDATION_RESULT=1
|
| 739 |
+
fi
|
| 740 |
+
|
| 741 |
+
# Check if framework was properly embedded
|
| 742 |
+
if [ -d "${TEMP_DIR}/Payload/${APP_NAME}.app/Frameworks/llama.framework" ]; then
|
| 743 |
+
echo "✅ llama.framework properly embedded"
|
| 744 |
+
else
|
| 745 |
+
echo "❌ llama.framework not properly embedded"
|
| 746 |
+
FINAL_VALIDATION_RESULT=1
|
| 747 |
+
fi
|
| 748 |
+
|
| 749 |
+
# Check if framework binary exists
|
| 750 |
+
if [ -f "${TEMP_DIR}/Payload/${APP_NAME}.app/Frameworks/llama.framework/llama" ]; then
|
| 751 |
+
echo "✅ Framework binary exists"
|
| 752 |
+
|
| 753 |
+
# Further validate framework by checking architecture
|
| 754 |
+
ARCHS=$(lipo -info "${TEMP_DIR}/Payload/${APP_NAME}.app/Frameworks/llama.framework/llama" 2>/dev/null | grep -o "arm64\\|armv7\\|x86_64" | tr '\n' ' ')
|
| 755 |
+
if [ -n "$ARCHS" ]; then
|
| 756 |
+
echo "✅ Framework architecture(s): $ARCHS"
|
| 757 |
+
else
|
| 758 |
+
echo "⚠️ Could not determine framework architecture"
|
| 759 |
+
fi
|
| 760 |
+
else
|
| 761 |
+
echo "❌ Framework binary missing"
|
| 762 |
+
FINAL_VALIDATION_RESULT=1
|
| 763 |
+
fi
|
| 764 |
+
|
| 765 |
+
if [ $FINAL_VALIDATION_RESULT -eq 0 ]; then
|
| 766 |
+
echo "✅ Alternative validation PASSED: App built successfully with embedded framework"
|
| 767 |
+
else
|
| 768 |
+
echo "❌ Alternative validation FAILED: Issues found with the app or framework"
|
| 769 |
+
fi
|
| 770 |
+
elif grep -q "You must specify authentication credentials" "${VALIDATION_OUTPUT}" && [ -z "$AUTH_ARGS" ]; then
|
| 771 |
+
echo "✅ iOS Validation PASSED: IPA successfully validated"
|
| 772 |
+
echo "Results saved to ${VALIDATION_OUTPUT}"
|
| 773 |
+
else
|
| 774 |
+
echo "❌ iOS Validation FAILED: IPA validation found issues"
|
| 775 |
+
echo "See validation output at ${VALIDATION_OUTPUT}"
|
| 776 |
+
echo ""
|
| 777 |
+
echo "==== VALIDATION ERRORS ===="
|
| 778 |
+
|
| 779 |
+
# Try to extract specific errors from the output
|
| 780 |
+
if grep -q "Error" "${VALIDATION_OUTPUT}"; then
|
| 781 |
+
grep -A 5 "Error" "${VALIDATION_OUTPUT}"
|
| 782 |
+
else
|
| 783 |
+
# If no specific error found, show the whole log
|
| 784 |
+
cat "${VALIDATION_OUTPUT}"
|
| 785 |
+
fi
|
| 786 |
+
|
| 787 |
+
# Additional debugging: check IPA contents
|
| 788 |
+
echo ""
|
| 789 |
+
echo "==== IPA CONTENTS ===="
|
| 790 |
+
mkdir -p "${TEMP_DIR}/ipa_contents"
|
| 791 |
+
unzip -q "${IPA_PATH}" -d "${TEMP_DIR}/ipa_contents"
|
| 792 |
+
ls -la "${TEMP_DIR}/ipa_contents/Payload/${APP_NAME}.app/"
|
| 793 |
+
|
| 794 |
+
# Check for code signing issues
|
| 795 |
+
echo ""
|
| 796 |
+
echo "==== CODE SIGNING INFO ===="
|
| 797 |
+
codesign -vv -d "${TEMP_DIR}/ipa_contents/Payload/${APP_NAME}.app" 2>&1 || echo "Code signing verification failed"
|
| 798 |
+
|
| 799 |
+
# Check embedded frameworks
|
| 800 |
+
echo ""
|
| 801 |
+
echo "==== FRAMEWORK INFO ===="
|
| 802 |
+
ls -la "${TEMP_DIR}/ipa_contents/Payload/${APP_NAME}.app/Frameworks/" 2>/dev/null || echo "No Frameworks directory found"
|
| 803 |
+
fi
|
| 804 |
+
|
| 805 |
+
# Don't clean up on error to allow inspection
|
| 806 |
+
if [ $FINAL_VALIDATION_RESULT -ne 0 ]; then
|
| 807 |
+
echo ""
|
| 808 |
+
echo "Temporary files kept for inspection at: ${TEMP_DIR}"
|
| 809 |
+
echo "===== iOS Validation Process Failed ====="
|
| 810 |
+
exit 1
|
| 811 |
+
fi
|
| 812 |
+
|
| 813 |
+
# Clean up temporary files but keep build artifacts
|
| 814 |
+
if [ $FINAL_VALIDATION_RESULT -eq 0 ]; then
|
| 815 |
+
echo "Cleaning up temporary files..."
|
| 816 |
+
#rm -rf "${TEMP_DIR}"
|
| 817 |
+
fi
|
| 818 |
+
|
| 819 |
+
echo "===== iOS Validation Process Completed ====="
|
| 820 |
+
exit $FINAL_VALIDATION_RESULT
|
backend/llama.cpp/scripts/apple/validate-macos.sh
ADDED
|
@@ -0,0 +1,781 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# validate-macos.sh - Validate macOS Application with embedded llama.xcframework using SwiftUI
|
| 3 |
+
|
| 4 |
+
# Authentication options (optional) (can be set via environment variables)
|
| 5 |
+
# To use: export APPLE_ID=your.email@example.com
|
| 6 |
+
# export APPLE_PASSWORD=your-app-specific-password
|
| 7 |
+
# ./validate-macos.sh
|
| 8 |
+
APPLE_ID=${APPLE_ID:-""}
|
| 9 |
+
APPLE_PASSWORD=${APPLE_PASSWORD:-""}
|
| 10 |
+
|
| 11 |
+
# Ensure the script exits on error
|
| 12 |
+
set -e
|
| 13 |
+
|
| 14 |
+
# Function to print usage instructions
|
| 15 |
+
print_usage() {
|
| 16 |
+
echo "Usage: ./validate-macos.sh [OPTIONS]"
|
| 17 |
+
echo ""
|
| 18 |
+
echo "Options:"
|
| 19 |
+
echo " --help Show this help message"
|
| 20 |
+
echo " --apple-id EMAIL Apple ID email for validation"
|
| 21 |
+
echo " --apple-password PWD App-specific password for Apple ID"
|
| 22 |
+
echo ""
|
| 23 |
+
echo "Environment variables:"
|
| 24 |
+
echo " APPLE_ID Apple ID email for validation"
|
| 25 |
+
echo " APPLE_PASSWORD App-specific password for Apple ID"
|
| 26 |
+
echo ""
|
| 27 |
+
echo "Notes:"
|
| 28 |
+
echo " - Command line options take precedence over environment variables"
|
| 29 |
+
echo " - Authentication is optional. If not provided, alternative validation will be performed"
|
| 30 |
+
echo " - For APPLE_PASSWORD, use an app-specific password generated at https://appleid.apple.com/account/manage"
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
# Parse command line arguments
|
| 34 |
+
while [[ $# -gt 0 ]]; do
|
| 35 |
+
case $1 in
|
| 36 |
+
--help)
|
| 37 |
+
print_usage
|
| 38 |
+
exit 0
|
| 39 |
+
;;
|
| 40 |
+
--apple-id)
|
| 41 |
+
APPLE_ID="$2"
|
| 42 |
+
shift 2
|
| 43 |
+
;;
|
| 44 |
+
--apple-password)
|
| 45 |
+
APPLE_PASSWORD="$2"
|
| 46 |
+
shift 2
|
| 47 |
+
;;
|
| 48 |
+
*)
|
| 49 |
+
echo "Unknown option: $1"
|
| 50 |
+
print_usage
|
| 51 |
+
exit 1
|
| 52 |
+
;;
|
| 53 |
+
esac
|
| 54 |
+
done
|
| 55 |
+
|
| 56 |
+
# Function to clean up in case of error
|
| 57 |
+
cleanup() {
|
| 58 |
+
# Don't clean up temp files on error to help with debugging
|
| 59 |
+
echo "===== macOS Validation Process Failed ====="
|
| 60 |
+
exit 1
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
# Set up trap to call cleanup function on error
|
| 64 |
+
trap cleanup ERR
|
| 65 |
+
|
| 66 |
+
set -e # Exit on any error
|
| 67 |
+
|
| 68 |
+
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )"
|
| 69 |
+
BUILD_DIR="${ROOT_DIR}/validation-builds/ios"
|
| 70 |
+
|
| 71 |
+
# Configuration
|
| 72 |
+
APP_NAME="MacOSLlamaTest"
|
| 73 |
+
BUNDLE_ID="org.ggml.MacOSLlamaTest"
|
| 74 |
+
XCFRAMEWORK_PATH="${ROOT_DIR}/build-apple/llama.xcframework"
|
| 75 |
+
TEMP_DIR="${BUILD_DIR}/temp"
|
| 76 |
+
ARCHIVE_PATH="${BUILD_DIR}/${APP_NAME}.xcarchive"
|
| 77 |
+
APP_PATH="${BUILD_DIR}/${APP_NAME}.app"
|
| 78 |
+
ZIP_PATH="${BUILD_DIR}/${APP_NAME}.zip"
|
| 79 |
+
VALIDATION_DIR="${BUILD_DIR}/validation"
|
| 80 |
+
|
| 81 |
+
# Create necessary directories
|
| 82 |
+
mkdir -p "${BUILD_DIR}"
|
| 83 |
+
mkdir -p "${TEMP_DIR}"
|
| 84 |
+
mkdir -p "${VALIDATION_DIR}"
|
| 85 |
+
|
| 86 |
+
echo "===== macOS Validation Process Started ====="
|
| 87 |
+
|
| 88 |
+
# 1. Create a simple test app project
|
| 89 |
+
echo "Creating test macOS app project..."
|
| 90 |
+
mkdir -p "${TEMP_DIR}/${APP_NAME}/${APP_NAME}"
|
| 91 |
+
cat > "${TEMP_DIR}/${APP_NAME}/${APP_NAME}/Info.plist" << EOF
|
| 92 |
+
<?xml version="1.0" encoding="UTF-8"?>
|
| 93 |
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
| 94 |
+
<plist version="1.0">
|
| 95 |
+
<dict>
|
| 96 |
+
<key>CFBundleDevelopmentRegion</key>
|
| 97 |
+
<string>en</string>
|
| 98 |
+
<key>CFBundleExecutable</key>
|
| 99 |
+
<string>${APP_NAME}</string>
|
| 100 |
+
<key>CFBundleIdentifier</key>
|
| 101 |
+
<string>${BUNDLE_ID}</string>
|
| 102 |
+
<key>CFBundleInfoDictionaryVersion</key>
|
| 103 |
+
<string>6.0</string>
|
| 104 |
+
<key>CFBundleName</key>
|
| 105 |
+
<string>${APP_NAME}</string>
|
| 106 |
+
<key>CFBundlePackageType</key>
|
| 107 |
+
<string>APPL</string>
|
| 108 |
+
<key>CFBundleShortVersionString</key>
|
| 109 |
+
<string>1.0</string>
|
| 110 |
+
<key>CFBundleVersion</key>
|
| 111 |
+
<string>1</string>
|
| 112 |
+
<key>LSMinimumSystemVersion</key>
|
| 113 |
+
<string>12.0</string>
|
| 114 |
+
<key>NSHumanReadableCopyright</key>
|
| 115 |
+
<string>Copyright © 2025 GGML. All rights reserved.</string>
|
| 116 |
+
<key>NSPrincipalClass</key>
|
| 117 |
+
<string>NSApplication</string>
|
| 118 |
+
</dict>
|
| 119 |
+
</plist>
|
| 120 |
+
EOF
|
| 121 |
+
|
| 122 |
+
# Create SwiftUI app files
|
| 123 |
+
mkdir -p "${TEMP_DIR}/${APP_NAME}/${APP_NAME}/Sources"
|
| 124 |
+
|
| 125 |
+
# Create App.swift
|
| 126 |
+
cat > "${TEMP_DIR}/${APP_NAME}/${APP_NAME}/Sources/App.swift" << EOF
|
| 127 |
+
import SwiftUI
|
| 128 |
+
import llama
|
| 129 |
+
|
| 130 |
+
@main
|
| 131 |
+
struct LlamaTestApp: App {
|
| 132 |
+
var body: some Scene {
|
| 133 |
+
WindowGroup {
|
| 134 |
+
ContentView()
|
| 135 |
+
}
|
| 136 |
+
}
|
| 137 |
+
}
|
| 138 |
+
EOF
|
| 139 |
+
|
| 140 |
+
# Create ContentView.swift with macOS specific elements
|
| 141 |
+
cat > "${TEMP_DIR}/${APP_NAME}/${APP_NAME}/Sources/ContentView.swift" << EOF
|
| 142 |
+
import SwiftUI
|
| 143 |
+
import llama
|
| 144 |
+
|
| 145 |
+
struct ContentView: View {
|
| 146 |
+
// Test that we can initialize a llama context params struct
|
| 147 |
+
let params = llama_context_default_params()
|
| 148 |
+
|
| 149 |
+
var body: some View {
|
| 150 |
+
VStack(spacing: 20) {
|
| 151 |
+
Text("Llama Framework Test on macOS")
|
| 152 |
+
.font(.largeTitle)
|
| 153 |
+
.padding()
|
| 154 |
+
|
| 155 |
+
Text("llama_context_default_params() created successfully")
|
| 156 |
+
.font(.headline)
|
| 157 |
+
.multilineTextAlignment(.center)
|
| 158 |
+
.padding()
|
| 159 |
+
|
| 160 |
+
// Display some param values to confirm the framework is working
|
| 161 |
+
Text("n_ctx: \(params.n_ctx)")
|
| 162 |
+
.font(.body)
|
| 163 |
+
|
| 164 |
+
Text("n_batch: \(params.n_batch)")
|
| 165 |
+
.font(.body)
|
| 166 |
+
|
| 167 |
+
Spacer()
|
| 168 |
+
}
|
| 169 |
+
.padding()
|
| 170 |
+
.frame(width: 600, height: 400)
|
| 171 |
+
}
|
| 172 |
+
}
|
| 173 |
+
|
| 174 |
+
struct ContentView_Previews: PreviewProvider {
|
| 175 |
+
static var previews: some View {
|
| 176 |
+
ContentView()
|
| 177 |
+
}
|
| 178 |
+
}
|
| 179 |
+
EOF
|
| 180 |
+
|
| 181 |
+
# Create project.pbxproj, fixing the framework search paths issues
|
| 182 |
+
mkdir -p "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj"
|
| 183 |
+
cat > "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << 'EOF'
|
| 184 |
+
// !$*UTF8*$!
|
| 185 |
+
{
|
| 186 |
+
archiveVersion = 1;
|
| 187 |
+
classes = {
|
| 188 |
+
};
|
| 189 |
+
objectVersion = 54;
|
| 190 |
+
objects = {
|
| 191 |
+
|
| 192 |
+
/* Begin PBXBuildFile section */
|
| 193 |
+
11111111111111111111111 /* App.swift in Sources */ = {isa = PBXBuildFile; fileRef = 22222222222222222222222; };
|
| 194 |
+
33333333333333333333333 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 44444444444444444444444; };
|
| 195 |
+
55555555555555555555555 /* llama.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 66666666666666666666666; };
|
| 196 |
+
77777777777777777777777 /* llama.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 66666666666666666666666; };
|
| 197 |
+
/* End PBXBuildFile section */
|
| 198 |
+
|
| 199 |
+
/* Begin PBXCopyFilesBuildPhase section */
|
| 200 |
+
88888888888888888888888 /* Embed Frameworks */ = {
|
| 201 |
+
isa = PBXCopyFilesBuildPhase;
|
| 202 |
+
buildActionMask = 2147483647;
|
| 203 |
+
dstPath = "";
|
| 204 |
+
dstSubfolderSpec = 10;
|
| 205 |
+
files = (
|
| 206 |
+
77777777777777777777777 /* llama.xcframework in Embed Frameworks */,
|
| 207 |
+
);
|
| 208 |
+
name = "Embed Frameworks";
|
| 209 |
+
runOnlyForDeploymentPostprocessing = 0;
|
| 210 |
+
};
|
| 211 |
+
/* End PBXCopyFilesBuildPhase section */
|
| 212 |
+
|
| 213 |
+
/* Begin PBXFileReference section */
|
| 214 |
+
EOF
|
| 215 |
+
|
| 216 |
+
# Continue with the project.pbxproj file, using the APP_NAME variable appropriately
|
| 217 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << EOF
|
| 218 |
+
99999999999999999999999 /* ${APP_NAME}.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "${APP_NAME}.app"; sourceTree = BUILT_PRODUCTS_DIR; };
|
| 219 |
+
22222222222222222222222 /* App.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App.swift; sourceTree = "<group>"; };
|
| 220 |
+
44444444444444444444444 /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = "<group>"; };
|
| 221 |
+
AAAAAAAAAAAAAAAAAAAAAAA /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
|
| 222 |
+
66666666666666666666666 /* llama.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = llama.xcframework; sourceTree = "<group>"; };
|
| 223 |
+
/* End PBXFileReference section */
|
| 224 |
+
EOF
|
| 225 |
+
|
| 226 |
+
# Add the rest of the project file with fixed framework search paths
|
| 227 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << 'EOF'
|
| 228 |
+
/* Begin PBXFrameworksBuildPhase section */
|
| 229 |
+
BBBBBBBBBBBBBBBBBBBBBBBB /* Frameworks */ = {
|
| 230 |
+
isa = PBXFrameworksBuildPhase;
|
| 231 |
+
buildActionMask = 2147483647;
|
| 232 |
+
files = (
|
| 233 |
+
55555555555555555555555 /* llama.xcframework in Frameworks */,
|
| 234 |
+
);
|
| 235 |
+
runOnlyForDeploymentPostprocessing = 0;
|
| 236 |
+
};
|
| 237 |
+
/* End PBXFrameworksBuildPhase section */
|
| 238 |
+
|
| 239 |
+
/* Begin PBXGroup section */
|
| 240 |
+
EOF
|
| 241 |
+
|
| 242 |
+
# Continue with the project.pbxproj file, using the APP_NAME variable appropriately
|
| 243 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << EOF
|
| 244 |
+
CCCCCCCCCCCCCCCCCCCCCCCC /* Products */ = {
|
| 245 |
+
isa = PBXGroup;
|
| 246 |
+
children = (
|
| 247 |
+
99999999999999999999999 /* ${APP_NAME}.app */,
|
| 248 |
+
);
|
| 249 |
+
name = Products;
|
| 250 |
+
sourceTree = "<group>";
|
| 251 |
+
};
|
| 252 |
+
EOF
|
| 253 |
+
|
| 254 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << 'EOF'
|
| 255 |
+
DDDDDDDDDDDDDDDDDDDDDDDD /* Frameworks */ = {
|
| 256 |
+
isa = PBXGroup;
|
| 257 |
+
children = (
|
| 258 |
+
66666666666666666666666 /* llama.xcframework */,
|
| 259 |
+
);
|
| 260 |
+
name = Frameworks;
|
| 261 |
+
sourceTree = "<group>";
|
| 262 |
+
};
|
| 263 |
+
EEEEEEEEEEEEEEEEEEEEEEEE = {
|
| 264 |
+
isa = PBXGroup;
|
| 265 |
+
children = (
|
| 266 |
+
FFFFFFFFFFFFFFFFFFFFFFFF /* MacOSLlamaTest */,
|
| 267 |
+
CCCCCCCCCCCCCCCCCCCCCCCC /* Products */,
|
| 268 |
+
DDDDDDDDDDDDDDDDDDDDDDDD /* Frameworks */,
|
| 269 |
+
);
|
| 270 |
+
sourceTree = "<group>";
|
| 271 |
+
};
|
| 272 |
+
FFFFFFFFFFFFFFFFFFFFFFFF /* MacOSLlamaTest */ = {
|
| 273 |
+
isa = PBXGroup;
|
| 274 |
+
children = (
|
| 275 |
+
1111111111111111111111AA /* Sources */,
|
| 276 |
+
AAAAAAAAAAAAAAAAAAAAAAA /* Info.plist */,
|
| 277 |
+
);
|
| 278 |
+
path = "MacOSLlamaTest";
|
| 279 |
+
sourceTree = "<group>";
|
| 280 |
+
};
|
| 281 |
+
1111111111111111111111AA /* Sources */ = {
|
| 282 |
+
isa = PBXGroup;
|
| 283 |
+
children = (
|
| 284 |
+
22222222222222222222222 /* App.swift */,
|
| 285 |
+
44444444444444444444444 /* ContentView.swift */,
|
| 286 |
+
);
|
| 287 |
+
path = Sources;
|
| 288 |
+
sourceTree = "<group>";
|
| 289 |
+
};
|
| 290 |
+
/* End PBXGroup section */
|
| 291 |
+
EOF
|
| 292 |
+
|
| 293 |
+
# Continue with the project.pbxproj file, using the APP_NAME variable appropriately
|
| 294 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << EOF
|
| 295 |
+
/* Begin PBXNativeTarget section */
|
| 296 |
+
3333333333333333333333AA /* ${APP_NAME} */ = {
|
| 297 |
+
isa = PBXNativeTarget;
|
| 298 |
+
buildConfigurationList = 4444444444444444444444AA /* Build configuration list for PBXNativeTarget "${APP_NAME}" */;
|
| 299 |
+
buildPhases = (
|
| 300 |
+
5555555555555555555555AA /* Sources */,
|
| 301 |
+
BBBBBBBBBBBBBBBBBBBBBBBB /* Frameworks */,
|
| 302 |
+
6666666666666666666666AA /* Resources */,
|
| 303 |
+
88888888888888888888888 /* Embed Frameworks */,
|
| 304 |
+
);
|
| 305 |
+
buildRules = (
|
| 306 |
+
);
|
| 307 |
+
dependencies = (
|
| 308 |
+
);
|
| 309 |
+
name = "${APP_NAME}";
|
| 310 |
+
productName = "${APP_NAME}";
|
| 311 |
+
productReference = 99999999999999999999999 /* ${APP_NAME}.app */;
|
| 312 |
+
productType = "com.apple.product-type.application";
|
| 313 |
+
};
|
| 314 |
+
/* End PBXNativeTarget section */
|
| 315 |
+
|
| 316 |
+
/* Begin PBXProject section */
|
| 317 |
+
7777777777777777777777AA /* Project object */ = {
|
| 318 |
+
isa = PBXProject;
|
| 319 |
+
attributes = {
|
| 320 |
+
LastSwiftUpdateCheck = 1240;
|
| 321 |
+
LastUpgradeCheck = 1240;
|
| 322 |
+
TargetAttributes = {
|
| 323 |
+
3333333333333333333333AA = {
|
| 324 |
+
CreatedOnToolsVersion = 12.4;
|
| 325 |
+
};
|
| 326 |
+
};
|
| 327 |
+
};
|
| 328 |
+
buildConfigurationList = 8888888888888888888888AA /* Build configuration list for PBXProject "${APP_NAME}" */;
|
| 329 |
+
compatibilityVersion = "Xcode 12.0";
|
| 330 |
+
developmentRegion = en;
|
| 331 |
+
hasScannedForEncodings = 0;
|
| 332 |
+
knownRegions = (
|
| 333 |
+
en,
|
| 334 |
+
Base,
|
| 335 |
+
);
|
| 336 |
+
mainGroup = EEEEEEEEEEEEEEEEEEEEEEEE;
|
| 337 |
+
productRefGroup = CCCCCCCCCCCCCCCCCCCCCCCC /* Products */;
|
| 338 |
+
projectDirPath = "";
|
| 339 |
+
projectRoot = "";
|
| 340 |
+
targets = (
|
| 341 |
+
3333333333333333333333AA /* ${APP_NAME} */,
|
| 342 |
+
);
|
| 343 |
+
};
|
| 344 |
+
/* End PBXProject section */
|
| 345 |
+
EOF
|
| 346 |
+
|
| 347 |
+
# Add the rest of the file with correct FRAMEWORK_SEARCH_PATHS and macOS settings
|
| 348 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << 'EOF'
|
| 349 |
+
/* Begin PBXResourcesBuildPhase section */
|
| 350 |
+
6666666666666666666666AA /* Resources */ = {
|
| 351 |
+
isa = PBXResourcesBuildPhase;
|
| 352 |
+
buildActionMask = 2147483647;
|
| 353 |
+
files = (
|
| 354 |
+
);
|
| 355 |
+
runOnlyForDeploymentPostprocessing = 0;
|
| 356 |
+
};
|
| 357 |
+
/* End PBXResourcesBuildPhase section */
|
| 358 |
+
|
| 359 |
+
/* Begin PBXSourcesBuildPhase section */
|
| 360 |
+
5555555555555555555555AA /* Sources */ = {
|
| 361 |
+
isa = PBXSourcesBuildPhase;
|
| 362 |
+
buildActionMask = 2147483647;
|
| 363 |
+
files = (
|
| 364 |
+
33333333333333333333333 /* ContentView.swift in Sources */,
|
| 365 |
+
11111111111111111111111 /* App.swift in Sources */,
|
| 366 |
+
);
|
| 367 |
+
runOnlyForDeploymentPostprocessing = 0;
|
| 368 |
+
};
|
| 369 |
+
/* End PBXSourcesBuildPhase section */
|
| 370 |
+
|
| 371 |
+
/* Begin XCBuildConfiguration section */
|
| 372 |
+
9999999999999999999999AA /* Debug */ = {
|
| 373 |
+
isa = XCBuildConfiguration;
|
| 374 |
+
buildSettings = {
|
| 375 |
+
ALWAYS_SEARCH_USER_PATHS = NO;
|
| 376 |
+
CLANG_ANALYZER_NONNULL = YES;
|
| 377 |
+
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
|
| 378 |
+
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
|
| 379 |
+
CLANG_CXX_LIBRARY = "libc++";
|
| 380 |
+
CLANG_ENABLE_MODULES = YES;
|
| 381 |
+
CLANG_ENABLE_OBJC_ARC = YES;
|
| 382 |
+
CLANG_ENABLE_OBJC_WEAK = YES;
|
| 383 |
+
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
|
| 384 |
+
CLANG_WARN_BOOL_CONVERSION = YES;
|
| 385 |
+
CLANG_WARN_COMMA = YES;
|
| 386 |
+
CLANG_WARN_CONSTANT_CONVERSION = YES;
|
| 387 |
+
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
|
| 388 |
+
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
|
| 389 |
+
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
|
| 390 |
+
CLANG_WARN_EMPTY_BODY = YES;
|
| 391 |
+
CLANG_WARN_ENUM_CONVERSION = YES;
|
| 392 |
+
CLANG_WARN_INFINITE_RECURSION = YES;
|
| 393 |
+
CLANG_WARN_INT_CONVERSION = YES;
|
| 394 |
+
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
|
| 395 |
+
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
|
| 396 |
+
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
|
| 397 |
+
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
|
| 398 |
+
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
|
| 399 |
+
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
|
| 400 |
+
CLANG_WARN_STRICT_PROTOTYPES = YES;
|
| 401 |
+
CLANG_WARN_SUSPICIOUS_MOVE = YES;
|
| 402 |
+
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
|
| 403 |
+
CLANG_WARN_UNREACHABLE_CODE = YES;
|
| 404 |
+
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
|
| 405 |
+
COPY_PHASE_STRIP = NO;
|
| 406 |
+
DEBUG_INFORMATION_FORMAT = dwarf;
|
| 407 |
+
ENABLE_STRICT_OBJC_MSGSEND = YES;
|
| 408 |
+
ENABLE_TESTABILITY = YES;
|
| 409 |
+
GCC_C_LANGUAGE_STANDARD = gnu11;
|
| 410 |
+
GCC_DYNAMIC_NO_PIC = NO;
|
| 411 |
+
GCC_NO_COMMON_BLOCKS = YES;
|
| 412 |
+
GCC_OPTIMIZATION_LEVEL = 0;
|
| 413 |
+
GCC_PREPROCESSOR_DEFINITIONS = (
|
| 414 |
+
"DEBUG=1",
|
| 415 |
+
"$(inherited)",
|
| 416 |
+
);
|
| 417 |
+
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
|
| 418 |
+
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
|
| 419 |
+
GCC_WARN_UNDECLARED_SELECTOR = YES;
|
| 420 |
+
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
|
| 421 |
+
GCC_WARN_UNUSED_FUNCTION = YES;
|
| 422 |
+
GCC_WARN_UNUSED_VARIABLE = YES;
|
| 423 |
+
MACOSX_DEPLOYMENT_TARGET = 12.0;
|
| 424 |
+
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
|
| 425 |
+
MTL_FAST_MATH = YES;
|
| 426 |
+
ONLY_ACTIVE_ARCH = YES;
|
| 427 |
+
SDKROOT = macosx;
|
| 428 |
+
SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
|
| 429 |
+
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
|
| 430 |
+
};
|
| 431 |
+
name = Debug;
|
| 432 |
+
};
|
| 433 |
+
AAAAAAAAAAAAAAAAAAAAABBB /* Release */ = {
|
| 434 |
+
isa = XCBuildConfiguration;
|
| 435 |
+
buildSettings = {
|
| 436 |
+
ALWAYS_SEARCH_USER_PATHS = NO;
|
| 437 |
+
CLANG_ANALYZER_NONNULL = YES;
|
| 438 |
+
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
|
| 439 |
+
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
|
| 440 |
+
CLANG_CXX_LIBRARY = "libc++";
|
| 441 |
+
CLANG_ENABLE_MODULES = YES;
|
| 442 |
+
CLANG_ENABLE_OBJC_ARC = YES;
|
| 443 |
+
CLANG_ENABLE_OBJC_WEAK = YES;
|
| 444 |
+
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
|
| 445 |
+
CLANG_WARN_BOOL_CONVERSION = YES;
|
| 446 |
+
CLANG_WARN_COMMA = YES;
|
| 447 |
+
CLANG_WARN_CONSTANT_CONVERSION = YES;
|
| 448 |
+
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
|
| 449 |
+
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
|
| 450 |
+
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
|
| 451 |
+
CLANG_WARN_EMPTY_BODY = YES;
|
| 452 |
+
CLANG_WARN_ENUM_CONVERSION = YES;
|
| 453 |
+
CLANG_WARN_INFINITE_RECURSION = YES;
|
| 454 |
+
CLANG_WARN_INT_CONVERSION = YES;
|
| 455 |
+
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
|
| 456 |
+
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
|
| 457 |
+
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
|
| 458 |
+
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
|
| 459 |
+
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
|
| 460 |
+
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
|
| 461 |
+
CLANG_WARN_STRICT_PROTOTYPES = YES;
|
| 462 |
+
CLANG_WARN_SUSPICIOUS_MOVE = YES;
|
| 463 |
+
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
|
| 464 |
+
CLANG_WARN_UNREACHABLE_CODE = YES;
|
| 465 |
+
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
|
| 466 |
+
COPY_PHASE_STRIP = NO;
|
| 467 |
+
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
|
| 468 |
+
ENABLE_NS_ASSERTIONS = NO;
|
| 469 |
+
ENABLE_STRICT_OBJC_MSGSEND = YES;
|
| 470 |
+
GCC_C_LANGUAGE_STANDARD = gnu11;
|
| 471 |
+
GCC_NO_COMMON_BLOCKS = YES;
|
| 472 |
+
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
|
| 473 |
+
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
|
| 474 |
+
GCC_WARN_UNDECLARED_SELECTOR = YES;
|
| 475 |
+
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
|
| 476 |
+
GCC_WARN_UNUSED_FUNCTION = YES;
|
| 477 |
+
GCC_WARN_UNUSED_VARIABLE = YES;
|
| 478 |
+
MACOSX_DEPLOYMENT_TARGET = 12.0;
|
| 479 |
+
MTL_ENABLE_DEBUG_INFO = NO;
|
| 480 |
+
MTL_FAST_MATH = YES;
|
| 481 |
+
SDKROOT = macosx;
|
| 482 |
+
SWIFT_COMPILATION_MODE = wholemodule;
|
| 483 |
+
SWIFT_OPTIMIZATION_LEVEL = "-O";
|
| 484 |
+
};
|
| 485 |
+
name = Release;
|
| 486 |
+
};
|
| 487 |
+
BBBBBBBBBBBBBBBBBBBBBBCCC /* Debug */ = {
|
| 488 |
+
isa = XCBuildConfiguration;
|
| 489 |
+
buildSettings = {
|
| 490 |
+
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
| 491 |
+
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
|
| 492 |
+
CODE_SIGN_STYLE = Manual;
|
| 493 |
+
COMBINE_HIDPI_IMAGES = YES;
|
| 494 |
+
DEVELOPMENT_TEAM = "";
|
| 495 |
+
ENABLE_HARDENED_RUNTIME = YES;
|
| 496 |
+
ENABLE_PREVIEWS = YES;
|
| 497 |
+
FRAMEWORK_SEARCH_PATHS = "$(PROJECT_DIR)";
|
| 498 |
+
INFOPLIST_FILE = "MacOSLlamaTest/Info.plist";
|
| 499 |
+
LD_RUNPATH_SEARCH_PATHS = (
|
| 500 |
+
"$(inherited)",
|
| 501 |
+
"@executable_path/../Frameworks",
|
| 502 |
+
);
|
| 503 |
+
PRODUCT_BUNDLE_IDENTIFIER = "org.ggml.MacOSLlamaTest";
|
| 504 |
+
PRODUCT_NAME = "$(TARGET_NAME)";
|
| 505 |
+
PROVISIONING_PROFILE_SPECIFIER = "";
|
| 506 |
+
SWIFT_VERSION = 5.0;
|
| 507 |
+
};
|
| 508 |
+
name = Debug;
|
| 509 |
+
};
|
| 510 |
+
CCCCCCCCCCCCCCCCCCCCCCDDD /* Release */ = {
|
| 511 |
+
isa = XCBuildConfiguration;
|
| 512 |
+
buildSettings = {
|
| 513 |
+
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
| 514 |
+
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
|
| 515 |
+
CODE_SIGN_STYLE = Manual;
|
| 516 |
+
COMBINE_HIDPI_IMAGES = YES;
|
| 517 |
+
DEVELOPMENT_TEAM = "";
|
| 518 |
+
ENABLE_HARDENED_RUNTIME = YES;
|
| 519 |
+
ENABLE_PREVIEWS = YES;
|
| 520 |
+
FRAMEWORK_SEARCH_PATHS = (
|
| 521 |
+
"$(inherited)",
|
| 522 |
+
"$(PROJECT_DIR)",
|
| 523 |
+
);
|
| 524 |
+
INFOPLIST_FILE = "MacOSLlamaTest/Info.plist";
|
| 525 |
+
LD_RUNPATH_SEARCH_PATHS = (
|
| 526 |
+
"$(inherited)",
|
| 527 |
+
"@executable_path/../Frameworks",
|
| 528 |
+
);
|
| 529 |
+
PRODUCT_BUNDLE_IDENTIFIER = "org.ggml.MacOSLlamaTest";
|
| 530 |
+
PRODUCT_NAME = "$(TARGET_NAME)";
|
| 531 |
+
PROVISIONING_PROFILE_SPECIFIER = "";
|
| 532 |
+
SWIFT_VERSION = 5.0;
|
| 533 |
+
};
|
| 534 |
+
name = Release;
|
| 535 |
+
};
|
| 536 |
+
/* End XCBuildConfiguration section */
|
| 537 |
+
EOF
|
| 538 |
+
|
| 539 |
+
# Finish the project.pbxproj file
|
| 540 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << EOF
|
| 541 |
+
/* Begin XCConfigurationList section */
|
| 542 |
+
8888888888888888888888AA /* Build configuration list for PBXProject "${APP_NAME}" */ = {
|
| 543 |
+
isa = XCConfigurationList;
|
| 544 |
+
buildConfigurations = (
|
| 545 |
+
9999999999999999999999AA /* Debug */,
|
| 546 |
+
AAAAAAAAAAAAAAAAAAAAABBB /* Release */,
|
| 547 |
+
);
|
| 548 |
+
defaultConfigurationIsVisible = 0;
|
| 549 |
+
defaultConfigurationName = Release;
|
| 550 |
+
};
|
| 551 |
+
4444444444444444444444AA /* Build configuration list for PBXNativeTarget "${APP_NAME}" */ = {
|
| 552 |
+
isa = XCConfigurationList;
|
| 553 |
+
buildConfigurations = (
|
| 554 |
+
BBBBBBBBBBBBBBBBBBBBBBCCC /* Debug */,
|
| 555 |
+
CCCCCCCCCCCCCCCCCCCCCCDDD /* Release */,
|
| 556 |
+
);
|
| 557 |
+
defaultConfigurationIsVisible = 0;
|
| 558 |
+
defaultConfigurationName = Release;
|
| 559 |
+
};
|
| 560 |
+
/* End XCConfigurationList section */
|
| 561 |
+
};
|
| 562 |
+
rootObject = 7777777777777777777777AA /* Project object */;
|
| 563 |
+
}
|
| 564 |
+
EOF
|
| 565 |
+
|
| 566 |
+
# 2. Copy XCFramework to test project
|
| 567 |
+
echo "Copying XCFramework to test project..."
|
| 568 |
+
cp -R "${XCFRAMEWORK_PATH}" "${TEMP_DIR}/${APP_NAME}/"
|
| 569 |
+
|
| 570 |
+
# 3. Build and archive the app
|
| 571 |
+
echo "Building and archiving test app..."
|
| 572 |
+
cd "${TEMP_DIR}/${APP_NAME}"
|
| 573 |
+
|
| 574 |
+
# Create a simple xcscheme file to avoid xcodebuild scheme issues
|
| 575 |
+
mkdir -p "${APP_NAME}.xcodeproj/xcshareddata/xcschemes"
|
| 576 |
+
cat > "${APP_NAME}.xcodeproj/xcshareddata/xcschemes/${APP_NAME}.xcscheme" << EOF
|
| 577 |
+
<?xml version="1.0" encoding="UTF-8"?>
|
| 578 |
+
<Scheme
|
| 579 |
+
LastUpgradeVersion = "1240"
|
| 580 |
+
version = "1.3">
|
| 581 |
+
<BuildAction
|
| 582 |
+
parallelizeBuildables = "YES"
|
| 583 |
+
buildImplicitDependencies = "YES">
|
| 584 |
+
<BuildActionEntries>
|
| 585 |
+
<BuildActionEntry
|
| 586 |
+
buildForTesting = "YES"
|
| 587 |
+
buildForRunning = "YES"
|
| 588 |
+
buildForProfiling = "YES"
|
| 589 |
+
buildForArchiving = "YES"
|
| 590 |
+
buildForAnalyzing = "YES">
|
| 591 |
+
<BuildableReference
|
| 592 |
+
BuildableIdentifier = "primary"
|
| 593 |
+
BlueprintIdentifier = "3333333333333333333333AA"
|
| 594 |
+
BuildableName = "${APP_NAME}.app"
|
| 595 |
+
BlueprintName = "${APP_NAME}"
|
| 596 |
+
ReferencedContainer = "container:${APP_NAME}.xcodeproj">
|
| 597 |
+
</BuildableReference>
|
| 598 |
+
</BuildActionEntry>
|
| 599 |
+
</BuildActionEntries>
|
| 600 |
+
</BuildAction>
|
| 601 |
+
<TestAction
|
| 602 |
+
buildConfiguration = "Debug"
|
| 603 |
+
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
|
| 604 |
+
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
|
| 605 |
+
shouldUseLaunchSchemeArgsEnv = "YES">
|
| 606 |
+
<Testables>
|
| 607 |
+
</Testables>
|
| 608 |
+
</TestAction>
|
| 609 |
+
<LaunchAction
|
| 610 |
+
buildConfiguration = "Debug"
|
| 611 |
+
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
|
| 612 |
+
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
|
| 613 |
+
launchStyle = "0"
|
| 614 |
+
useCustomWorkingDirectory = "NO"
|
| 615 |
+
ignoresPersistentStateOnLaunch = "NO"
|
| 616 |
+
debugDocumentVersioning = "YES"
|
| 617 |
+
debugServiceExtension = "internal"
|
| 618 |
+
allowLocationSimulation = "YES">
|
| 619 |
+
<BuildableProductRunnable
|
| 620 |
+
runnableDebuggingMode = "0">
|
| 621 |
+
<BuildableReference
|
| 622 |
+
BuildableIdentifier = "primary"
|
| 623 |
+
BlueprintIdentifier = "3333333333333333333333AA"
|
| 624 |
+
BuildableName = "${APP_NAME}.app"
|
| 625 |
+
BlueprintName = "${APP_NAME}"
|
| 626 |
+
ReferencedContainer = "container:${APP_NAME}.xcodeproj">
|
| 627 |
+
</BuildableReference>
|
| 628 |
+
</BuildableProductRunnable>
|
| 629 |
+
</LaunchAction>
|
| 630 |
+
<ProfileAction
|
| 631 |
+
buildConfiguration = "Release"
|
| 632 |
+
shouldUseLaunchSchemeArgsEnv = "YES"
|
| 633 |
+
savedToolIdentifier = ""
|
| 634 |
+
useCustomWorkingDirectory = "NO"
|
| 635 |
+
debugDocumentVersioning = "YES">
|
| 636 |
+
<BuildableProductRunnable
|
| 637 |
+
runnableDebuggingMode = "0">
|
| 638 |
+
<BuildableReference
|
| 639 |
+
BuildableIdentifier = "primary"
|
| 640 |
+
BlueprintIdentifier = "3333333333333333333333AA"
|
| 641 |
+
BuildableName = "${APP_NAME}.app"
|
| 642 |
+
BlueprintName = "${APP_NAME}"
|
| 643 |
+
ReferencedContainer = "container:${APP_NAME}.xcodeproj">
|
| 644 |
+
</BuildableReference>
|
| 645 |
+
</BuildableProductRunnable>
|
| 646 |
+
</ProfileAction>
|
| 647 |
+
<AnalyzeAction
|
| 648 |
+
buildConfiguration = "Debug">
|
| 649 |
+
</AnalyzeAction>
|
| 650 |
+
<ArchiveAction
|
| 651 |
+
buildConfiguration = "Release"
|
| 652 |
+
revealArchiveInOrganizer = "YES">
|
| 653 |
+
</ArchiveAction>
|
| 654 |
+
</Scheme>
|
| 655 |
+
EOF
|
| 656 |
+
|
| 657 |
+
# Now use xcodebuild with an explicitly defined product name for macOS
|
| 658 |
+
xcodebuild -project "${APP_NAME}.xcodeproj" -scheme "${APP_NAME}" -sdk macosx -configuration Release archive -archivePath "${ARCHIVE_PATH}" CODE_SIGN_IDENTITY="-" CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO PRODUCT_NAME="${APP_NAME}" SWIFT_OPTIMIZATION_LEVEL="-Onone" -quiet
|
| 659 |
+
|
| 660 |
+
# 4. Create a package for distribution
|
| 661 |
+
echo "Creating distributable package from archive..."
|
| 662 |
+
cp -R "${ARCHIVE_PATH}/Products/Applications/${APP_NAME}.app" "${APP_PATH}"
|
| 663 |
+
|
| 664 |
+
# Check and log app structure
|
| 665 |
+
echo "App structure:"
|
| 666 |
+
ls -la "${APP_PATH}"
|
| 667 |
+
echo "Frameworks:"
|
| 668 |
+
ls -la "${APP_PATH}/Contents/Frameworks/" 2>/dev/null || echo "No Frameworks directory found"
|
| 669 |
+
|
| 670 |
+
# Create a zip file for potential distribution
|
| 671 |
+
cd "${BUILD_DIR}"
|
| 672 |
+
zip -r "${ZIP_PATH}" "${APP_NAME}.app"
|
| 673 |
+
|
| 674 |
+
# Check embedded provisioning profile
|
| 675 |
+
echo "Checking provisioning profile (if any)..."
|
| 676 |
+
PROVISIONING_PROFILE=$(find "${APP_PATH}/Contents" -name "embedded.provisionprofile" 2>/dev/null)
|
| 677 |
+
if [ -n "$PROVISIONING_PROFILE" ]; then
|
| 678 |
+
echo "Found embedded provisioning profile:"
|
| 679 |
+
security cms -D -i "$PROVISIONING_PROFILE" || echo "Unable to decode provisioning profile"
|
| 680 |
+
else
|
| 681 |
+
echo "No embedded provisioning profile found (expected for ad-hoc builds)"
|
| 682 |
+
fi
|
| 683 |
+
|
| 684 |
+
# 5. Validate the app
|
| 685 |
+
echo "Validating macOS app..."
|
| 686 |
+
VALIDATION_OUTPUT="${VALIDATION_DIR}/validation_output.txt"
|
| 687 |
+
|
| 688 |
+
# Check if authentication credentials are provided
|
| 689 |
+
AUTH_ARGS=""
|
| 690 |
+
if [ -n "$APPLE_ID" ] && [ -n "$APPLE_PASSWORD" ]; then
|
| 691 |
+
echo "Using Apple ID authentication for validation..."
|
| 692 |
+
AUTH_ARGS="--username \"$APPLE_ID\" --password \"$APPLE_PASSWORD\""
|
| 693 |
+
else
|
| 694 |
+
echo "No authentication credentials provided. Will perform basic validation."
|
| 695 |
+
echo "To use your personal developer account, you can run the script with:"
|
| 696 |
+
echo " APPLE_ID='your.email@example.com' APPLE_PASSWORD='your-app-specific-password' ./validate-macos.sh"
|
| 697 |
+
echo "Note: You need to create an app-specific password at https://appleid.apple.com/account/manage"
|
| 698 |
+
fi
|
| 699 |
+
|
| 700 |
+
# For macOS we need to use notarytool or alternative checks because altool doesn't support macOS apps in the same way
|
| 701 |
+
echo "Note: For macOS, formal notarization process would require Apple Developer credentials."
|
| 702 |
+
echo "Performing alternative validation checks..."
|
| 703 |
+
|
| 704 |
+
# Final validation result
|
| 705 |
+
FINAL_VALIDATION_RESULT=0
|
| 706 |
+
|
| 707 |
+
# Check if app was created successfully
|
| 708 |
+
if [ -d "${APP_PATH}" ] && [ -s "${APP_PATH}/Contents/MacOS/${APP_NAME}" ]; then
|
| 709 |
+
echo "✅ App package created successfully"
|
| 710 |
+
else
|
| 711 |
+
echo "❌ App package not created or binary missing"
|
| 712 |
+
FINAL_VALIDATION_RESULT=1
|
| 713 |
+
fi
|
| 714 |
+
|
| 715 |
+
# Check if app binary exists and is executable
|
| 716 |
+
if [ -f "${APP_PATH}/Contents/MacOS/${APP_NAME}" ] && [ -x "${APP_PATH}/Contents/MacOS/${APP_NAME}" ]; then
|
| 717 |
+
echo "✅ App binary exists and is executable"
|
| 718 |
+
else
|
| 719 |
+
echo "❌ App binary missing or not executable"
|
| 720 |
+
FINAL_VALIDATION_RESULT=1
|
| 721 |
+
fi
|
| 722 |
+
|
| 723 |
+
# Check if framework was properly embedded
|
| 724 |
+
if [ -d "${APP_PATH}/Contents/Frameworks/llama.framework" ]; then
|
| 725 |
+
echo "✅ llama.framework properly embedded"
|
| 726 |
+
else
|
| 727 |
+
echo "❌ llama.framework not properly embedded"
|
| 728 |
+
FINAL_VALIDATION_RESULT=1
|
| 729 |
+
fi
|
| 730 |
+
|
| 731 |
+
# Check if framework binary exists
|
| 732 |
+
if [ -f "${APP_PATH}/Contents/Frameworks/llama.framework/Versions/A/llama" ]; then
|
| 733 |
+
echo "✅ Framework binary exists"
|
| 734 |
+
|
| 735 |
+
# Further validate framework by checking architecture
|
| 736 |
+
ARCHS=$(lipo -info "${APP_PATH}/Contents/Frameworks/llama.framework/Versions/A/llama" 2>/dev/null | grep -o "arm64\\|x86_64" | tr '\n' ' ')
|
| 737 |
+
if [ -n "$ARCHS" ]; then
|
| 738 |
+
echo "✅ Framework architecture(s): $ARCHS"
|
| 739 |
+
else
|
| 740 |
+
echo "⚠️ Could not determine framework architecture"
|
| 741 |
+
fi
|
| 742 |
+
else
|
| 743 |
+
echo "❌ Framework binary missing"
|
| 744 |
+
FINAL_VALIDATION_RESULT=1
|
| 745 |
+
fi
|
| 746 |
+
|
| 747 |
+
# Check code signing
|
| 748 |
+
echo ""
|
| 749 |
+
echo "==== CODE SIGNING INFO ===="
|
| 750 |
+
codesign -vv -d "${APP_PATH}" 2>&1 || echo "Code signing verification not available (expected for ad-hoc builds)"
|
| 751 |
+
|
| 752 |
+
if [ $FINAL_VALIDATION_RESULT -eq 0 ]; then
|
| 753 |
+
if [ -n "$AUTH_ARGS" ]; then
|
| 754 |
+
echo ""
|
| 755 |
+
echo "To notarize this app with Apple (requires Apple Developer account):"
|
| 756 |
+
echo "xcrun notarytool submit \"${ZIP_PATH}\" --apple-id \"your-apple-id\" --password \"your-app-specific-password\" --team-id \"your-team-id\" --wait"
|
| 757 |
+
echo ""
|
| 758 |
+
fi
|
| 759 |
+
echo "✅ Validation PASSED: macOS app built successfully with embedded framework"
|
| 760 |
+
else
|
| 761 |
+
echo "❌ Validation FAILED: Issues found with the app or framework"
|
| 762 |
+
fi
|
| 763 |
+
|
| 764 |
+
# Don't clean up on error to allow inspection
|
| 765 |
+
if [ $FINAL_VALIDATION_RESULT -ne 0 ]; then
|
| 766 |
+
echo ""
|
| 767 |
+
echo "Temporary files kept for inspection at: ${TEMP_DIR}"
|
| 768 |
+
echo "===== macOS Validation Process Failed ====="
|
| 769 |
+
exit 1
|
| 770 |
+
fi
|
| 771 |
+
|
| 772 |
+
# Clean up temporary files but keep build artifacts
|
| 773 |
+
if [ $FINAL_VALIDATION_RESULT -eq 0 ]; then
|
| 774 |
+
echo "Cleaning up temporary files..."
|
| 775 |
+
#rm -rf "${TEMP_DIR}"
|
| 776 |
+
fi
|
| 777 |
+
|
| 778 |
+
echo "===== macOS Validation Process Completed ====="
|
| 779 |
+
echo "App package available at: ${APP_PATH}"
|
| 780 |
+
echo "Zipped app available at: ${ZIP_PATH}"
|
| 781 |
+
exit $FINAL_VALIDATION_RESULT
|
backend/llama.cpp/scripts/apple/validate-tvos.sh
ADDED
|
@@ -0,0 +1,813 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# validate-tvos.sh - Validate tvOS Application with embedded llama.xcframework using SwiftUI
|
| 3 |
+
|
| 4 |
+
# Authentication options (optional) (can be set via environment variables)
|
| 5 |
+
# To use: export APPLE_ID=your.email@example.com
|
| 6 |
+
# export APPLE_PASSWORD=your-app-specific-password
|
| 7 |
+
# ./validate-tvos.sh
|
| 8 |
+
APPLE_ID=${APPLE_ID:-""}
|
| 9 |
+
APPLE_PASSWORD=${APPLE_PASSWORD:-""}
|
| 10 |
+
|
| 11 |
+
# Ensure the script exits on error
|
| 12 |
+
set -e
|
| 13 |
+
|
| 14 |
+
# Function to print usage instructions
|
| 15 |
+
print_usage() {
|
| 16 |
+
echo "Usage: ./validate-tvos.sh [OPTIONS]"
|
| 17 |
+
echo ""
|
| 18 |
+
echo "Options:"
|
| 19 |
+
echo " --help Show this help message"
|
| 20 |
+
echo " --apple-id EMAIL Apple ID email for validation"
|
| 21 |
+
echo " --apple-password PWD App-specific password for Apple ID"
|
| 22 |
+
echo ""
|
| 23 |
+
echo "Environment variables:"
|
| 24 |
+
echo " APPLE_ID Apple ID email for validation"
|
| 25 |
+
echo " APPLE_PASSWORD App-specific password for Apple ID"
|
| 26 |
+
echo ""
|
| 27 |
+
echo "Notes:"
|
| 28 |
+
echo " - Command line options take precedence over environment variables"
|
| 29 |
+
echo " - Authentication is optional. If not provided, alternative validation will be performed"
|
| 30 |
+
echo " - For APPLE_PASSWORD, use an app-specific password generated at https://appleid.apple.com/account/manage"
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
# Parse command line arguments
|
| 34 |
+
while [[ $# -gt 0 ]]; do
|
| 35 |
+
case $1 in
|
| 36 |
+
--help)
|
| 37 |
+
print_usage
|
| 38 |
+
exit 0
|
| 39 |
+
;;
|
| 40 |
+
--apple-id)
|
| 41 |
+
APPLE_ID="$2"
|
| 42 |
+
shift 2
|
| 43 |
+
;;
|
| 44 |
+
--apple-password)
|
| 45 |
+
APPLE_PASSWORD="$2"
|
| 46 |
+
shift 2
|
| 47 |
+
;;
|
| 48 |
+
*)
|
| 49 |
+
echo "Unknown option: $1"
|
| 50 |
+
print_usage
|
| 51 |
+
exit 1
|
| 52 |
+
;;
|
| 53 |
+
esac
|
| 54 |
+
done
|
| 55 |
+
|
| 56 |
+
# Function to clean up in case of error
|
| 57 |
+
cleanup() {
|
| 58 |
+
# Don't clean up temp files on error to help with debugging
|
| 59 |
+
echo "===== tvOS Validation Process Failed ====="
|
| 60 |
+
exit 1
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
# Set up trap to call cleanup function on error
|
| 64 |
+
trap cleanup ERR
|
| 65 |
+
|
| 66 |
+
set -e # Exit on any error
|
| 67 |
+
|
| 68 |
+
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )"
|
| 69 |
+
BUILD_DIR="${ROOT_DIR}/validation-builds/ios"
|
| 70 |
+
|
| 71 |
+
# Configuration
|
| 72 |
+
APP_NAME="TVOSLlamaTest"
|
| 73 |
+
BUNDLE_ID="org.ggml.TVOSLlamaTest"
|
| 74 |
+
XCFRAMEWORK_PATH="${ROOT_DIR}/build-apple/llama.xcframework"
|
| 75 |
+
TEMP_DIR="${BUILD_DIR}/temp"
|
| 76 |
+
ARCHIVE_PATH="${BUILD_DIR}/${APP_NAME}.xcarchive"
|
| 77 |
+
IPA_PATH="${BUILD_DIR}/${APP_NAME}.ipa"
|
| 78 |
+
VALIDATION_DIR="${BUILD_DIR}/validation"
|
| 79 |
+
|
| 80 |
+
# Create necessary directories
|
| 81 |
+
mkdir -p "${BUILD_DIR}"
|
| 82 |
+
mkdir -p "${TEMP_DIR}"
|
| 83 |
+
mkdir -p "${VALIDATION_DIR}"
|
| 84 |
+
|
| 85 |
+
echo "===== tvOS Validation Process Started ====="
|
| 86 |
+
|
| 87 |
+
# 1. Create a simple test app project
|
| 88 |
+
echo "Creating test tvOS app project..."
|
| 89 |
+
mkdir -p "${TEMP_DIR}/${APP_NAME}/${APP_NAME}"
|
| 90 |
+
cat > "${TEMP_DIR}/${APP_NAME}/${APP_NAME}/Info.plist" << EOF
|
| 91 |
+
<?xml version="1.0" encoding="UTF-8"?>
|
| 92 |
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
| 93 |
+
<plist version="1.0">
|
| 94 |
+
<dict>
|
| 95 |
+
<key>CFBundleDevelopmentRegion</key>
|
| 96 |
+
<string>en</string>
|
| 97 |
+
<key>CFBundleExecutable</key>
|
| 98 |
+
<string>${APP_NAME}</string>
|
| 99 |
+
<key>CFBundleIdentifier</key>
|
| 100 |
+
<string>${BUNDLE_ID}</string>
|
| 101 |
+
<key>CFBundleInfoDictionaryVersion</key>
|
| 102 |
+
<string>6.0</string>
|
| 103 |
+
<key>CFBundleName</key>
|
| 104 |
+
<string>${APP_NAME}</string>
|
| 105 |
+
<key>CFBundlePackageType</key>
|
| 106 |
+
<string>APPL</string>
|
| 107 |
+
<key>CFBundleShortVersionString</key>
|
| 108 |
+
<string>1.0</string>
|
| 109 |
+
<key>CFBundleVersion</key>
|
| 110 |
+
<string>1</string>
|
| 111 |
+
<key>UIRequiredDeviceCapabilities</key>
|
| 112 |
+
<array>
|
| 113 |
+
<string>arm64</string>
|
| 114 |
+
</array>
|
| 115 |
+
</dict>
|
| 116 |
+
</plist>
|
| 117 |
+
EOF
|
| 118 |
+
|
| 119 |
+
# Create SwiftUI app files
|
| 120 |
+
mkdir -p "${TEMP_DIR}/${APP_NAME}/${APP_NAME}/Sources"
|
| 121 |
+
|
| 122 |
+
# Create App.swift
|
| 123 |
+
cat > "${TEMP_DIR}/${APP_NAME}/${APP_NAME}/Sources/App.swift" << EOF
|
| 124 |
+
import SwiftUI
|
| 125 |
+
import llama
|
| 126 |
+
|
| 127 |
+
@main
|
| 128 |
+
struct LlamaTestApp: App {
|
| 129 |
+
var body: some Scene {
|
| 130 |
+
WindowGroup {
|
| 131 |
+
ContentView()
|
| 132 |
+
}
|
| 133 |
+
}
|
| 134 |
+
}
|
| 135 |
+
EOF
|
| 136 |
+
|
| 137 |
+
# Create ContentView.swift with tvOS specific elements
|
| 138 |
+
cat > "${TEMP_DIR}/${APP_NAME}/${APP_NAME}/Sources/ContentView.swift" << EOF
|
| 139 |
+
import SwiftUI
|
| 140 |
+
import llama
|
| 141 |
+
|
| 142 |
+
struct ContentView: View {
|
| 143 |
+
// Test that we can initialize a llama context params struct
|
| 144 |
+
let params = llama_context_default_params()
|
| 145 |
+
|
| 146 |
+
var body: some View {
|
| 147 |
+
VStack(spacing: 40) {
|
| 148 |
+
Text("Llama Framework Test on tvOS")
|
| 149 |
+
.font(.largeTitle)
|
| 150 |
+
.padding()
|
| 151 |
+
|
| 152 |
+
Text("llama_context_default_params() created successfully")
|
| 153 |
+
.font(.headline)
|
| 154 |
+
.multilineTextAlignment(.center)
|
| 155 |
+
.padding()
|
| 156 |
+
|
| 157 |
+
// Display some param values to confirm the framework is working
|
| 158 |
+
Text("n_ctx: \(params.n_ctx)")
|
| 159 |
+
.font(.title2)
|
| 160 |
+
|
| 161 |
+
Text("n_batch: \(params.n_batch)")
|
| 162 |
+
.font(.title2)
|
| 163 |
+
|
| 164 |
+
Spacer()
|
| 165 |
+
}
|
| 166 |
+
.padding(50)
|
| 167 |
+
// Larger size suitable for TV display
|
| 168 |
+
}
|
| 169 |
+
}
|
| 170 |
+
|
| 171 |
+
struct ContentView_Previews: PreviewProvider {
|
| 172 |
+
static var previews: some View {
|
| 173 |
+
ContentView()
|
| 174 |
+
}
|
| 175 |
+
}
|
| 176 |
+
EOF
|
| 177 |
+
|
| 178 |
+
# Create project.pbxproj, fixing the framework search paths issues
|
| 179 |
+
mkdir -p "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj"
|
| 180 |
+
cat > "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << 'EOF'
|
| 181 |
+
// !$*UTF8*$!
|
| 182 |
+
{
|
| 183 |
+
archiveVersion = 1;
|
| 184 |
+
classes = {
|
| 185 |
+
};
|
| 186 |
+
objectVersion = 54;
|
| 187 |
+
objects = {
|
| 188 |
+
|
| 189 |
+
/* Begin PBXBuildFile section */
|
| 190 |
+
11111111111111111111111 /* App.swift in Sources */ = {isa = PBXBuildFile; fileRef = 22222222222222222222222; };
|
| 191 |
+
33333333333333333333333 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 44444444444444444444444; };
|
| 192 |
+
55555555555555555555555 /* llama.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 66666666666666666666666; };
|
| 193 |
+
77777777777777777777777 /* llama.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 66666666666666666666666; };
|
| 194 |
+
/* End PBXBuildFile section */
|
| 195 |
+
|
| 196 |
+
/* Begin PBXCopyFilesBuildPhase section */
|
| 197 |
+
88888888888888888888888 /* Embed Frameworks */ = {
|
| 198 |
+
isa = PBXCopyFilesBuildPhase;
|
| 199 |
+
buildActionMask = 2147483647;
|
| 200 |
+
dstPath = "";
|
| 201 |
+
dstSubfolderSpec = 10;
|
| 202 |
+
files = (
|
| 203 |
+
77777777777777777777777 /* llama.xcframework in Embed Frameworks */,
|
| 204 |
+
);
|
| 205 |
+
name = "Embed Frameworks";
|
| 206 |
+
runOnlyForDeploymentPostprocessing = 0;
|
| 207 |
+
};
|
| 208 |
+
/* End PBXCopyFilesBuildPhase section */
|
| 209 |
+
|
| 210 |
+
/* Begin PBXFileReference section */
|
| 211 |
+
EOF
|
| 212 |
+
|
| 213 |
+
# Continue with the project.pbxproj file, using the APP_NAME variable appropriately
|
| 214 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << EOF
|
| 215 |
+
99999999999999999999999 /* ${APP_NAME}.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "${APP_NAME}.app"; sourceTree = BUILT_PRODUCTS_DIR; };
|
| 216 |
+
22222222222222222222222 /* App.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App.swift; sourceTree = "<group>"; };
|
| 217 |
+
44444444444444444444444 /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = "<group>"; };
|
| 218 |
+
AAAAAAAAAAAAAAAAAAAAAAA /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
|
| 219 |
+
66666666666666666666666 /* llama.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = llama.xcframework; sourceTree = "<group>"; };
|
| 220 |
+
/* End PBXFileReference section */
|
| 221 |
+
EOF
|
| 222 |
+
|
| 223 |
+
# Add the rest of the project file with fixed framework search paths
|
| 224 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << 'EOF'
|
| 225 |
+
/* Begin PBXFrameworksBuildPhase section */
|
| 226 |
+
BBBBBBBBBBBBBBBBBBBBBBBB /* Frameworks */ = {
|
| 227 |
+
isa = PBXFrameworksBuildPhase;
|
| 228 |
+
buildActionMask = 2147483647;
|
| 229 |
+
files = (
|
| 230 |
+
55555555555555555555555 /* llama.xcframework in Frameworks */,
|
| 231 |
+
);
|
| 232 |
+
runOnlyForDeploymentPostprocessing = 0;
|
| 233 |
+
};
|
| 234 |
+
/* End PBXFrameworksBuildPhase section */
|
| 235 |
+
|
| 236 |
+
/* Begin PBXGroup section */
|
| 237 |
+
EOF
|
| 238 |
+
|
| 239 |
+
# Continue with the project.pbxproj file, using the APP_NAME variable appropriately
|
| 240 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << EOF
|
| 241 |
+
CCCCCCCCCCCCCCCCCCCCCCCC /* Products */ = {
|
| 242 |
+
isa = PBXGroup;
|
| 243 |
+
children = (
|
| 244 |
+
99999999999999999999999 /* ${APP_NAME}.app */,
|
| 245 |
+
);
|
| 246 |
+
name = Products;
|
| 247 |
+
sourceTree = "<group>";
|
| 248 |
+
};
|
| 249 |
+
EOF
|
| 250 |
+
|
| 251 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << 'EOF'
|
| 252 |
+
DDDDDDDDDDDDDDDDDDDDDDDD /* Frameworks */ = {
|
| 253 |
+
isa = PBXGroup;
|
| 254 |
+
children = (
|
| 255 |
+
66666666666666666666666 /* llama.xcframework */,
|
| 256 |
+
);
|
| 257 |
+
name = Frameworks;
|
| 258 |
+
sourceTree = "<group>";
|
| 259 |
+
};
|
| 260 |
+
EEEEEEEEEEEEEEEEEEEEEEEE = {
|
| 261 |
+
isa = PBXGroup;
|
| 262 |
+
children = (
|
| 263 |
+
FFFFFFFFFFFFFFFFFFFFFFFF /* TVOSLlamaTest */,
|
| 264 |
+
CCCCCCCCCCCCCCCCCCCCCCCC /* Products */,
|
| 265 |
+
DDDDDDDDDDDDDDDDDDDDDDDD /* Frameworks */,
|
| 266 |
+
);
|
| 267 |
+
sourceTree = "<group>";
|
| 268 |
+
};
|
| 269 |
+
FFFFFFFFFFFFFFFFFFFFFFFF /* TVOSLlamaTest */ = {
|
| 270 |
+
isa = PBXGroup;
|
| 271 |
+
children = (
|
| 272 |
+
1111111111111111111111AA /* Sources */,
|
| 273 |
+
AAAAAAAAAAAAAAAAAAAAAAA /* Info.plist */,
|
| 274 |
+
);
|
| 275 |
+
path = "TVOSLlamaTest";
|
| 276 |
+
sourceTree = "<group>";
|
| 277 |
+
};
|
| 278 |
+
1111111111111111111111AA /* Sources */ = {
|
| 279 |
+
isa = PBXGroup;
|
| 280 |
+
children = (
|
| 281 |
+
22222222222222222222222 /* App.swift */,
|
| 282 |
+
44444444444444444444444 /* ContentView.swift */,
|
| 283 |
+
);
|
| 284 |
+
path = Sources;
|
| 285 |
+
sourceTree = "<group>";
|
| 286 |
+
};
|
| 287 |
+
/* End PBXGroup section */
|
| 288 |
+
EOF
|
| 289 |
+
|
| 290 |
+
# Continue with the project.pbxproj file, using the APP_NAME variable appropriately
|
| 291 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << EOF
|
| 292 |
+
/* Begin PBXNativeTarget section */
|
| 293 |
+
3333333333333333333333AA /* ${APP_NAME} */ = {
|
| 294 |
+
isa = PBXNativeTarget;
|
| 295 |
+
buildConfigurationList = 4444444444444444444444AA /* Build configuration list for PBXNativeTarget "${APP_NAME}" */;
|
| 296 |
+
buildPhases = (
|
| 297 |
+
5555555555555555555555AA /* Sources */,
|
| 298 |
+
BBBBBBBBBBBBBBBBBBBBBBBB /* Frameworks */,
|
| 299 |
+
6666666666666666666666AA /* Resources */,
|
| 300 |
+
88888888888888888888888 /* Embed Frameworks */,
|
| 301 |
+
);
|
| 302 |
+
buildRules = (
|
| 303 |
+
);
|
| 304 |
+
dependencies = (
|
| 305 |
+
);
|
| 306 |
+
name = "${APP_NAME}";
|
| 307 |
+
productName = "${APP_NAME}";
|
| 308 |
+
productReference = 99999999999999999999999 /* ${APP_NAME}.app */;
|
| 309 |
+
productType = "com.apple.product-type.application";
|
| 310 |
+
};
|
| 311 |
+
/* End PBXNativeTarget section */
|
| 312 |
+
|
| 313 |
+
/* Begin PBXProject section */
|
| 314 |
+
7777777777777777777777AA /* Project object */ = {
|
| 315 |
+
isa = PBXProject;
|
| 316 |
+
attributes = {
|
| 317 |
+
LastSwiftUpdateCheck = 1240;
|
| 318 |
+
LastUpgradeCheck = 1240;
|
| 319 |
+
TargetAttributes = {
|
| 320 |
+
3333333333333333333333AA = {
|
| 321 |
+
CreatedOnToolsVersion = 12.4;
|
| 322 |
+
};
|
| 323 |
+
};
|
| 324 |
+
};
|
| 325 |
+
buildConfigurationList = 8888888888888888888888AA /* Build configuration list for PBXProject "${APP_NAME}" */;
|
| 326 |
+
compatibilityVersion = "Xcode 12.0";
|
| 327 |
+
developmentRegion = en;
|
| 328 |
+
hasScannedForEncodings = 0;
|
| 329 |
+
knownRegions = (
|
| 330 |
+
en,
|
| 331 |
+
Base,
|
| 332 |
+
);
|
| 333 |
+
mainGroup = EEEEEEEEEEEEEEEEEEEEEEEE;
|
| 334 |
+
productRefGroup = CCCCCCCCCCCCCCCCCCCCCCCC /* Products */;
|
| 335 |
+
projectDirPath = "";
|
| 336 |
+
projectRoot = "";
|
| 337 |
+
targets = (
|
| 338 |
+
3333333333333333333333AA /* ${APP_NAME} */,
|
| 339 |
+
);
|
| 340 |
+
};
|
| 341 |
+
/* End PBXProject section */
|
| 342 |
+
EOF
|
| 343 |
+
|
| 344 |
+
# Add the rest of the file with correct FRAMEWORK_SEARCH_PATHS and tvOS settings
|
| 345 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << 'EOF'
|
| 346 |
+
/* Begin PBXResourcesBuildPhase section */
|
| 347 |
+
6666666666666666666666AA /* Resources */ = {
|
| 348 |
+
isa = PBXResourcesBuildPhase;
|
| 349 |
+
buildActionMask = 2147483647;
|
| 350 |
+
files = (
|
| 351 |
+
);
|
| 352 |
+
runOnlyForDeploymentPostprocessing = 0;
|
| 353 |
+
};
|
| 354 |
+
/* End PBXResourcesBuildPhase section */
|
| 355 |
+
|
| 356 |
+
/* Begin PBXSourcesBuildPhase section */
|
| 357 |
+
5555555555555555555555AA /* Sources */ = {
|
| 358 |
+
isa = PBXSourcesBuildPhase;
|
| 359 |
+
buildActionMask = 2147483647;
|
| 360 |
+
files = (
|
| 361 |
+
33333333333333333333333 /* ContentView.swift in Sources */,
|
| 362 |
+
11111111111111111111111 /* App.swift in Sources */,
|
| 363 |
+
);
|
| 364 |
+
runOnlyForDeploymentPostprocessing = 0;
|
| 365 |
+
};
|
| 366 |
+
/* End PBXSourcesBuildPhase section */
|
| 367 |
+
|
| 368 |
+
/* Begin XCBuildConfiguration section */
|
| 369 |
+
9999999999999999999999AA /* Debug */ = {
|
| 370 |
+
isa = XCBuildConfiguration;
|
| 371 |
+
buildSettings = {
|
| 372 |
+
ALWAYS_SEARCH_USER_PATHS = NO;
|
| 373 |
+
CLANG_ANALYZER_NONNULL = YES;
|
| 374 |
+
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
|
| 375 |
+
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
|
| 376 |
+
CLANG_CXX_LIBRARY = "libc++";
|
| 377 |
+
CLANG_ENABLE_MODULES = YES;
|
| 378 |
+
CLANG_ENABLE_OBJC_ARC = YES;
|
| 379 |
+
CLANG_ENABLE_OBJC_WEAK = YES;
|
| 380 |
+
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
|
| 381 |
+
CLANG_WARN_BOOL_CONVERSION = YES;
|
| 382 |
+
CLANG_WARN_COMMA = YES;
|
| 383 |
+
CLANG_WARN_CONSTANT_CONVERSION = YES;
|
| 384 |
+
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
|
| 385 |
+
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
|
| 386 |
+
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
|
| 387 |
+
CLANG_WARN_EMPTY_BODY = YES;
|
| 388 |
+
CLANG_WARN_ENUM_CONVERSION = YES;
|
| 389 |
+
CLANG_WARN_INFINITE_RECURSION = YES;
|
| 390 |
+
CLANG_WARN_INT_CONVERSION = YES;
|
| 391 |
+
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
|
| 392 |
+
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
|
| 393 |
+
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
|
| 394 |
+
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
|
| 395 |
+
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
|
| 396 |
+
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
|
| 397 |
+
CLANG_WARN_STRICT_PROTOTYPES = YES;
|
| 398 |
+
CLANG_WARN_SUSPICIOUS_MOVE = YES;
|
| 399 |
+
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
|
| 400 |
+
CLANG_WARN_UNREACHABLE_CODE = YES;
|
| 401 |
+
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
|
| 402 |
+
COPY_PHASE_STRIP = NO;
|
| 403 |
+
DEBUG_INFORMATION_FORMAT = dwarf;
|
| 404 |
+
ENABLE_STRICT_OBJC_MSGSEND = YES;
|
| 405 |
+
ENABLE_TESTABILITY = YES;
|
| 406 |
+
GCC_C_LANGUAGE_STANDARD = gnu11;
|
| 407 |
+
GCC_DYNAMIC_NO_PIC = NO;
|
| 408 |
+
GCC_NO_COMMON_BLOCKS = YES;
|
| 409 |
+
GCC_OPTIMIZATION_LEVEL = 0;
|
| 410 |
+
GCC_PREPROCESSOR_DEFINITIONS = (
|
| 411 |
+
"DEBUG=1",
|
| 412 |
+
"$(inherited)",
|
| 413 |
+
);
|
| 414 |
+
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
|
| 415 |
+
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
|
| 416 |
+
GCC_WARN_UNDECLARED_SELECTOR = YES;
|
| 417 |
+
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
|
| 418 |
+
GCC_WARN_UNUSED_FUNCTION = YES;
|
| 419 |
+
GCC_WARN_UNUSED_VARIABLE = YES;
|
| 420 |
+
TVOS_DEPLOYMENT_TARGET = 15.0;
|
| 421 |
+
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
|
| 422 |
+
MTL_FAST_MATH = YES;
|
| 423 |
+
ONLY_ACTIVE_ARCH = YES;
|
| 424 |
+
SDKROOT = appletvos;
|
| 425 |
+
SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
|
| 426 |
+
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
|
| 427 |
+
};
|
| 428 |
+
name = Debug;
|
| 429 |
+
};
|
| 430 |
+
AAAAAAAAAAAAAAAAAAAAABBB /* Release */ = {
|
| 431 |
+
isa = XCBuildConfiguration;
|
| 432 |
+
buildSettings = {
|
| 433 |
+
ALWAYS_SEARCH_USER_PATHS = NO;
|
| 434 |
+
CLANG_ANALYZER_NONNULL = YES;
|
| 435 |
+
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
|
| 436 |
+
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
|
| 437 |
+
CLANG_CXX_LIBRARY = "libc++";
|
| 438 |
+
CLANG_ENABLE_MODULES = YES;
|
| 439 |
+
CLANG_ENABLE_OBJC_ARC = YES;
|
| 440 |
+
CLANG_ENABLE_OBJC_WEAK = YES;
|
| 441 |
+
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
|
| 442 |
+
CLANG_WARN_BOOL_CONVERSION = YES;
|
| 443 |
+
CLANG_WARN_COMMA = YES;
|
| 444 |
+
CLANG_WARN_CONSTANT_CONVERSION = YES;
|
| 445 |
+
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
|
| 446 |
+
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
|
| 447 |
+
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
|
| 448 |
+
CLANG_WARN_EMPTY_BODY = YES;
|
| 449 |
+
CLANG_WARN_ENUM_CONVERSION = YES;
|
| 450 |
+
CLANG_WARN_INFINITE_RECURSION = YES;
|
| 451 |
+
CLANG_WARN_INT_CONVERSION = YES;
|
| 452 |
+
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
|
| 453 |
+
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
|
| 454 |
+
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
|
| 455 |
+
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
|
| 456 |
+
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
|
| 457 |
+
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
|
| 458 |
+
CLANG_WARN_STRICT_PROTOTYPES = YES;
|
| 459 |
+
CLANG_WARN_SUSPICIOUS_MOVE = YES;
|
| 460 |
+
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
|
| 461 |
+
CLANG_WARN_UNREACHABLE_CODE = YES;
|
| 462 |
+
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
|
| 463 |
+
COPY_PHASE_STRIP = NO;
|
| 464 |
+
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
|
| 465 |
+
ENABLE_NS_ASSERTIONS = NO;
|
| 466 |
+
ENABLE_STRICT_OBJC_MSGSEND = YES;
|
| 467 |
+
GCC_C_LANGUAGE_STANDARD = gnu11;
|
| 468 |
+
GCC_NO_COMMON_BLOCKS = YES;
|
| 469 |
+
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
|
| 470 |
+
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
|
| 471 |
+
GCC_WARN_UNDECLARED_SELECTOR = YES;
|
| 472 |
+
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
|
| 473 |
+
GCC_WARN_UNUSED_FUNCTION = YES;
|
| 474 |
+
GCC_WARN_UNUSED_VARIABLE = YES;
|
| 475 |
+
TVOS_DEPLOYMENT_TARGET = 15.0;
|
| 476 |
+
MTL_ENABLE_DEBUG_INFO = NO;
|
| 477 |
+
MTL_FAST_MATH = YES;
|
| 478 |
+
SDKROOT = appletvos;
|
| 479 |
+
SWIFT_COMPILATION_MODE = wholemodule;
|
| 480 |
+
SWIFT_OPTIMIZATION_LEVEL = "-O";
|
| 481 |
+
VALIDATE_PRODUCT = YES;
|
| 482 |
+
};
|
| 483 |
+
name = Release;
|
| 484 |
+
};
|
| 485 |
+
BBBBBBBBBBBBBBBBBBBBBBCCC /* Debug */ = {
|
| 486 |
+
isa = XCBuildConfiguration;
|
| 487 |
+
buildSettings = {
|
| 488 |
+
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
| 489 |
+
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
|
| 490 |
+
CODE_SIGN_STYLE = Manual;
|
| 491 |
+
DEVELOPMENT_TEAM = "";
|
| 492 |
+
ENABLE_PREVIEWS = YES;
|
| 493 |
+
FRAMEWORK_SEARCH_PATHS = "$(PROJECT_DIR)";
|
| 494 |
+
INFOPLIST_FILE = "TVOSLlamaTest/Info.plist";
|
| 495 |
+
LD_RUNPATH_SEARCH_PATHS = (
|
| 496 |
+
"$(inherited)",
|
| 497 |
+
"@executable_path/Frameworks",
|
| 498 |
+
);
|
| 499 |
+
PRODUCT_BUNDLE_IDENTIFIER = "org.ggml.TVOSLlamaTest";
|
| 500 |
+
PRODUCT_NAME = "$(TARGET_NAME)";
|
| 501 |
+
PROVISIONING_PROFILE_SPECIFIER = "";
|
| 502 |
+
SWIFT_VERSION = 5.0;
|
| 503 |
+
TARGETED_DEVICE_FAMILY = 3;
|
| 504 |
+
};
|
| 505 |
+
name = Debug;
|
| 506 |
+
};
|
| 507 |
+
CCCCCCCCCCCCCCCCCCCCCCDDD /* Release */ = {
|
| 508 |
+
isa = XCBuildConfiguration;
|
| 509 |
+
buildSettings = {
|
| 510 |
+
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
| 511 |
+
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
|
| 512 |
+
CODE_SIGN_STYLE = Manual;
|
| 513 |
+
DEVELOPMENT_TEAM = "";
|
| 514 |
+
ENABLE_PREVIEWS = YES;
|
| 515 |
+
FRAMEWORK_SEARCH_PATHS = (
|
| 516 |
+
"$(inherited)",
|
| 517 |
+
"$(PROJECT_DIR)",
|
| 518 |
+
);
|
| 519 |
+
INFOPLIST_FILE = "TVOSLlamaTest/Info.plist";
|
| 520 |
+
LD_RUNPATH_SEARCH_PATHS = (
|
| 521 |
+
"$(inherited)",
|
| 522 |
+
"@executable_path/Frameworks",
|
| 523 |
+
);
|
| 524 |
+
PRODUCT_BUNDLE_IDENTIFIER = "org.ggml.TVOSLlamaTest";
|
| 525 |
+
PRODUCT_NAME = "$(TARGET_NAME)";
|
| 526 |
+
PROVISIONING_PROFILE_SPECIFIER = "";
|
| 527 |
+
SWIFT_VERSION = 5.0;
|
| 528 |
+
TARGETED_DEVICE_FAMILY = 3;
|
| 529 |
+
};
|
| 530 |
+
name = Release;
|
| 531 |
+
};
|
| 532 |
+
/* End XCBuildConfiguration section */
|
| 533 |
+
EOF
|
| 534 |
+
|
| 535 |
+
# Finish the project.pbxproj file
|
| 536 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << EOF
|
| 537 |
+
/* Begin XCConfigurationList section */
|
| 538 |
+
8888888888888888888888AA /* Build configuration list for PBXProject "${APP_NAME}" */ = {
|
| 539 |
+
isa = XCConfigurationList;
|
| 540 |
+
buildConfigurations = (
|
| 541 |
+
9999999999999999999999AA /* Debug */,
|
| 542 |
+
AAAAAAAAAAAAAAAAAAAAABBB /* Release */,
|
| 543 |
+
);
|
| 544 |
+
defaultConfigurationIsVisible = 0;
|
| 545 |
+
defaultConfigurationName = Release;
|
| 546 |
+
};
|
| 547 |
+
4444444444444444444444AA /* Build configuration list for PBXNativeTarget "${APP_NAME}" */ = {
|
| 548 |
+
isa = XCConfigurationList;
|
| 549 |
+
buildConfigurations = (
|
| 550 |
+
BBBBBBBBBBBBBBBBBBBBBBCCC /* Debug */,
|
| 551 |
+
CCCCCCCCCCCCCCCCCCCCCCDDD /* Release */,
|
| 552 |
+
);
|
| 553 |
+
defaultConfigurationIsVisible = 0;
|
| 554 |
+
defaultConfigurationName = Release;
|
| 555 |
+
};
|
| 556 |
+
/* End XCConfigurationList section */
|
| 557 |
+
};
|
| 558 |
+
rootObject = 7777777777777777777777AA /* Project object */;
|
| 559 |
+
}
|
| 560 |
+
EOF
|
| 561 |
+
|
| 562 |
+
# 2. Copy XCFramework to test project
|
| 563 |
+
echo "Copying XCFramework to test project..."
|
| 564 |
+
cp -R "${XCFRAMEWORK_PATH}" "${TEMP_DIR}/${APP_NAME}/"
|
| 565 |
+
|
| 566 |
+
# 3. Build and archive the app
|
| 567 |
+
echo "Building and archiving test app..."
|
| 568 |
+
cd "${TEMP_DIR}/${APP_NAME}"
|
| 569 |
+
|
| 570 |
+
# Create a simple xcscheme file to avoid xcodebuild scheme issues
|
| 571 |
+
mkdir -p "${APP_NAME}.xcodeproj/xcshareddata/xcschemes"
|
| 572 |
+
cat > "${APP_NAME}.xcodeproj/xcshareddata/xcschemes/${APP_NAME}.xcscheme" << EOF
|
| 573 |
+
<?xml version="1.0" encoding="UTF-8"?>
|
| 574 |
+
<Scheme
|
| 575 |
+
LastUpgradeVersion = "1240"
|
| 576 |
+
version = "1.3">
|
| 577 |
+
<BuildAction
|
| 578 |
+
parallelizeBuildables = "YES"
|
| 579 |
+
buildImplicitDependencies = "YES">
|
| 580 |
+
<BuildActionEntries>
|
| 581 |
+
<BuildActionEntry
|
| 582 |
+
buildForTesting = "YES"
|
| 583 |
+
buildForRunning = "YES"
|
| 584 |
+
buildForProfiling = "YES"
|
| 585 |
+
buildForArchiving = "YES"
|
| 586 |
+
buildForAnalyzing = "YES">
|
| 587 |
+
<BuildableReference
|
| 588 |
+
BuildableIdentifier = "primary"
|
| 589 |
+
BlueprintIdentifier = "3333333333333333333333AA"
|
| 590 |
+
BuildableName = "${APP_NAME}.app"
|
| 591 |
+
BlueprintName = "${APP_NAME}"
|
| 592 |
+
ReferencedContainer = "container:${APP_NAME}.xcodeproj">
|
| 593 |
+
</BuildableReference>
|
| 594 |
+
</BuildActionEntry>
|
| 595 |
+
</BuildActionEntries>
|
| 596 |
+
</BuildAction>
|
| 597 |
+
<TestAction
|
| 598 |
+
buildConfiguration = "Debug"
|
| 599 |
+
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
|
| 600 |
+
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
|
| 601 |
+
shouldUseLaunchSchemeArgsEnv = "YES">
|
| 602 |
+
<Testables>
|
| 603 |
+
</Testables>
|
| 604 |
+
</TestAction>
|
| 605 |
+
<LaunchAction
|
| 606 |
+
buildConfiguration = "Debug"
|
| 607 |
+
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
|
| 608 |
+
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
|
| 609 |
+
launchStyle = "0"
|
| 610 |
+
useCustomWorkingDirectory = "NO"
|
| 611 |
+
ignoresPersistentStateOnLaunch = "NO"
|
| 612 |
+
debugDocumentVersioning = "YES"
|
| 613 |
+
debugServiceExtension = "internal"
|
| 614 |
+
allowLocationSimulation = "YES">
|
| 615 |
+
<BuildableProductRunnable
|
| 616 |
+
runnableDebuggingMode = "0">
|
| 617 |
+
<BuildableReference
|
| 618 |
+
BuildableIdentifier = "primary"
|
| 619 |
+
BlueprintIdentifier = "3333333333333333333333AA"
|
| 620 |
+
BuildableName = "${APP_NAME}.app"
|
| 621 |
+
BlueprintName = "${APP_NAME}"
|
| 622 |
+
ReferencedContainer = "container:${APP_NAME}.xcodeproj">
|
| 623 |
+
</BuildableReference>
|
| 624 |
+
</BuildableProductRunnable>
|
| 625 |
+
</LaunchAction>
|
| 626 |
+
<ProfileAction
|
| 627 |
+
buildConfiguration = "Release"
|
| 628 |
+
shouldUseLaunchSchemeArgsEnv = "YES"
|
| 629 |
+
savedToolIdentifier = ""
|
| 630 |
+
useCustomWorkingDirectory = "NO"
|
| 631 |
+
debugDocumentVersioning = "YES">
|
| 632 |
+
<BuildableProductRunnable
|
| 633 |
+
runnableDebuggingMode = "0">
|
| 634 |
+
<BuildableReference
|
| 635 |
+
BuildableIdentifier = "primary"
|
| 636 |
+
BlueprintIdentifier = "3333333333333333333333AA"
|
| 637 |
+
BuildableName = "${APP_NAME}.app"
|
| 638 |
+
BlueprintName = "${APP_NAME}"
|
| 639 |
+
ReferencedContainer = "container:${APP_NAME}.xcodeproj">
|
| 640 |
+
</BuildableReference>
|
| 641 |
+
</BuildableProductRunnable>
|
| 642 |
+
</ProfileAction>
|
| 643 |
+
<AnalyzeAction
|
| 644 |
+
buildConfiguration = "Debug">
|
| 645 |
+
</AnalyzeAction>
|
| 646 |
+
<ArchiveAction
|
| 647 |
+
buildConfiguration = "Release"
|
| 648 |
+
revealArchiveInOrganizer = "YES">
|
| 649 |
+
</ArchiveAction>
|
| 650 |
+
</Scheme>
|
| 651 |
+
EOF
|
| 652 |
+
|
| 653 |
+
# Now use xcodebuild with an explicitly defined product name for tvOS
|
| 654 |
+
xcodebuild -project "${APP_NAME}.xcodeproj" -scheme "${APP_NAME}" -sdk appletvos -configuration Release archive -archivePath "${ARCHIVE_PATH}" CODE_SIGN_IDENTITY="-" CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO PRODUCT_NAME="${APP_NAME}" SWIFT_OPTIMIZATION_LEVEL="-Onone" -quiet
|
| 655 |
+
|
| 656 |
+
# 4. Create IPA from archive
|
| 657 |
+
echo "Creating IPA from archive..."
|
| 658 |
+
mkdir -p "${TEMP_DIR}/Payload"
|
| 659 |
+
cp -R "${ARCHIVE_PATH}/Products/Applications/${APP_NAME}.app" "${TEMP_DIR}/Payload/"
|
| 660 |
+
|
| 661 |
+
# Check and log app structure before zipping
|
| 662 |
+
echo "App structure:"
|
| 663 |
+
ls -la "${TEMP_DIR}/Payload/${APP_NAME}.app/"
|
| 664 |
+
echo "Frameworks:"
|
| 665 |
+
ls -la "${TEMP_DIR}/Payload/${APP_NAME}.app/Frameworks/" 2>/dev/null || echo "No Frameworks directory found"
|
| 666 |
+
|
| 667 |
+
cd "${TEMP_DIR}"
|
| 668 |
+
zip -r "${IPA_PATH}" Payload
|
| 669 |
+
|
| 670 |
+
# Check embedded provisioning profile
|
| 671 |
+
echo "Checking provisioning profile (if any)..."
|
| 672 |
+
PROVISIONING_PROFILE=$(find "${ARCHIVE_PATH}/Products/Applications/${APP_NAME}.app" -name "embedded.mobileprovision" 2>/dev/null)
|
| 673 |
+
if [ -n "$PROVISIONING_PROFILE" ]; then
|
| 674 |
+
echo "Found embedded provisioning profile:"
|
| 675 |
+
security cms -D -i "$PROVISIONING_PROFILE" || echo "Unable to decode provisioning profile"
|
| 676 |
+
else
|
| 677 |
+
echo "No embedded provisioning profile found (expected for ad-hoc builds)"
|
| 678 |
+
fi
|
| 679 |
+
|
| 680 |
+
# 5. Validate the IPA
|
| 681 |
+
echo "Validating IPA..."
|
| 682 |
+
VALIDATION_OUTPUT="${VALIDATION_DIR}/validation_output.txt"
|
| 683 |
+
|
| 684 |
+
# Check if authentication credentials are provided
|
| 685 |
+
AUTH_ARGS=""
|
| 686 |
+
if [ -n "$APPLE_ID" ] && [ -n "$APPLE_PASSWORD" ]; then
|
| 687 |
+
echo "Using Apple ID authentication for validation..."
|
| 688 |
+
AUTH_ARGS="--username \"$APPLE_ID\" --password \"$APPLE_PASSWORD\""
|
| 689 |
+
else
|
| 690 |
+
echo "No authentication credentials provided. Will perform basic validation."
|
| 691 |
+
echo "To use your personal developer account, you can run the script with:"
|
| 692 |
+
echo " APPLE_ID='your.email@example.com' APPLE_PASSWORD='your-app-specific-password' ./validate-tvos.sh"
|
| 693 |
+
echo "Note: You need to create an app-specific password at https://appleid.apple.com/account/manage"
|
| 694 |
+
fi
|
| 695 |
+
|
| 696 |
+
# Run validation with detailed output
|
| 697 |
+
echo "Running validation with altool..."
|
| 698 |
+
if [ -n "$AUTH_ARGS" ]; then
|
| 699 |
+
# Use eval to properly handle the quoted arguments
|
| 700 |
+
eval "xcrun altool --validate-app -f \"${IPA_PATH}\" --type tvos --output-format xml $AUTH_ARGS" 2>&1 | tee "${VALIDATION_OUTPUT}"
|
| 701 |
+
else
|
| 702 |
+
xcrun altool --validate-app -f "${IPA_PATH}" --type tvos --output-format xml 2>&1 | tee "${VALIDATION_OUTPUT}"
|
| 703 |
+
fi
|
| 704 |
+
VALIDATION_RESULT=$?
|
| 705 |
+
|
| 706 |
+
# Final validation result
|
| 707 |
+
FINAL_VALIDATION_RESULT=0
|
| 708 |
+
|
| 709 |
+
# Check if validation failed because the app isn't in App Store Connect
|
| 710 |
+
if grep -q "No suitable application records were found" "${VALIDATION_OUTPUT}"; then
|
| 711 |
+
echo "⚠️ App Store Connect Warning: The app bundle identifier is not found in App Store Connect"
|
| 712 |
+
echo "This is expected for apps that haven't been registered in App Store Connect yet."
|
| 713 |
+
echo "This doesn't indicate a problem with the build or framework."
|
| 714 |
+
|
| 715 |
+
# Perform alternative validation
|
| 716 |
+
echo "Performing alternative validation checks..."
|
| 717 |
+
|
| 718 |
+
# Check if IPA was created successfully
|
| 719 |
+
if [ -f "${IPA_PATH}" ] && [ -s "${IPA_PATH}" ]; then
|
| 720 |
+
echo "✅ IPA file created successfully"
|
| 721 |
+
else
|
| 722 |
+
echo "❌ IPA file not created or empty"
|
| 723 |
+
FINAL_VALIDATION_RESULT=1
|
| 724 |
+
fi
|
| 725 |
+
|
| 726 |
+
# Check if app binary exists and is executable
|
| 727 |
+
if [ -f "${TEMP_DIR}/Payload/${APP_NAME}.app/${APP_NAME}" ] && [ -x "${TEMP_DIR}/Payload/${APP_NAME}.app/${APP_NAME}" ]; then
|
| 728 |
+
echo "✅ App binary exists and is executable"
|
| 729 |
+
else
|
| 730 |
+
echo "❌ App binary missing or not executable"
|
| 731 |
+
FINAL_VALIDATION_RESULT=1
|
| 732 |
+
fi
|
| 733 |
+
|
| 734 |
+
# Check if framework was properly embedded
|
| 735 |
+
if [ -d "${TEMP_DIR}/Payload/${APP_NAME}.app/Frameworks/llama.framework" ]; then
|
| 736 |
+
echo "✅ llama.framework properly embedded"
|
| 737 |
+
else
|
| 738 |
+
echo "❌ llama.framework not properly embedded"
|
| 739 |
+
FINAL_VALIDATION_RESULT=1
|
| 740 |
+
fi
|
| 741 |
+
|
| 742 |
+
# Check if framework binary exists
|
| 743 |
+
if [ -f "${TEMP_DIR}/Payload/${APP_NAME}.app/Frameworks/llama.framework/llama" ]; then
|
| 744 |
+
echo "✅ Framework binary exists"
|
| 745 |
+
|
| 746 |
+
# Further validate framework by checking architecture
|
| 747 |
+
ARCHS=$(lipo -info "${TEMP_DIR}/Payload/${APP_NAME}.app/Frameworks/llama.framework/llama" 2>/dev/null | grep -o "arm64\\|x86_64" | tr '\n' ' ')
|
| 748 |
+
if [ -n "$ARCHS" ]; then
|
| 749 |
+
echo "✅ Framework architecture(s): $ARCHS"
|
| 750 |
+
else
|
| 751 |
+
echo "⚠️ Could not determine framework architecture"
|
| 752 |
+
fi
|
| 753 |
+
else
|
| 754 |
+
echo "❌ Framework binary missing"
|
| 755 |
+
FINAL_VALIDATION_RESULT=1
|
| 756 |
+
fi
|
| 757 |
+
|
| 758 |
+
if [ $FINAL_VALIDATION_RESULT -eq 0 ]; then
|
| 759 |
+
echo "✅ Alternative validation PASSED: App built successfully with embedded framework"
|
| 760 |
+
else
|
| 761 |
+
echo "❌ Alternative validation FAILED: Issues found with the app or framework"
|
| 762 |
+
fi
|
| 763 |
+
elif grep -q "You must specify authentication credentials" "${VALIDATION_OUTPUT}" && [ -z "$AUTH_ARGS" ]; then
|
| 764 |
+
echo "✅ tvOS Validation PASSED: IPA successfully validated"
|
| 765 |
+
echo "Results saved to ${VALIDATION_OUTPUT}"
|
| 766 |
+
else
|
| 767 |
+
echo "❌ tvOS Validation FAILED: IPA validation found issues"
|
| 768 |
+
echo "See validation output at ${VALIDATION_OUTPUT}"
|
| 769 |
+
echo ""
|
| 770 |
+
echo "==== VALIDATION ERRORS ===="
|
| 771 |
+
|
| 772 |
+
# Try to extract specific errors from the output
|
| 773 |
+
if grep -q "Error" "${VALIDATION_OUTPUT}"; then
|
| 774 |
+
grep -A 5 "Error" "${VALIDATION_OUTPUT}"
|
| 775 |
+
else
|
| 776 |
+
# If no specific error found, show the whole log
|
| 777 |
+
cat "${VALIDATION_OUTPUT}"
|
| 778 |
+
fi
|
| 779 |
+
|
| 780 |
+
# Additional debugging: check IPA contents
|
| 781 |
+
echo ""
|
| 782 |
+
echo "==== IPA CONTENTS ===="
|
| 783 |
+
mkdir -p "${TEMP_DIR}/ipa_contents"
|
| 784 |
+
unzip -q "${IPA_PATH}" -d "${TEMP_DIR}/ipa_contents"
|
| 785 |
+
ls -la "${TEMP_DIR}/ipa_contents/Payload/${APP_NAME}.app/"
|
| 786 |
+
|
| 787 |
+
# Check for code signing issues
|
| 788 |
+
echo ""
|
| 789 |
+
echo "==== CODE SIGNING INFO ===="
|
| 790 |
+
codesign -vv -d "${TEMP_DIR}/ipa_contents/Payload/${APP_NAME}.app" 2>&1 || echo "Code signing verification failed"
|
| 791 |
+
|
| 792 |
+
# Check embedded frameworks
|
| 793 |
+
echo ""
|
| 794 |
+
echo "==== FRAMEWORK INFO ===="
|
| 795 |
+
ls -la "${TEMP_DIR}/ipa_contents/Payload/${APP_NAME}.app/Frameworks/" 2>/dev/null || echo "No Frameworks directory found"
|
| 796 |
+
fi
|
| 797 |
+
|
| 798 |
+
# Don't clean up on error to allow inspection
|
| 799 |
+
if [ $FINAL_VALIDATION_RESULT -ne 0 ]; then
|
| 800 |
+
echo ""
|
| 801 |
+
echo "Temporary files kept for inspection at: ${TEMP_DIR}"
|
| 802 |
+
echo "===== tvOS Validation Process Failed ====="
|
| 803 |
+
exit 1
|
| 804 |
+
fi
|
| 805 |
+
|
| 806 |
+
# Clean up temporary files but keep build artifacts
|
| 807 |
+
if [ $FINAL_VALIDATION_RESULT -eq 0 ]; then
|
| 808 |
+
echo "Cleaning up temporary files..."
|
| 809 |
+
#rm -rf "${TEMP_DIR}"
|
| 810 |
+
fi
|
| 811 |
+
|
| 812 |
+
echo "===== tvOS Validation Process Completed ====="
|
| 813 |
+
exit $FINAL_VALIDATION_RESULT
|
backend/llama.cpp/scripts/apple/validate-visionos.sh
ADDED
|
@@ -0,0 +1,811 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# validate-visionos.sh - Validate visionOS Application with embedded llama.xcframework using SwiftUI
|
| 3 |
+
|
| 4 |
+
# Authentication options (optional) (can be set via environment variables)
|
| 5 |
+
# To use: export APPLE_ID=your.email@example.com
|
| 6 |
+
# export APPLE_PASSWORD=your-app-specific-password
|
| 7 |
+
# ./validate-visionos.sh
|
| 8 |
+
APPLE_ID=${APPLE_ID:-""}
|
| 9 |
+
APPLE_PASSWORD=${APPLE_PASSWORD:-""}
|
| 10 |
+
|
| 11 |
+
# Ensure the script exits on error
|
| 12 |
+
set -e
|
| 13 |
+
|
| 14 |
+
# Function to print usage instructions
|
| 15 |
+
print_usage() {
|
| 16 |
+
echo "Usage: ./validate-visionos.sh [OPTIONS]"
|
| 17 |
+
echo ""
|
| 18 |
+
echo "Options:"
|
| 19 |
+
echo " --help Show this help message"
|
| 20 |
+
echo " --apple-id EMAIL Apple ID email for validation"
|
| 21 |
+
echo " --apple-password PWD App-specific password for Apple ID"
|
| 22 |
+
echo ""
|
| 23 |
+
echo "Environment variables:"
|
| 24 |
+
echo " APPLE_ID Apple ID email for validation"
|
| 25 |
+
echo " APPLE_PASSWORD App-specific password for Apple ID"
|
| 26 |
+
echo ""
|
| 27 |
+
echo "Notes:"
|
| 28 |
+
echo " - Command line options take precedence over environment variables"
|
| 29 |
+
echo " - Authentication is optional. If not provided, alternative validation will be performed"
|
| 30 |
+
echo " - For APPLE_PASSWORD, use an app-specific password generated at https://appleid.apple.com/account/manage"
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
# Parse command line arguments
|
| 34 |
+
while [[ $# -gt 0 ]]; do
|
| 35 |
+
case $1 in
|
| 36 |
+
--help)
|
| 37 |
+
print_usage
|
| 38 |
+
exit 0
|
| 39 |
+
;;
|
| 40 |
+
--apple-id)
|
| 41 |
+
APPLE_ID="$2"
|
| 42 |
+
shift 2
|
| 43 |
+
;;
|
| 44 |
+
--apple-password)
|
| 45 |
+
APPLE_PASSWORD="$2"
|
| 46 |
+
shift 2
|
| 47 |
+
;;
|
| 48 |
+
*)
|
| 49 |
+
echo "Unknown option: $1"
|
| 50 |
+
print_usage
|
| 51 |
+
exit 1
|
| 52 |
+
;;
|
| 53 |
+
esac
|
| 54 |
+
done
|
| 55 |
+
|
| 56 |
+
# Function to clean up in case of error
|
| 57 |
+
cleanup() {
|
| 58 |
+
# Don't clean up temp files on error to help with debugging
|
| 59 |
+
echo "===== visionOS Validation Process Failed ====="
|
| 60 |
+
exit 1
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
# Set up trap to call cleanup function on error
|
| 64 |
+
trap cleanup ERR
|
| 65 |
+
|
| 66 |
+
set -e # Exit on any error
|
| 67 |
+
|
| 68 |
+
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )"
|
| 69 |
+
BUILD_DIR="${ROOT_DIR}/validation-builds/visionos"
|
| 70 |
+
|
| 71 |
+
# Configuration
|
| 72 |
+
APP_NAME="VisionOSLlamaTest"
|
| 73 |
+
BUNDLE_ID="org.ggml.VisionOSLlamaTest"
|
| 74 |
+
XCFRAMEWORK_PATH="${ROOT_DIR}/build-apple/llama.xcframework"
|
| 75 |
+
TEMP_DIR="${BUILD_DIR}/temp"
|
| 76 |
+
ARCHIVE_PATH="${BUILD_DIR}/${APP_NAME}.xcarchive"
|
| 77 |
+
IPA_PATH="${BUILD_DIR}/${APP_NAME}.ipa"
|
| 78 |
+
VALIDATION_DIR="${BUILD_DIR}/validation"
|
| 79 |
+
|
| 80 |
+
# Create necessary directories
|
| 81 |
+
mkdir -p "${BUILD_DIR}"
|
| 82 |
+
mkdir -p "${TEMP_DIR}"
|
| 83 |
+
mkdir -p "${VALIDATION_DIR}"
|
| 84 |
+
|
| 85 |
+
echo "===== visionOS Validation Process Started ====="
|
| 86 |
+
|
| 87 |
+
# 1. Create a simple test app project
|
| 88 |
+
echo "Creating test visionOS app project..."
|
| 89 |
+
mkdir -p "${TEMP_DIR}/${APP_NAME}/${APP_NAME}"
|
| 90 |
+
cat > "${TEMP_DIR}/${APP_NAME}/${APP_NAME}/Info.plist" << EOF
|
| 91 |
+
<?xml version="1.0" encoding="UTF-8"?>
|
| 92 |
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
| 93 |
+
<plist version="1.0">
|
| 94 |
+
<dict>
|
| 95 |
+
<key>CFBundleDevelopmentRegion</key>
|
| 96 |
+
<string>en</string>
|
| 97 |
+
<key>CFBundleExecutable</key>
|
| 98 |
+
<string>${APP_NAME}</string>
|
| 99 |
+
<key>CFBundleIdentifier</key>
|
| 100 |
+
<string>${BUNDLE_ID}</string>
|
| 101 |
+
<key>CFBundleInfoDictionaryVersion</key>
|
| 102 |
+
<string>6.0</string>
|
| 103 |
+
<key>CFBundleName</key>
|
| 104 |
+
<string>${APP_NAME}</string>
|
| 105 |
+
<key>CFBundlePackageType</key>
|
| 106 |
+
<string>APPL</string>
|
| 107 |
+
<key>CFBundleShortVersionString</key>
|
| 108 |
+
<string>1.0</string>
|
| 109 |
+
<key>CFBundleVersion</key>
|
| 110 |
+
<string>1</string>
|
| 111 |
+
</dict>
|
| 112 |
+
</plist>
|
| 113 |
+
EOF
|
| 114 |
+
|
| 115 |
+
# Create SwiftUI app files
|
| 116 |
+
mkdir -p "${TEMP_DIR}/${APP_NAME}/${APP_NAME}/Sources"
|
| 117 |
+
|
| 118 |
+
# Create App.swift
|
| 119 |
+
cat > "${TEMP_DIR}/${APP_NAME}/${APP_NAME}/Sources/App.swift" << EOF
|
| 120 |
+
import SwiftUI
|
| 121 |
+
import llama
|
| 122 |
+
|
| 123 |
+
@main
|
| 124 |
+
struct LlamaTestApp: App {
|
| 125 |
+
var body: some Scene {
|
| 126 |
+
WindowGroup {
|
| 127 |
+
ContentView()
|
| 128 |
+
}
|
| 129 |
+
}
|
| 130 |
+
}
|
| 131 |
+
EOF
|
| 132 |
+
|
| 133 |
+
# Create ContentView.swift with visionOS specific elements
|
| 134 |
+
cat > "${TEMP_DIR}/${APP_NAME}/${APP_NAME}/Sources/ContentView.swift" << EOF
|
| 135 |
+
import SwiftUI
|
| 136 |
+
import llama
|
| 137 |
+
|
| 138 |
+
struct ContentView: View {
|
| 139 |
+
// Test that we can initialize a llama context params struct
|
| 140 |
+
let params = llama_context_default_params()
|
| 141 |
+
|
| 142 |
+
var body: some View {
|
| 143 |
+
VStack(spacing: 20) {
|
| 144 |
+
Text("Llama Framework Test on visionOS")
|
| 145 |
+
.font(.largeTitle)
|
| 146 |
+
.padding()
|
| 147 |
+
|
| 148 |
+
Text("llama_context_default_params() created successfully")
|
| 149 |
+
.font(.headline)
|
| 150 |
+
.multilineTextAlignment(.center)
|
| 151 |
+
.padding()
|
| 152 |
+
|
| 153 |
+
// Display some param values to confirm the framework is working
|
| 154 |
+
Text("n_ctx: \(params.n_ctx)")
|
| 155 |
+
.font(.body)
|
| 156 |
+
|
| 157 |
+
Text("n_batch: \(params.n_batch)")
|
| 158 |
+
.font(.body)
|
| 159 |
+
|
| 160 |
+
Spacer()
|
| 161 |
+
}
|
| 162 |
+
.padding()
|
| 163 |
+
.frame(width: 500, height: 400)
|
| 164 |
+
}
|
| 165 |
+
}
|
| 166 |
+
|
| 167 |
+
struct ContentView_Previews: PreviewProvider {
|
| 168 |
+
static var previews: some View {
|
| 169 |
+
ContentView()
|
| 170 |
+
}
|
| 171 |
+
}
|
| 172 |
+
EOF
|
| 173 |
+
|
| 174 |
+
# Create project.pbxproj, fixing the framework search paths issues
|
| 175 |
+
mkdir -p "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj"
|
| 176 |
+
cat > "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << 'EOF'
|
| 177 |
+
// !$*UTF8*$!
|
| 178 |
+
{
|
| 179 |
+
archiveVersion = 1;
|
| 180 |
+
classes = {
|
| 181 |
+
};
|
| 182 |
+
objectVersion = 54;
|
| 183 |
+
objects = {
|
| 184 |
+
|
| 185 |
+
/* Begin PBXBuildFile section */
|
| 186 |
+
11111111111111111111111 /* App.swift in Sources */ = {isa = PBXBuildFile; fileRef = 22222222222222222222222; };
|
| 187 |
+
33333333333333333333333 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 44444444444444444444444; };
|
| 188 |
+
55555555555555555555555 /* llama.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 66666666666666666666666; };
|
| 189 |
+
77777777777777777777777 /* llama.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 66666666666666666666666; };
|
| 190 |
+
/* End PBXBuildFile section */
|
| 191 |
+
|
| 192 |
+
/* Begin PBXCopyFilesBuildPhase section */
|
| 193 |
+
88888888888888888888888 /* Embed Frameworks */ = {
|
| 194 |
+
isa = PBXCopyFilesBuildPhase;
|
| 195 |
+
buildActionMask = 2147483647;
|
| 196 |
+
dstPath = "";
|
| 197 |
+
dstSubfolderSpec = 10;
|
| 198 |
+
files = (
|
| 199 |
+
77777777777777777777777 /* llama.xcframework in Embed Frameworks */,
|
| 200 |
+
);
|
| 201 |
+
name = "Embed Frameworks";
|
| 202 |
+
runOnlyForDeploymentPostprocessing = 0;
|
| 203 |
+
};
|
| 204 |
+
/* End PBXCopyFilesBuildPhase section */
|
| 205 |
+
|
| 206 |
+
/* Begin PBXFileReference section */
|
| 207 |
+
EOF
|
| 208 |
+
|
| 209 |
+
# Continue with the project.pbxproj file, using the APP_NAME variable appropriately
|
| 210 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << EOF
|
| 211 |
+
99999999999999999999999 /* ${APP_NAME}.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "${APP_NAME}.app"; sourceTree = BUILT_PRODUCTS_DIR; };
|
| 212 |
+
22222222222222222222222 /* App.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = App.swift; sourceTree = "<group>"; };
|
| 213 |
+
44444444444444444444444 /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = "<group>"; };
|
| 214 |
+
AAAAAAAAAAAAAAAAAAAAAAA /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
|
| 215 |
+
66666666666666666666666 /* llama.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = llama.xcframework; sourceTree = "<group>"; };
|
| 216 |
+
/* End PBXFileReference section */
|
| 217 |
+
EOF
|
| 218 |
+
|
| 219 |
+
# Add the rest of the project file with fixed framework search paths
|
| 220 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << 'EOF'
|
| 221 |
+
/* Begin PBXFrameworksBuildPhase section */
|
| 222 |
+
BBBBBBBBBBBBBBBBBBBBBBBB /* Frameworks */ = {
|
| 223 |
+
isa = PBXFrameworksBuildPhase;
|
| 224 |
+
buildActionMask = 2147483647;
|
| 225 |
+
files = (
|
| 226 |
+
55555555555555555555555 /* llama.xcframework in Frameworks */,
|
| 227 |
+
);
|
| 228 |
+
runOnlyForDeploymentPostprocessing = 0;
|
| 229 |
+
};
|
| 230 |
+
/* End PBXFrameworksBuildPhase section */
|
| 231 |
+
|
| 232 |
+
/* Begin PBXGroup section */
|
| 233 |
+
EOF
|
| 234 |
+
|
| 235 |
+
# Continue with the project.pbxproj file, using the APP_NAME variable appropriately
|
| 236 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << EOF
|
| 237 |
+
CCCCCCCCCCCCCCCCCCCCCCCC /* Products */ = {
|
| 238 |
+
isa = PBXGroup;
|
| 239 |
+
children = (
|
| 240 |
+
99999999999999999999999 /* ${APP_NAME}.app */,
|
| 241 |
+
);
|
| 242 |
+
name = Products;
|
| 243 |
+
sourceTree = "<group>";
|
| 244 |
+
};
|
| 245 |
+
EOF
|
| 246 |
+
|
| 247 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << 'EOF'
|
| 248 |
+
DDDDDDDDDDDDDDDDDDDDDDDD /* Frameworks */ = {
|
| 249 |
+
isa = PBXGroup;
|
| 250 |
+
children = (
|
| 251 |
+
66666666666666666666666 /* llama.xcframework */,
|
| 252 |
+
);
|
| 253 |
+
name = Frameworks;
|
| 254 |
+
sourceTree = "<group>";
|
| 255 |
+
};
|
| 256 |
+
EEEEEEEEEEEEEEEEEEEEEEEE = {
|
| 257 |
+
isa = PBXGroup;
|
| 258 |
+
children = (
|
| 259 |
+
FFFFFFFFFFFFFFFFFFFFFFFF /* VisionOSLlamaTest */,
|
| 260 |
+
CCCCCCCCCCCCCCCCCCCCCCCC /* Products */,
|
| 261 |
+
DDDDDDDDDDDDDDDDDDDDDDDD /* Frameworks */,
|
| 262 |
+
);
|
| 263 |
+
sourceTree = "<group>";
|
| 264 |
+
};
|
| 265 |
+
FFFFFFFFFFFFFFFFFFFFFFFF /* VisionOSLlamaTest */ = {
|
| 266 |
+
isa = PBXGroup;
|
| 267 |
+
children = (
|
| 268 |
+
1111111111111111111111AA /* Sources */,
|
| 269 |
+
AAAAAAAAAAAAAAAAAAAAAAA /* Info.plist */,
|
| 270 |
+
);
|
| 271 |
+
path = "VisionOSLlamaTest";
|
| 272 |
+
sourceTree = "<group>";
|
| 273 |
+
};
|
| 274 |
+
1111111111111111111111AA /* Sources */ = {
|
| 275 |
+
isa = PBXGroup;
|
| 276 |
+
children = (
|
| 277 |
+
22222222222222222222222 /* App.swift */,
|
| 278 |
+
44444444444444444444444 /* ContentView.swift */,
|
| 279 |
+
);
|
| 280 |
+
path = Sources;
|
| 281 |
+
sourceTree = "<group>";
|
| 282 |
+
};
|
| 283 |
+
/* End PBXGroup section */
|
| 284 |
+
EOF
|
| 285 |
+
|
| 286 |
+
# Continue with the project.pbxproj file, using the APP_NAME variable appropriately
|
| 287 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << EOF
|
| 288 |
+
/* Begin PBXNativeTarget section */
|
| 289 |
+
3333333333333333333333AA /* ${APP_NAME} */ = {
|
| 290 |
+
isa = PBXNativeTarget;
|
| 291 |
+
buildConfigurationList = 4444444444444444444444AA /* Build configuration list for PBXNativeTarget "${APP_NAME}" */;
|
| 292 |
+
buildPhases = (
|
| 293 |
+
5555555555555555555555AA /* Sources */,
|
| 294 |
+
BBBBBBBBBBBBBBBBBBBBBBBB /* Frameworks */,
|
| 295 |
+
6666666666666666666666AA /* Resources */,
|
| 296 |
+
88888888888888888888888 /* Embed Frameworks */,
|
| 297 |
+
);
|
| 298 |
+
buildRules = (
|
| 299 |
+
);
|
| 300 |
+
dependencies = (
|
| 301 |
+
);
|
| 302 |
+
name = "${APP_NAME}";
|
| 303 |
+
productName = "${APP_NAME}";
|
| 304 |
+
productReference = 99999999999999999999999 /* ${APP_NAME}.app */;
|
| 305 |
+
productType = "com.apple.product-type.application";
|
| 306 |
+
};
|
| 307 |
+
/* End PBXNativeTarget section */
|
| 308 |
+
|
| 309 |
+
/* Begin PBXProject section */
|
| 310 |
+
7777777777777777777777AA /* Project object */ = {
|
| 311 |
+
isa = PBXProject;
|
| 312 |
+
attributes = {
|
| 313 |
+
LastSwiftUpdateCheck = 1510;
|
| 314 |
+
LastUpgradeCheck = 1510;
|
| 315 |
+
TargetAttributes = {
|
| 316 |
+
3333333333333333333333AA = {
|
| 317 |
+
CreatedOnToolsVersion = 15.1;
|
| 318 |
+
};
|
| 319 |
+
};
|
| 320 |
+
};
|
| 321 |
+
buildConfigurationList = 8888888888888888888888AA /* Build configuration list for PBXProject "${APP_NAME}" */;
|
| 322 |
+
compatibilityVersion = "Xcode 15.0";
|
| 323 |
+
developmentRegion = en;
|
| 324 |
+
hasScannedForEncodings = 0;
|
| 325 |
+
knownRegions = (
|
| 326 |
+
en,
|
| 327 |
+
Base,
|
| 328 |
+
);
|
| 329 |
+
mainGroup = EEEEEEEEEEEEEEEEEEEEEEEE;
|
| 330 |
+
productRefGroup = CCCCCCCCCCCCCCCCCCCCCCCC /* Products */;
|
| 331 |
+
projectDirPath = "";
|
| 332 |
+
projectRoot = "";
|
| 333 |
+
targets = (
|
| 334 |
+
3333333333333333333333AA /* ${APP_NAME} */,
|
| 335 |
+
);
|
| 336 |
+
};
|
| 337 |
+
/* End PBXProject section */
|
| 338 |
+
EOF
|
| 339 |
+
|
| 340 |
+
# Add the rest of the file with correct FRAMEWORK_SEARCH_PATHS
|
| 341 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << 'EOF'
|
| 342 |
+
/* Begin PBXResourcesBuildPhase section */
|
| 343 |
+
6666666666666666666666AA /* Resources */ = {
|
| 344 |
+
isa = PBXResourcesBuildPhase;
|
| 345 |
+
buildActionMask = 2147483647;
|
| 346 |
+
files = (
|
| 347 |
+
);
|
| 348 |
+
runOnlyForDeploymentPostprocessing = 0;
|
| 349 |
+
};
|
| 350 |
+
/* End PBXResourcesBuildPhase section */
|
| 351 |
+
|
| 352 |
+
/* Begin PBXSourcesBuildPhase section */
|
| 353 |
+
5555555555555555555555AA /* Sources */ = {
|
| 354 |
+
isa = PBXSourcesBuildPhase;
|
| 355 |
+
buildActionMask = 2147483647;
|
| 356 |
+
files = (
|
| 357 |
+
33333333333333333333333 /* ContentView.swift in Sources */,
|
| 358 |
+
11111111111111111111111 /* App.swift in Sources */,
|
| 359 |
+
);
|
| 360 |
+
runOnlyForDeploymentPostprocessing = 0;
|
| 361 |
+
};
|
| 362 |
+
/* End PBXSourcesBuildPhase section */
|
| 363 |
+
|
| 364 |
+
/* Begin XCBuildConfiguration section */
|
| 365 |
+
9999999999999999999999AA /* Debug */ = {
|
| 366 |
+
isa = XCBuildConfiguration;
|
| 367 |
+
buildSettings = {
|
| 368 |
+
ALWAYS_SEARCH_USER_PATHS = NO;
|
| 369 |
+
CLANG_ANALYZER_NONNULL = YES;
|
| 370 |
+
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
|
| 371 |
+
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
|
| 372 |
+
CLANG_CXX_LIBRARY = "libc++";
|
| 373 |
+
CLANG_ENABLE_MODULES = YES;
|
| 374 |
+
CLANG_ENABLE_OBJC_ARC = YES;
|
| 375 |
+
CLANG_ENABLE_OBJC_WEAK = YES;
|
| 376 |
+
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
|
| 377 |
+
CLANG_WARN_BOOL_CONVERSION = YES;
|
| 378 |
+
CLANG_WARN_COMMA = YES;
|
| 379 |
+
CLANG_WARN_CONSTANT_CONVERSION = YES;
|
| 380 |
+
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
|
| 381 |
+
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
|
| 382 |
+
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
|
| 383 |
+
CLANG_WARN_EMPTY_BODY = YES;
|
| 384 |
+
CLANG_WARN_ENUM_CONVERSION = YES;
|
| 385 |
+
CLANG_WARN_INFINITE_RECURSION = YES;
|
| 386 |
+
CLANG_WARN_INT_CONVERSION = YES;
|
| 387 |
+
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
|
| 388 |
+
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
|
| 389 |
+
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
|
| 390 |
+
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
|
| 391 |
+
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
|
| 392 |
+
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
|
| 393 |
+
CLANG_WARN_STRICT_PROTOTYPES = YES;
|
| 394 |
+
CLANG_WARN_SUSPICIOUS_MOVE = YES;
|
| 395 |
+
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
|
| 396 |
+
CLANG_WARN_UNREACHABLE_CODE = YES;
|
| 397 |
+
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
|
| 398 |
+
COPY_PHASE_STRIP = NO;
|
| 399 |
+
DEBUG_INFORMATION_FORMAT = dwarf;
|
| 400 |
+
ENABLE_STRICT_OBJC_MSGSEND = YES;
|
| 401 |
+
ENABLE_TESTABILITY = YES;
|
| 402 |
+
GCC_C_LANGUAGE_STANDARD = gnu11;
|
| 403 |
+
GCC_DYNAMIC_NO_PIC = NO;
|
| 404 |
+
GCC_NO_COMMON_BLOCKS = YES;
|
| 405 |
+
GCC_OPTIMIZATION_LEVEL = 0;
|
| 406 |
+
GCC_PREPROCESSOR_DEFINITIONS = (
|
| 407 |
+
"DEBUG=1",
|
| 408 |
+
"$(inherited)",
|
| 409 |
+
);
|
| 410 |
+
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
|
| 411 |
+
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
|
| 412 |
+
GCC_WARN_UNDECLARED_SELECTOR = YES;
|
| 413 |
+
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
|
| 414 |
+
GCC_WARN_UNUSED_FUNCTION = YES;
|
| 415 |
+
GCC_WARN_UNUSED_VARIABLE = YES;
|
| 416 |
+
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
|
| 417 |
+
MTL_FAST_MATH = YES;
|
| 418 |
+
ONLY_ACTIVE_ARCH = YES;
|
| 419 |
+
SDKROOT = xros;
|
| 420 |
+
SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
|
| 421 |
+
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
|
| 422 |
+
XROS_DEPLOYMENT_TARGET = 1.0;
|
| 423 |
+
};
|
| 424 |
+
name = Debug;
|
| 425 |
+
};
|
| 426 |
+
AAAAAAAAAAAAAAAAAAAAABBB /* Release */ = {
|
| 427 |
+
isa = XCBuildConfiguration;
|
| 428 |
+
buildSettings = {
|
| 429 |
+
ALWAYS_SEARCH_USER_PATHS = NO;
|
| 430 |
+
CLANG_ANALYZER_NONNULL = YES;
|
| 431 |
+
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
|
| 432 |
+
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
|
| 433 |
+
CLANG_CXX_LIBRARY = "libc++";
|
| 434 |
+
CLANG_ENABLE_MODULES = YES;
|
| 435 |
+
CLANG_ENABLE_OBJC_ARC = YES;
|
| 436 |
+
CLANG_ENABLE_OBJC_WEAK = YES;
|
| 437 |
+
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
|
| 438 |
+
CLANG_WARN_BOOL_CONVERSION = YES;
|
| 439 |
+
CLANG_WARN_COMMA = YES;
|
| 440 |
+
CLANG_WARN_CONSTANT_CONVERSION = YES;
|
| 441 |
+
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
|
| 442 |
+
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
|
| 443 |
+
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
|
| 444 |
+
CLANG_WARN_EMPTY_BODY = YES;
|
| 445 |
+
CLANG_WARN_ENUM_CONVERSION = YES;
|
| 446 |
+
CLANG_WARN_INFINITE_RECURSION = YES;
|
| 447 |
+
CLANG_WARN_INT_CONVERSION = YES;
|
| 448 |
+
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
|
| 449 |
+
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
|
| 450 |
+
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
|
| 451 |
+
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
|
| 452 |
+
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
|
| 453 |
+
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
|
| 454 |
+
CLANG_WARN_STRICT_PROTOTYPES = YES;
|
| 455 |
+
CLANG_WARN_SUSPICIOUS_MOVE = YES;
|
| 456 |
+
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
|
| 457 |
+
CLANG_WARN_UNREACHABLE_CODE = YES;
|
| 458 |
+
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
|
| 459 |
+
COPY_PHASE_STRIP = NO;
|
| 460 |
+
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
|
| 461 |
+
ENABLE_NS_ASSERTIONS = NO;
|
| 462 |
+
ENABLE_STRICT_OBJC_MSGSEND = YES;
|
| 463 |
+
GCC_C_LANGUAGE_STANDARD = gnu11;
|
| 464 |
+
GCC_NO_COMMON_BLOCKS = YES;
|
| 465 |
+
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
|
| 466 |
+
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
|
| 467 |
+
GCC_WARN_UNDECLARED_SELECTOR = YES;
|
| 468 |
+
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
|
| 469 |
+
GCC_WARN_UNUSED_FUNCTION = YES;
|
| 470 |
+
GCC_WARN_UNUSED_VARIABLE = YES;
|
| 471 |
+
MTL_ENABLE_DEBUG_INFO = NO;
|
| 472 |
+
MTL_FAST_MATH = YES;
|
| 473 |
+
SDKROOT = xros;
|
| 474 |
+
SWIFT_COMPILATION_MODE = wholemodule;
|
| 475 |
+
SWIFT_OPTIMIZATION_LEVEL = "-O";
|
| 476 |
+
VALIDATE_PRODUCT = YES;
|
| 477 |
+
XROS_DEPLOYMENT_TARGET = 1.0;
|
| 478 |
+
};
|
| 479 |
+
name = Release;
|
| 480 |
+
};
|
| 481 |
+
BBBBBBBBBBBBBBBBBBBBBBCCC /* Debug */ = {
|
| 482 |
+
isa = XCBuildConfiguration;
|
| 483 |
+
buildSettings = {
|
| 484 |
+
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
| 485 |
+
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
|
| 486 |
+
CODE_SIGN_STYLE = Manual;
|
| 487 |
+
DEVELOPMENT_TEAM = "";
|
| 488 |
+
ENABLE_PREVIEWS = YES;
|
| 489 |
+
FRAMEWORK_SEARCH_PATHS = "$(PROJECT_DIR)";
|
| 490 |
+
INFOPLIST_FILE = "VisionOSLlamaTest/Info.plist";
|
| 491 |
+
LD_RUNPATH_SEARCH_PATHS = (
|
| 492 |
+
"$(inherited)",
|
| 493 |
+
"@executable_path/Frameworks",
|
| 494 |
+
);
|
| 495 |
+
PRODUCT_BUNDLE_IDENTIFIER = "org.ggml.VisionOSLlamaTest";
|
| 496 |
+
PRODUCT_NAME = "$(TARGET_NAME)";
|
| 497 |
+
PROVISIONING_PROFILE_SPECIFIER = "";
|
| 498 |
+
SUPPORTED_PLATFORMS = "xros xrsimulator";
|
| 499 |
+
SWIFT_VERSION = 5.0;
|
| 500 |
+
TARGETED_DEVICE_FAMILY = "1,2,7";
|
| 501 |
+
};
|
| 502 |
+
name = Debug;
|
| 503 |
+
};
|
| 504 |
+
CCCCCCCCCCCCCCCCCCCCCCDDD /* Release */ = {
|
| 505 |
+
isa = XCBuildConfiguration;
|
| 506 |
+
buildSettings = {
|
| 507 |
+
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
| 508 |
+
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
|
| 509 |
+
CODE_SIGN_STYLE = Manual;
|
| 510 |
+
DEVELOPMENT_TEAM = "";
|
| 511 |
+
ENABLE_PREVIEWS = YES;
|
| 512 |
+
FRAMEWORK_SEARCH_PATHS = (
|
| 513 |
+
"$(inherited)",
|
| 514 |
+
"$(PROJECT_DIR)",
|
| 515 |
+
);
|
| 516 |
+
INFOPLIST_FILE = "VisionOSLlamaTest/Info.plist";
|
| 517 |
+
LD_RUNPATH_SEARCH_PATHS = (
|
| 518 |
+
"$(inherited)",
|
| 519 |
+
"@executable_path/Frameworks",
|
| 520 |
+
);
|
| 521 |
+
PRODUCT_BUNDLE_IDENTIFIER = "org.ggml.VisionOSLlamaTest";
|
| 522 |
+
PRODUCT_NAME = "$(TARGET_NAME)";
|
| 523 |
+
PROVISIONING_PROFILE_SPECIFIER = "";
|
| 524 |
+
SUPPORTED_PLATFORMS = "xros xrsimulator";
|
| 525 |
+
SWIFT_VERSION = 5.0;
|
| 526 |
+
TARGETED_DEVICE_FAMILY = "1,2,7";
|
| 527 |
+
};
|
| 528 |
+
name = Release;
|
| 529 |
+
};
|
| 530 |
+
/* End XCBuildConfiguration section */
|
| 531 |
+
EOF
|
| 532 |
+
|
| 533 |
+
# Finish the project.pbxproj file
|
| 534 |
+
cat >> "${TEMP_DIR}/${APP_NAME}/${APP_NAME}.xcodeproj/project.pbxproj" << EOF
|
| 535 |
+
/* Begin XCConfigurationList section */
|
| 536 |
+
8888888888888888888888AA /* Build configuration list for PBXProject "${APP_NAME}" */ = {
|
| 537 |
+
isa = XCConfigurationList;
|
| 538 |
+
buildConfigurations = (
|
| 539 |
+
9999999999999999999999AA /* Debug */,
|
| 540 |
+
AAAAAAAAAAAAAAAAAAAAABBB /* Release */,
|
| 541 |
+
);
|
| 542 |
+
defaultConfigurationIsVisible = 0;
|
| 543 |
+
defaultConfigurationName = Release;
|
| 544 |
+
};
|
| 545 |
+
4444444444444444444444AA /* Build configuration list for PBXNativeTarget "${APP_NAME}" */ = {
|
| 546 |
+
isa = XCConfigurationList;
|
| 547 |
+
buildConfigurations = (
|
| 548 |
+
BBBBBBBBBBBBBBBBBBBBBBCCC /* Debug */,
|
| 549 |
+
CCCCCCCCCCCCCCCCCCCCCCDDD /* Release */,
|
| 550 |
+
);
|
| 551 |
+
defaultConfigurationIsVisible = 0;
|
| 552 |
+
defaultConfigurationName = Release;
|
| 553 |
+
};
|
| 554 |
+
/* End XCConfigurationList section */
|
| 555 |
+
};
|
| 556 |
+
rootObject = 7777777777777777777777AA /* Project object */;
|
| 557 |
+
}
|
| 558 |
+
EOF
|
| 559 |
+
|
| 560 |
+
# 2. Copy XCFramework to test project
|
| 561 |
+
echo "Copying XCFramework to test project..."
|
| 562 |
+
cp -R "${XCFRAMEWORK_PATH}" "${TEMP_DIR}/${APP_NAME}/"
|
| 563 |
+
|
| 564 |
+
# 3. Build and archive the app
|
| 565 |
+
echo "Building and archiving test app..."
|
| 566 |
+
cd "${TEMP_DIR}/${APP_NAME}"
|
| 567 |
+
|
| 568 |
+
# Create a simple xcscheme file to avoid xcodebuild scheme issues
|
| 569 |
+
mkdir -p "${APP_NAME}.xcodeproj/xcshareddata/xcschemes"
|
| 570 |
+
cat > "${APP_NAME}.xcodeproj/xcshareddata/xcschemes/${APP_NAME}.xcscheme" << EOF
|
| 571 |
+
<?xml version="1.0" encoding="UTF-8"?>
|
| 572 |
+
<Scheme
|
| 573 |
+
LastUpgradeVersion = "1510"
|
| 574 |
+
version = "1.3">
|
| 575 |
+
<BuildAction
|
| 576 |
+
parallelizeBuildables = "YES"
|
| 577 |
+
buildImplicitDependencies = "YES">
|
| 578 |
+
<BuildActionEntries>
|
| 579 |
+
<BuildActionEntry
|
| 580 |
+
buildForTesting = "YES"
|
| 581 |
+
buildForRunning = "YES"
|
| 582 |
+
buildForProfiling = "YES"
|
| 583 |
+
buildForArchiving = "YES"
|
| 584 |
+
buildForAnalyzing = "YES">
|
| 585 |
+
<BuildableReference
|
| 586 |
+
BuildableIdentifier = "primary"
|
| 587 |
+
BlueprintIdentifier = "3333333333333333333333AA"
|
| 588 |
+
BuildableName = "${APP_NAME}.app"
|
| 589 |
+
BlueprintName = "${APP_NAME}"
|
| 590 |
+
ReferencedContainer = "container:${APP_NAME}.xcodeproj">
|
| 591 |
+
</BuildableReference>
|
| 592 |
+
</BuildActionEntry>
|
| 593 |
+
</BuildActionEntries>
|
| 594 |
+
</BuildAction>
|
| 595 |
+
<TestAction
|
| 596 |
+
buildConfiguration = "Debug"
|
| 597 |
+
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
|
| 598 |
+
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
|
| 599 |
+
shouldUseLaunchSchemeArgsEnv = "YES">
|
| 600 |
+
<Testables>
|
| 601 |
+
</Testables>
|
| 602 |
+
</TestAction>
|
| 603 |
+
<LaunchAction
|
| 604 |
+
buildConfiguration = "Debug"
|
| 605 |
+
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
|
| 606 |
+
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
|
| 607 |
+
launchStyle = "0"
|
| 608 |
+
useCustomWorkingDirectory = "NO"
|
| 609 |
+
ignoresPersistentStateOnLaunch = "NO"
|
| 610 |
+
debugDocumentVersioning = "YES"
|
| 611 |
+
debugServiceExtension = "internal"
|
| 612 |
+
allowLocationSimulation = "YES">
|
| 613 |
+
<BuildableProductRunnable
|
| 614 |
+
runnableDebuggingMode = "0">
|
| 615 |
+
<BuildableReference
|
| 616 |
+
BuildableIdentifier = "primary"
|
| 617 |
+
BlueprintIdentifier = "3333333333333333333333AA"
|
| 618 |
+
BuildableName = "${APP_NAME}.app"
|
| 619 |
+
BlueprintName = "${APP_NAME}"
|
| 620 |
+
ReferencedContainer = "container:${APP_NAME}.xcodeproj">
|
| 621 |
+
</BuildableReference>
|
| 622 |
+
</BuildableProductRunnable>
|
| 623 |
+
</LaunchAction>
|
| 624 |
+
<ProfileAction
|
| 625 |
+
buildConfiguration = "Release"
|
| 626 |
+
shouldUseLaunchSchemeArgsEnv = "YES"
|
| 627 |
+
savedToolIdentifier = ""
|
| 628 |
+
useCustomWorkingDirectory = "NO"
|
| 629 |
+
debugDocumentVersioning = "YES">
|
| 630 |
+
<BuildableProductRunnable
|
| 631 |
+
runnableDebuggingMode = "0">
|
| 632 |
+
<BuildableReference
|
| 633 |
+
BuildableIdentifier = "primary"
|
| 634 |
+
BlueprintIdentifier = "3333333333333333333333AA"
|
| 635 |
+
BuildableName = "${APP_NAME}.app"
|
| 636 |
+
BlueprintName = "${APP_NAME}"
|
| 637 |
+
ReferencedContainer = "container:${APP_NAME}.xcodeproj">
|
| 638 |
+
</BuildableReference>
|
| 639 |
+
</BuildableProductRunnable>
|
| 640 |
+
</ProfileAction>
|
| 641 |
+
<AnalyzeAction
|
| 642 |
+
buildConfiguration = "Debug">
|
| 643 |
+
</AnalyzeAction>
|
| 644 |
+
<ArchiveAction
|
| 645 |
+
buildConfiguration = "Release"
|
| 646 |
+
revealArchiveInOrganizer = "YES">
|
| 647 |
+
</ArchiveAction>
|
| 648 |
+
</Scheme>
|
| 649 |
+
EOF
|
| 650 |
+
|
| 651 |
+
# Now use xcodebuild with an explicitly defined product name for visionOS
|
| 652 |
+
xcodebuild -project "${APP_NAME}.xcodeproj" -scheme "${APP_NAME}" -sdk xros -configuration Release archive -archivePath "${ARCHIVE_PATH}" CODE_SIGN_IDENTITY="-" CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO PRODUCT_NAME="${APP_NAME}" SWIFT_OPTIMIZATION_LEVEL="-Onone" -quiet
|
| 653 |
+
|
| 654 |
+
# 4. Create IPA from archive
|
| 655 |
+
echo "Creating IPA from archive..."
|
| 656 |
+
mkdir -p "${TEMP_DIR}/Payload"
|
| 657 |
+
cp -R "${ARCHIVE_PATH}/Products/Applications/${APP_NAME}.app" "${TEMP_DIR}/Payload/"
|
| 658 |
+
|
| 659 |
+
# Check and log app structure before zipping
|
| 660 |
+
echo "App structure:"
|
| 661 |
+
ls -la "${TEMP_DIR}/Payload/${APP_NAME}.app/"
|
| 662 |
+
echo "Frameworks:"
|
| 663 |
+
ls -la "${TEMP_DIR}/Payload/${APP_NAME}.app/Frameworks/" 2>/dev/null || echo "No Frameworks directory found"
|
| 664 |
+
|
| 665 |
+
cd "${TEMP_DIR}"
|
| 666 |
+
zip -r "${IPA_PATH}" Payload
|
| 667 |
+
|
| 668 |
+
# Check embedded provisioning profile
|
| 669 |
+
echo "Checking provisioning profile (if any)..."
|
| 670 |
+
PROVISIONING_PROFILE=$(find "${ARCHIVE_PATH}/Products/Applications/${APP_NAME}.app" -name "embedded.mobileprovision" 2>/dev/null)
|
| 671 |
+
if [ -n "$PROVISIONING_PROFILE" ]; then
|
| 672 |
+
echo "Found embedded provisioning profile:"
|
| 673 |
+
security cms -D -i "$PROVISIONING_PROFILE" || echo "Unable to decode provisioning profile"
|
| 674 |
+
else
|
| 675 |
+
echo "No embedded provisioning profile found (expected for ad-hoc builds)"
|
| 676 |
+
fi
|
| 677 |
+
|
| 678 |
+
# 5. Validate the IPA
|
| 679 |
+
echo "Validating IPA..."
|
| 680 |
+
VALIDATION_OUTPUT="${VALIDATION_DIR}/validation_output.txt"
|
| 681 |
+
|
| 682 |
+
# Check if authentication credentials are provided
|
| 683 |
+
AUTH_ARGS=""
|
| 684 |
+
if [ -n "$APPLE_ID" ] && [ -n "$APPLE_PASSWORD" ]; then
|
| 685 |
+
echo "Using Apple ID authentication for validation..."
|
| 686 |
+
AUTH_ARGS="--username \"$APPLE_ID\" --password \"$APPLE_PASSWORD\""
|
| 687 |
+
else
|
| 688 |
+
echo "No authentication credentials provided. Will perform basic validation."
|
| 689 |
+
echo "To use your personal developer account, you can run the script with:"
|
| 690 |
+
echo " APPLE_ID='your.email@example.com' APPLE_PASSWORD='your-app-specific-password' ./validate-visionos.sh"
|
| 691 |
+
echo "Note: You need to create an app-specific password at https://appleid.apple.com/account/manage"
|
| 692 |
+
fi
|
| 693 |
+
|
| 694 |
+
# Run validation with detailed output
|
| 695 |
+
echo "Running validation with altool..."
|
| 696 |
+
if [ -n "$AUTH_ARGS" ]; then
|
| 697 |
+
# Use eval to properly handle the quoted arguments
|
| 698 |
+
eval "xcrun altool --validate-app -f \"${IPA_PATH}\" --type visionos --output-format xml $AUTH_ARGS" 2>&1 | tee "${VALIDATION_OUTPUT}"
|
| 699 |
+
else
|
| 700 |
+
xcrun altool --validate-app -f "${IPA_PATH}" --type visionos --output-format xml 2>&1 | tee "${VALIDATION_OUTPUT}"
|
| 701 |
+
fi
|
| 702 |
+
VALIDATION_RESULT=$?
|
| 703 |
+
|
| 704 |
+
# Final validation result
|
| 705 |
+
FINAL_VALIDATION_RESULT=0
|
| 706 |
+
|
| 707 |
+
# Check if validation failed because the app isn't in App Store Connect
|
| 708 |
+
if grep -q "No suitable application records were found" "${VALIDATION_OUTPUT}"; then
|
| 709 |
+
echo "⚠️ App Store Connect Warning: The app bundle identifier is not found in App Store Connect"
|
| 710 |
+
echo "This is expected for apps that haven't been registered in App Store Connect yet."
|
| 711 |
+
echo "This doesn't indicate a problem with the build or framework."
|
| 712 |
+
|
| 713 |
+
# Perform alternative validation
|
| 714 |
+
echo "Performing alternative validation checks..."
|
| 715 |
+
|
| 716 |
+
# Check if IPA was created successfully
|
| 717 |
+
if [ -f "${IPA_PATH}" ] && [ -s "${IPA_PATH}" ]; then
|
| 718 |
+
echo "✅ IPA file created successfully"
|
| 719 |
+
else
|
| 720 |
+
echo "❌ IPA file not created or empty"
|
| 721 |
+
FINAL_VALIDATION_RESULT=1
|
| 722 |
+
fi
|
| 723 |
+
|
| 724 |
+
# Check if app binary exists and is executable
|
| 725 |
+
if [ -f "${TEMP_DIR}/Payload/${APP_NAME}.app/${APP_NAME}" ] && [ -x "${TEMP_DIR}/Payload/${APP_NAME}.app/${APP_NAME}" ]; then
|
| 726 |
+
echo "✅ App binary exists and is executable"
|
| 727 |
+
else
|
| 728 |
+
echo "❌ App binary missing or not executable"
|
| 729 |
+
FINAL_VALIDATION_RESULT=1
|
| 730 |
+
fi
|
| 731 |
+
|
| 732 |
+
# Check if framework was properly embedded
|
| 733 |
+
if [ -d "${TEMP_DIR}/Payload/${APP_NAME}.app/Frameworks/llama.framework" ]; then
|
| 734 |
+
echo "✅ llama.framework properly embedded"
|
| 735 |
+
else
|
| 736 |
+
echo "❌ llama.framework not properly embedded"
|
| 737 |
+
FINAL_VALIDATION_RESULT=1
|
| 738 |
+
fi
|
| 739 |
+
|
| 740 |
+
# Check if framework binary exists
|
| 741 |
+
if [ -f "${TEMP_DIR}/Payload/${APP_NAME}.app/Frameworks/llama.framework/llama" ]; then
|
| 742 |
+
echo "✅ Framework binary exists"
|
| 743 |
+
|
| 744 |
+
# Further validate framework by checking architecture
|
| 745 |
+
ARCHS=$(lipo -info "${TEMP_DIR}/Payload/${APP_NAME}.app/Frameworks/llama.framework/llama" 2>/dev/null | grep -o "arm64\\|x86_64" | tr '\n' ' ')
|
| 746 |
+
if [ -n "$ARCHS" ]; then
|
| 747 |
+
echo "✅ Framework architecture(s): $ARCHS"
|
| 748 |
+
else
|
| 749 |
+
echo "⚠️ Could not determine framework architecture"
|
| 750 |
+
fi
|
| 751 |
+
else
|
| 752 |
+
echo "❌ Framework binary missing"
|
| 753 |
+
FINAL_VALIDATION_RESULT=1
|
| 754 |
+
fi
|
| 755 |
+
|
| 756 |
+
if [ $FINAL_VALIDATION_RESULT -eq 0 ]; then
|
| 757 |
+
echo "✅ Alternative validation PASSED: App built successfully with embedded framework"
|
| 758 |
+
else
|
| 759 |
+
echo "❌ Alternative validation FAILED: Issues found with the app or framework"
|
| 760 |
+
fi
|
| 761 |
+
elif grep -q "You must specify authentication credentials" "${VALIDATION_OUTPUT}" && [ -z "$AUTH_ARGS" ]; then
|
| 762 |
+
echo "✅ visionOS Validation PASSED: IPA successfully validated"
|
| 763 |
+
echo "Results saved to ${VALIDATION_OUTPUT}"
|
| 764 |
+
else
|
| 765 |
+
echo "❌ visionOS Validation FAILED: IPA validation found issues"
|
| 766 |
+
echo "See validation output at ${VALIDATION_OUTPUT}"
|
| 767 |
+
echo ""
|
| 768 |
+
echo "==== VALIDATION ERRORS ===="
|
| 769 |
+
|
| 770 |
+
# Try to extract specific errors from the output
|
| 771 |
+
if grep -q "Error" "${VALIDATION_OUTPUT}"; then
|
| 772 |
+
grep -A 5 "Error" "${VALIDATION_OUTPUT}"
|
| 773 |
+
else
|
| 774 |
+
# If no specific error found, show the whole log
|
| 775 |
+
cat "${VALIDATION_OUTPUT}"
|
| 776 |
+
fi
|
| 777 |
+
|
| 778 |
+
# Additional debugging: check IPA contents
|
| 779 |
+
echo ""
|
| 780 |
+
echo "==== IPA CONTENTS ===="
|
| 781 |
+
mkdir -p "${TEMP_DIR}/ipa_contents"
|
| 782 |
+
unzip -q "${IPA_PATH}" -d "${TEMP_DIR}/ipa_contents"
|
| 783 |
+
ls -la "${TEMP_DIR}/ipa_contents/Payload/${APP_NAME}.app/"
|
| 784 |
+
|
| 785 |
+
# Check for code signing issues
|
| 786 |
+
echo ""
|
| 787 |
+
echo "==== CODE SIGNING INFO ===="
|
| 788 |
+
codesign -vv -d "${TEMP_DIR}/ipa_contents/Payload/${APP_NAME}.app" 2>&1 || echo "Code signing verification failed"
|
| 789 |
+
|
| 790 |
+
# Check embedded frameworks
|
| 791 |
+
echo ""
|
| 792 |
+
echo "==== FRAMEWORK INFO ===="
|
| 793 |
+
ls -la "${TEMP_DIR}/ipa_contents/Payload/${APP_NAME}.app/Frameworks/" 2>/dev/null || echo "No Frameworks directory found"
|
| 794 |
+
fi
|
| 795 |
+
|
| 796 |
+
# Don't clean up on error to allow inspection
|
| 797 |
+
if [ $FINAL_VALIDATION_RESULT -ne 0 ]; then
|
| 798 |
+
echo ""
|
| 799 |
+
echo "Temporary files kept for inspection at: ${TEMP_DIR}"
|
| 800 |
+
echo "===== visionOS Validation Process Failed ====="
|
| 801 |
+
exit 1
|
| 802 |
+
fi
|
| 803 |
+
|
| 804 |
+
# Clean up temporary files but keep build artifacts
|
| 805 |
+
if [ $FINAL_VALIDATION_RESULT -eq 0 ]; then
|
| 806 |
+
echo "Cleaning up temporary files..."
|
| 807 |
+
#rm -rf "${TEMP_DIR}"
|
| 808 |
+
fi
|
| 809 |
+
|
| 810 |
+
echo "===== visionOS Validation Process Completed ====="
|
| 811 |
+
exit $FINAL_VALIDATION_RESULT
|
backend/llama.cpp/scripts/bench-models.sh
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
|
| 3 |
+
RESULTS="bench-models-results.txt"
|
| 4 |
+
: > "$RESULTS"
|
| 5 |
+
|
| 6 |
+
ARGS_BB="-c 270336 -npp 512,4096,8192 -npl 1,2,4,8,16,32 -ntg 32"
|
| 7 |
+
ARGS_B="-d 0,4096,8192,16384,32768 -p 2048 -n 32"
|
| 8 |
+
|
| 9 |
+
QUICK=0
|
| 10 |
+
DIO=0
|
| 11 |
+
while (( "$#" )); do
|
| 12 |
+
case "$1" in
|
| 13 |
+
--quick) QUICK=1; shift ;;
|
| 14 |
+
--dio) DIO=1; shift ;;
|
| 15 |
+
*) shift ;;
|
| 16 |
+
esac
|
| 17 |
+
done
|
| 18 |
+
|
| 19 |
+
if (( QUICK )); then
|
| 20 |
+
ARGS_BB="-c 20480 -npp 512,4096 -npl 1,2,4 -ntg 32"
|
| 21 |
+
ARGS_B="-d 0 -p 2048 -n 32"
|
| 22 |
+
fi
|
| 23 |
+
|
| 24 |
+
if (( DIO )); then
|
| 25 |
+
ARGS_BB="${ARGS_BB} --no-mmap --direct-io"
|
| 26 |
+
ARGS_B="${ARGS_B} -mmp 0 -dio 1"
|
| 27 |
+
fi
|
| 28 |
+
|
| 29 |
+
run_model() {
|
| 30 |
+
local HFR=$1
|
| 31 |
+
local HFF=$2
|
| 32 |
+
|
| 33 |
+
printf "## ${HFR}\n" | tee -a "$RESULTS"
|
| 34 |
+
printf "\n" | tee -a "$RESULTS"
|
| 35 |
+
printf "Model: https://huggingface.co/${HFR}\n" | tee -a "$RESULTS"
|
| 36 |
+
printf "\n" | tee -a "$RESULTS"
|
| 37 |
+
|
| 38 |
+
printf -- "- \`llama-batched-bench\`\n" | tee -a "$RESULTS"
|
| 39 |
+
printf "\n" | tee -a "$RESULTS"
|
| 40 |
+
|
| 41 |
+
./bin/llama-batched-bench \
|
| 42 |
+
-hfr "${HFR}" -hff "${HFF}" \
|
| 43 |
+
-m "${HFF}" -fa 1 -ub 2048 \
|
| 44 |
+
${ARGS_BB} | tee -a "$RESULTS"
|
| 45 |
+
|
| 46 |
+
printf "\n" | tee -a "$RESULTS"
|
| 47 |
+
|
| 48 |
+
printf -- "- \`llama-bench\`\n" | tee -a "$RESULTS"
|
| 49 |
+
printf "\n" | tee -a "$RESULTS"
|
| 50 |
+
|
| 51 |
+
./bin/llama-bench \
|
| 52 |
+
-m "${HFF}" -fa 1 -ub 2048 \
|
| 53 |
+
${ARGS_B} | tee -a "$RESULTS"
|
| 54 |
+
|
| 55 |
+
printf "\n" | tee -a "$RESULTS"
|
| 56 |
+
|
| 57 |
+
printf "\n"
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
run_model "ggml-org/gpt-oss-20b-GGUF" "gpt-oss-20b-mxfp4.gguf"
|
| 61 |
+
run_model "ggml-org/gpt-oss-120b-GGUF" "gpt-oss-120b-mxfp4-00001-of-00003.gguf"
|
| 62 |
+
run_model "ggml-org/Qwen3-Coder-30B-A3B-Instruct-Q8_0-GGUF" "qwen3-coder-30b-a3b-instruct-q8_0.gguf"
|
| 63 |
+
run_model "ggml-org/Qwen2.5-Coder-7B-Q8_0-GGUF" "qwen2.5-coder-7b-q8_0.gguf"
|
| 64 |
+
run_model "ggml-org/gemma-3-4b-it-qat-GGUF" "gemma-3-4b-it-qat-Q4_0.gguf"
|
| 65 |
+
run_model "ggml-org/GLM-4.7-Flash-GGUF" "GLM-4.7-Flash-Q8_0.gguf"
|
| 66 |
+
|
| 67 |
+
if [[ -f models-extra.txt ]]; then
|
| 68 |
+
while read -r HFR HFF; do
|
| 69 |
+
[[ -z "$HFR" ]] && continue
|
| 70 |
+
run_model "$HFR" "$HFF"
|
| 71 |
+
done < models-extra.txt
|
| 72 |
+
fi
|
| 73 |
+
|
| 74 |
+
printf "\n=====================================\n"
|
| 75 |
+
printf "\n"
|
| 76 |
+
|
| 77 |
+
cat "$RESULTS"
|
| 78 |
+
|
| 79 |
+
printf "\n"
|
| 80 |
+
printf "Done! Results are written to $RESULTS\n"
|
| 81 |
+
printf "\n"
|
| 82 |
+
|
backend/llama.cpp/scripts/build-info.sh
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/sh
|
| 2 |
+
|
| 3 |
+
CC=$1
|
| 4 |
+
|
| 5 |
+
build_number="0"
|
| 6 |
+
build_commit="unknown"
|
| 7 |
+
build_compiler="unknown"
|
| 8 |
+
build_target="unknown"
|
| 9 |
+
|
| 10 |
+
if out=$(git rev-list --count HEAD); then
|
| 11 |
+
# git is broken on WSL so we need to strip extra newlines
|
| 12 |
+
build_number=$(printf '%s' "$out" | tr -d '\n')
|
| 13 |
+
fi
|
| 14 |
+
|
| 15 |
+
if out=$(git rev-parse --short HEAD); then
|
| 16 |
+
build_commit=$(printf '%s' "$out" | tr -d '\n')
|
| 17 |
+
fi
|
| 18 |
+
|
| 19 |
+
if out=$($CC --version | head -1); then
|
| 20 |
+
build_compiler=$out
|
| 21 |
+
fi
|
| 22 |
+
|
| 23 |
+
if out=$($CC -dumpmachine); then
|
| 24 |
+
build_target=$out
|
| 25 |
+
fi
|
| 26 |
+
|
| 27 |
+
echo "int LLAMA_BUILD_NUMBER = ${build_number};"
|
| 28 |
+
echo "char const *LLAMA_COMMIT = \"${build_commit}\";"
|
| 29 |
+
echo "char const *LLAMA_COMPILER = \"${build_compiler}\";"
|
| 30 |
+
echo "char const *LLAMA_BUILD_TARGET = \"${build_target}\";"
|
backend/llama.cpp/scripts/check-requirements.sh
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
|
| 4 |
+
#
|
| 5 |
+
# check-requirements.sh checks all requirements files for each top-level
|
| 6 |
+
# convert*.py script.
|
| 7 |
+
#
|
| 8 |
+
# WARNING: This is quite IO intensive, because a fresh venv is set up for every
|
| 9 |
+
# python script. As of 2023-12-22, this writes ~2.7GB of data. An adequately
|
| 10 |
+
# sized tmpfs /tmp or ramdisk is recommended if running this frequently.
|
| 11 |
+
#
|
| 12 |
+
# usage: check-requirements.sh [<working_dir>]
|
| 13 |
+
# check-requirements.sh nocleanup [<working_dir>]
|
| 14 |
+
#
|
| 15 |
+
# where:
|
| 16 |
+
# - <working_dir> is a directory that can be used as the base for
|
| 17 |
+
# setting up the venvs. Defaults to `/tmp`.
|
| 18 |
+
# - 'nocleanup' as the first argument will disable automatic cleanup
|
| 19 |
+
# of the files created by this script.
|
| 20 |
+
#
|
| 21 |
+
# requires:
|
| 22 |
+
# - bash >= 3.2.57
|
| 23 |
+
# - shellcheck
|
| 24 |
+
#
|
| 25 |
+
# For each script, it creates a fresh venv, `pip install`s the requirements, and
|
| 26 |
+
# finally imports the python script to check for `ImportError`.
|
| 27 |
+
#
|
| 28 |
+
|
| 29 |
+
log() {
|
| 30 |
+
local level=$1 msg=$2
|
| 31 |
+
printf >&2 '%s: %s\n' "$level" "$msg"
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
debug() {
|
| 35 |
+
log DEBUG "$@"
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
info() {
|
| 39 |
+
log INFO "$@"
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
fatal() {
|
| 43 |
+
log FATAL "$@"
|
| 44 |
+
exit 1
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
cleanup() {
|
| 48 |
+
if [[ -n ${workdir+x} && -d $workdir && -w $workdir ]]; then
|
| 49 |
+
info "Removing $workdir"
|
| 50 |
+
local count=0
|
| 51 |
+
rm -rfv -- "$workdir" | while read -r; do
|
| 52 |
+
if (( count++ > 750 )); then
|
| 53 |
+
printf .
|
| 54 |
+
count=0
|
| 55 |
+
fi
|
| 56 |
+
done
|
| 57 |
+
printf '\n'
|
| 58 |
+
info "Removed $workdir"
|
| 59 |
+
fi
|
| 60 |
+
}
|
| 61 |
+
|
| 62 |
+
do_cleanup=1
|
| 63 |
+
if [[ ${1-} == nocleanup ]]; then
|
| 64 |
+
do_cleanup=0; shift
|
| 65 |
+
fi
|
| 66 |
+
|
| 67 |
+
if (( do_cleanup )); then
|
| 68 |
+
trap exit INT TERM
|
| 69 |
+
trap cleanup EXIT
|
| 70 |
+
fi
|
| 71 |
+
|
| 72 |
+
this=$(realpath -- "$0"); readonly this
|
| 73 |
+
cd "$(dirname "$this")/.." # PWD should stay in llama.cpp project directory
|
| 74 |
+
|
| 75 |
+
shellcheck "$this"
|
| 76 |
+
|
| 77 |
+
readonly reqs_dir=requirements
|
| 78 |
+
|
| 79 |
+
if [[ ${1+x} ]]; then
|
| 80 |
+
tmp_dir=$(realpath -- "$1")
|
| 81 |
+
if [[ ! ( -d $tmp_dir && -w $tmp_dir ) ]]; then
|
| 82 |
+
fatal "$tmp_dir is not a writable directory"
|
| 83 |
+
fi
|
| 84 |
+
else
|
| 85 |
+
tmp_dir=/tmp
|
| 86 |
+
fi
|
| 87 |
+
|
| 88 |
+
workdir=$(mktemp -d "$tmp_dir/check-requirements.XXXX"); readonly workdir
|
| 89 |
+
info "Working directory: $workdir"
|
| 90 |
+
|
| 91 |
+
check_requirements() {
|
| 92 |
+
local reqs=$1
|
| 93 |
+
|
| 94 |
+
info "$reqs: beginning check"
|
| 95 |
+
pip --disable-pip-version-check install -qr "$reqs"
|
| 96 |
+
info "$reqs: OK"
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
check_convert_script() {
|
| 100 |
+
local py=$1 # e.g. ./convert_hf_to_gguf.py
|
| 101 |
+
local pyname=${py##*/} # e.g. convert_hf_to_gguf.py
|
| 102 |
+
pyname=${pyname%.py} # e.g. convert_hf_to_gguf
|
| 103 |
+
|
| 104 |
+
info "$py: beginning check"
|
| 105 |
+
|
| 106 |
+
local reqs="$reqs_dir/requirements-$pyname.txt"
|
| 107 |
+
if [[ ! -r $reqs ]]; then
|
| 108 |
+
fatal "$py missing requirements. Expected: $reqs"
|
| 109 |
+
fi
|
| 110 |
+
|
| 111 |
+
# Check that all sub-requirements are added to top-level requirements.txt
|
| 112 |
+
if ! grep -qF "$reqs" requirements.txt; then
|
| 113 |
+
fatal "$reqs needs to be added to requirements.txt"
|
| 114 |
+
fi
|
| 115 |
+
|
| 116 |
+
local venv="$workdir/$pyname-venv"
|
| 117 |
+
python3 -m venv "$venv"
|
| 118 |
+
|
| 119 |
+
(
|
| 120 |
+
# shellcheck source=/dev/null
|
| 121 |
+
source "$venv/bin/activate"
|
| 122 |
+
|
| 123 |
+
check_requirements "$reqs"
|
| 124 |
+
|
| 125 |
+
python - "$py" "$pyname" <<'EOF'
|
| 126 |
+
import sys
|
| 127 |
+
from importlib.machinery import SourceFileLoader
|
| 128 |
+
py, pyname = sys.argv[1:]
|
| 129 |
+
SourceFileLoader(pyname, py).load_module()
|
| 130 |
+
EOF
|
| 131 |
+
)
|
| 132 |
+
|
| 133 |
+
if (( do_cleanup )); then
|
| 134 |
+
rm -rf -- "$venv"
|
| 135 |
+
fi
|
| 136 |
+
|
| 137 |
+
info "$py: imports OK"
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
readonly ignore_eq_eq='check_requirements: ignore "=="'
|
| 141 |
+
|
| 142 |
+
for req in */**/requirements*.txt; do
|
| 143 |
+
# Make sure exact release versions aren't being pinned in the requirements
|
| 144 |
+
# Filters out the ignore string
|
| 145 |
+
if grep -vF "$ignore_eq_eq" "$req" | grep -q '=='; then
|
| 146 |
+
tab=$'\t'
|
| 147 |
+
cat >&2 <<EOF
|
| 148 |
+
FATAL: Avoid pinning exact package versions. Use '~=' instead.
|
| 149 |
+
You can suppress this error by appending the following to the line:
|
| 150 |
+
$tab# $ignore_eq_eq
|
| 151 |
+
EOF
|
| 152 |
+
exit 1
|
| 153 |
+
fi
|
| 154 |
+
done
|
| 155 |
+
|
| 156 |
+
all_venv="$workdir/all-venv"
|
| 157 |
+
python3 -m venv "$all_venv"
|
| 158 |
+
|
| 159 |
+
(
|
| 160 |
+
# shellcheck source=/dev/null
|
| 161 |
+
source "$all_venv/bin/activate"
|
| 162 |
+
check_requirements requirements.txt
|
| 163 |
+
)
|
| 164 |
+
|
| 165 |
+
if (( do_cleanup )); then
|
| 166 |
+
rm -rf -- "$all_venv"
|
| 167 |
+
fi
|
| 168 |
+
|
| 169 |
+
check_convert_script examples/convert_legacy_llama.py
|
| 170 |
+
for py in convert_*.py; do
|
| 171 |
+
# skip convert_hf_to_gguf_update.py
|
| 172 |
+
# TODO: the check is failing for some reason:
|
| 173 |
+
# https://github.com/ggml-org/llama.cpp/actions/runs/8875330981/job/24364557177?pr=6920
|
| 174 |
+
[[ $py == convert_hf_to_gguf_update.py ]] && continue
|
| 175 |
+
|
| 176 |
+
check_convert_script "$py"
|
| 177 |
+
done
|
| 178 |
+
|
| 179 |
+
info 'Done! No issues found.'
|
backend/llama.cpp/scripts/compare-commits.sh
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
|
| 3 |
+
if [ $# -lt 2 ]; then
|
| 4 |
+
echo "usage: ./scripts/compare-commits.sh <commit1> <commit2> [tool] [additional arguments]"
|
| 5 |
+
echo " tool: 'llama-bench' (default) or 'test-backend-ops'"
|
| 6 |
+
echo " additional arguments: passed to the selected tool"
|
| 7 |
+
exit 1
|
| 8 |
+
fi
|
| 9 |
+
|
| 10 |
+
set -e
|
| 11 |
+
set -x
|
| 12 |
+
|
| 13 |
+
# Parse arguments
|
| 14 |
+
commit1=$1
|
| 15 |
+
commit2=$2
|
| 16 |
+
tool=${3:-llama-bench}
|
| 17 |
+
additional_args="${@:4}"
|
| 18 |
+
|
| 19 |
+
# Validate tool argument
|
| 20 |
+
if [ "$tool" != "llama-bench" ] && [ "$tool" != "test-backend-ops" ]; then
|
| 21 |
+
echo "Error: tool must be 'llama-bench' or 'test-backend-ops'"
|
| 22 |
+
exit 1
|
| 23 |
+
fi
|
| 24 |
+
|
| 25 |
+
# verify at the start that the compare script has all the necessary dependencies installed
|
| 26 |
+
./scripts/compare-llama-bench.py --check
|
| 27 |
+
|
| 28 |
+
if ! command -v sqlite3 >/dev/null 2>&1; then
|
| 29 |
+
echo "Error: sqlite3 is not installed or not in PATH"
|
| 30 |
+
echo "Please install sqlite3 to use this script"
|
| 31 |
+
exit 1
|
| 32 |
+
fi
|
| 33 |
+
|
| 34 |
+
if [ "$tool" = "llama-bench" ]; then
|
| 35 |
+
db_file="llama-bench.sqlite"
|
| 36 |
+
target="llama-bench"
|
| 37 |
+
run_args="-o sql -oe md $additional_args"
|
| 38 |
+
else # test-backend-ops
|
| 39 |
+
db_file="test-backend-ops.sqlite"
|
| 40 |
+
target="test-backend-ops"
|
| 41 |
+
run_args="perf --output sql $additional_args"
|
| 42 |
+
fi
|
| 43 |
+
|
| 44 |
+
rm -f "$db_file" > /dev/null
|
| 45 |
+
|
| 46 |
+
# to test a backend, call the script with the corresponding environment variable (e.g. GGML_CUDA=1 ./scripts/compare-commits.sh ...)
|
| 47 |
+
if [ -n "$GGML_CUDA" ]; then
|
| 48 |
+
CMAKE_OPTS="${CMAKE_OPTS} -DGGML_CUDA=ON"
|
| 49 |
+
fi
|
| 50 |
+
|
| 51 |
+
dir="build-bench"
|
| 52 |
+
|
| 53 |
+
function run {
|
| 54 |
+
rm -fr ${dir} > /dev/null
|
| 55 |
+
cmake -B ${dir} -S . ${CMAKE_OPTS} > /dev/null
|
| 56 |
+
cmake --build ${dir} -t $target -j $(nproc) > /dev/null
|
| 57 |
+
${dir}/bin/$target $run_args | sqlite3 "$db_file"
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
git checkout $commit1 > /dev/null
|
| 61 |
+
run
|
| 62 |
+
|
| 63 |
+
git checkout $commit2 > /dev/null
|
| 64 |
+
run
|
| 65 |
+
|
| 66 |
+
./scripts/compare-llama-bench.py -b $commit1 -c $commit2 --tool $tool -i "$db_file"
|