File size: 3,081 Bytes
9e3a41c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 | [CmdletBinding()]
param(
[Parameter()]
[string]$ModelDir = 'C:\Models\Ornith-MTP',
[Parameter()]
[ValidateRange(1, 65535)]
[int]$Port = 18081,
[Parameter()]
[ValidateRange(4096, 131072)]
[int]$Context = 16384,
[Parameter()]
[string]$ContainerName = 'ornith-mtp-api'
)
$ErrorActionPreference = 'Stop'
$Image = 'ghcr.io/ggml-org/llama.cpp@sha256:1b3d1458ccda7287feab41b8001311acc03e24cde99ec0a2908fe83830562f38'
$ModelFile = 'ornith-1.0-35b-MTP-graft-down-Q4_0.gguf'
$ModelPath = Join-Path $ModelDir $ModelFile
if (-not (Get-Command docker -ErrorAction SilentlyContinue)) {
throw 'docker.exe was not found. Install and start Docker Desktop first.'
}
if (-not (Test-Path -LiteralPath $ModelPath -PathType Leaf)) {
throw "Model file was not found: $ModelPath"
}
$existing = docker ps -a --filter "name=^/${ContainerName}$" --format '{{.Names}}'
if ($LASTEXITCODE -ne 0) {
throw 'Docker is unavailable. Start Docker Desktop and verify WSL2/NVIDIA support.'
}
if ($existing -contains $ContainerName) {
throw "Container '$ContainerName' already exists. Stop it with: docker stop $ContainerName"
}
Write-Host "Starting $ContainerName on http://127.0.0.1:$Port ..."
$DockerArgs = @(
'run', '--detach', '--rm',
'--name', $ContainerName,
'--gpus', 'all',
'--publish', "127.0.0.1:${Port}:8080",
'--mount', "type=bind,source=$ModelDir,target=/models,readonly",
$Image,
'--model', "/models/$ModelFile",
'--alias', 'ornith-1.0-35b-mtp',
'--host', '0.0.0.0',
'--port', '8080',
'--ctx-size', [string]$Context,
'--parallel', '1',
'--n-gpu-layers', 'all',
'--cpu-moe',
'--no-mmap',
'--batch-size', '2048',
'--ubatch-size', '512',
'--flash-attn', 'on',
'--cache-type-k', 'q8_0',
'--cache-type-v', 'q8_0',
'--jinja',
'--metrics',
'--spec-type', 'ngram-mod,draft-mtp',
'--spec-draft-n-max', '2',
'--spec-draft-n-min', '1',
'--spec-draft-p-min', '0.20',
'--spec-ngram-mod-n-min', '48',
'--spec-ngram-mod-n-max', '64',
'--spec-ngram-mod-n-match', '24'
)
$ContainerId = & docker @DockerArgs
if ($LASTEXITCODE -ne 0) {
throw 'docker run failed. Review the Docker Desktop and NVIDIA configuration.'
}
Write-Host "Container: $ContainerId"
$HealthUrl = "http://127.0.0.1:$Port/health"
$Deadline = (Get-Date).AddMinutes(5)
do {
Start-Sleep -Seconds 2
try {
$Health = Invoke-RestMethod -Uri $HealthUrl -TimeoutSec 5
if ($Health.status -eq 'ok') {
Write-Host 'Ornith endpoint is ready.' -ForegroundColor Green
Write-Host "OpenAI base URL: http://127.0.0.1:$Port/v1"
Write-Host 'Model: ornith-1.0-35b-mtp'
Write-Host "Logs: docker logs -f $ContainerName"
Write-Host "Stop: docker stop $ContainerName"
exit 0
}
} catch {
# Model loading is still in progress.
}
} while ((Get-Date) -lt $Deadline)
Write-Warning "Endpoint did not become ready within five minutes. Inspect: docker logs $ContainerName"
exit 1
|