run-ornith-mtp-windows.ps1

[CmdletBinding()]
param(
    [Parameter()]
    [string]$ModelDir = 'C:\Models\Ornith-MTP',

    [Parameter()]
    [ValidateRange(1, 65535)]
    [int]$Port = 18081,

    [Parameter()]
    [ValidateRange(4096, 131072)]
    [int]$Context = 16384,

    [Parameter()]
    [string]$ContainerName = 'ornith-mtp-api'
)

$ErrorActionPreference = 'Stop'

$Image = 'ghcr.io/ggml-org/llama.cpp@sha256:1b3d1458ccda7287feab41b8001311acc03e24cde99ec0a2908fe83830562f38'
$ModelFile = 'ornith-1.0-35b-MTP-graft-down-Q4_0.gguf'
$ModelPath = Join-Path $ModelDir $ModelFile

if (-not (Get-Command docker -ErrorAction SilentlyContinue)) {
    throw 'docker.exe was not found. Install and start Docker Desktop first.'
}

if (-not (Test-Path -LiteralPath $ModelPath -PathType Leaf)) {
    throw "Model file was not found: $ModelPath"
}

$existing = docker ps -a --filter "name=^/${ContainerName}$" --format '{{.Names}}'
if ($LASTEXITCODE -ne 0) {
    throw 'Docker is unavailable. Start Docker Desktop and verify WSL2/NVIDIA support.'
}
if ($existing -contains $ContainerName) {
    throw "Container '$ContainerName' already exists. Stop it with: docker stop $ContainerName"
}

Write-Host "Starting $ContainerName on http://127.0.0.1:$Port ..."

$DockerArgs = @(
    'run', '--detach', '--rm',
    '--name', $ContainerName,
    '--gpus', 'all',
    '--publish', "127.0.0.1:${Port}:8080",
    '--mount', "type=bind,source=$ModelDir,target=/models,readonly",
    $Image,
    '--model', "/models/$ModelFile",
    '--alias', 'ornith-1.0-35b-mtp',
    '--host', '0.0.0.0',
    '--port', '8080',
    '--ctx-size', [string]$Context,
    '--parallel', '1',
    '--n-gpu-layers', 'all',
    '--cpu-moe',
    '--no-mmap',
    '--batch-size', '2048',
    '--ubatch-size', '512',
    '--flash-attn', 'on',
    '--cache-type-k', 'q8_0',
    '--cache-type-v', 'q8_0',
    '--jinja',
    '--metrics',
    '--spec-type', 'ngram-mod,draft-mtp',
    '--spec-draft-n-max', '2',
    '--spec-draft-n-min', '1',
    '--spec-draft-p-min', '0.20',
    '--spec-ngram-mod-n-min', '48',
    '--spec-ngram-mod-n-max', '64',
    '--spec-ngram-mod-n-match', '24'
)

$ContainerId = & docker @DockerArgs
if ($LASTEXITCODE -ne 0) {
    throw 'docker run failed. Review the Docker Desktop and NVIDIA configuration.'
}
Write-Host "Container: $ContainerId"

$HealthUrl = "http://127.0.0.1:$Port/health"
$Deadline = (Get-Date).AddMinutes(5)
do {
    Start-Sleep -Seconds 2
    try {
        $Health = Invoke-RestMethod -Uri $HealthUrl -TimeoutSec 5
        if ($Health.status -eq 'ok') {
            Write-Host 'Ornith endpoint is ready.' -ForegroundColor Green
            Write-Host "OpenAI base URL: http://127.0.0.1:$Port/v1"
            Write-Host 'Model: ornith-1.0-35b-mtp'
            Write-Host "Logs: docker logs -f $ContainerName"
            Write-Host "Stop: docker stop $ContainerName"
            exit 0
        }
    } catch {
        # Model loading is still in progress.
    }
} while ((Get-Date) -lt $Deadline)

Write-Warning "Endpoint did not become ready within five minutes. Inspect: docker logs $ContainerName"
exit 1