opencoti F5-opt W1 (#293) — PCIe/ReBAR profile boot-time reader + autodetect flags WHAT Consumes the shipped rebar/PCIe probe (perf/llamafile/rebar-probe.{cu,sh}) at server boot: reads the newest probe JSON (or $OPENCOTI_PCIE_PROFILE) for the measured pinned H2D bandwidth + PCIe link width/speed, falling back to a one-shot nvidia-smi query, then a conservative PCIe3 x8 default. The result (effective GB/s + link geometry) is cached in a process-global pcie_profile, logged at startup, and consumed later by M7 Rolling KV (#296) for streaming tile sizing (tile_bytes_max = compute_ms * eff_bw_GBps * 0.8). Adds: - common/pcie-profile.{h,cpp} (NEW — detection + global) - common_params.pcie_autodetect/pcie_bw_gbps (common.h) - --pcie-autodetect / --pcie-bw-gbps (common/arg.cpp) - common/pcie-profile.cpp TU (BUILD.mk COMMON_SRCS_CPP) - pcie_profile_init() + boot log (tools/server/server.cpp — hook) PROVENANCE (per opencoti upstream-sync directive) opencoti-ORIGINAL — no code copied from upstream llama.cpp. The PCIe-link sysfs read + ReBAR-active detection technique derives from lightseek's tools/windows/rebar_test.cu (Windows NVML/VirtualAlloc original), Linux-ported by opencoti in perf/llamafile/rebar-probe.cu (#293). The server-side reader here parses that probe's JSON. Upstream base: ggml-org/llama.cpp via Mozilla-Ocho/llamafile submodule 5e9c63546 (llamafile 0.10.1 pin). Apply order: after 0035 (independent — common/ + server additive). Surgical hook: tools/server/server.cpp boot call — registered in docs/protocols/UPSTREAM_SYNC.md. Co-Authored-By: Claude Opus 4.8 (1M context) diff -ruN state-pre-0036/llama.cpp/BUILD.mk state-post-0036/llama.cpp/BUILD.mk diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mk --- a/llama.cpp/BUILD.mk +++ b/llama.cpp/BUILD.mk @@ -242,6 +242,7 @@ COMMON_SRCS_CPP := \ llama.cpp/common/ngram-cache.cpp \ llama.cpp/common/ngram-map.cpp \ llama.cpp/common/ngram-mod.cpp \ + llama.cpp/common/pcie-profile.cpp \ llama.cpp/common/peg-parser.cpp \ llama.cpp/common/preset.cpp \ llama.cpp/common/reasoning-budget.cpp \ diff --git a/llama.cpp/common/arg.cpp b/llama.cpp/common/arg.cpp --- a/llama.cpp/common/arg.cpp +++ b/llama.cpp/common/arg.cpp @@ -1440,6 +1440,23 @@ common_params_context common_params_parser_init(common_params & params, llama_ex else throw std::invalid_argument("--neo-pipeline must be off|on|auto"); } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_NEO_PIPELINE")); + // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md + add_opt(common_arg( + {"--pcie-autodetect"}, "MODE", + "pcie profile: auto-detect host<->device bandwidth + PCIe link at boot (on (default), off). Reads the rebar-probe JSON / nvidia-smi; the result feeds M7 Rolling KV tile sizing.", + [](common_params & params, const std::string & value) { + if (value == "on" || value == "true" || value == "1") params.pcie_autodetect = true; + else if (value == "off" || value == "false" || value == "0") params.pcie_autodetect = false; + else throw std::invalid_argument("--pcie-autodetect must be on|off"); + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_PCIE_AUTODETECT")); + add_opt(common_arg( + {"--pcie-bw-gbps"}, "F", + string_format("pcie profile: manual override of effective host->device bandwidth in GB/s (default: %.1f = autodetect). > 0 skips detection.", (double) params.pcie_bw_gbps), + [](common_params & params, const std::string & value) { + params.pcie_bw_gbps = std::stof(value); + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_PCIE_BW_GBPS")); add_opt(common_arg( {"--chunks"}, "N", string_format("max number of chunks to process (default: %d, -1 = all)", params.n_chunks), diff --git a/llama.cpp/common/common.h b/llama.cpp/common/common.h --- a/llama.cpp/common/common.h +++ b/llama.cpp/common/common.h @@ -445,6 +445,12 @@ struct common_params { // active for the layer), 2 = auto (reserved for future load-based // engagement; same as `on` today). uint32_t neo_pipeline_mode = 0; + // opencoti F5-opt W1 (#293) pcie profile — see docs/features/rolling_kv.md + // Boot-time host<->device bandwidth + PCIe link detection, consumed by M7 + // Rolling KV (#296) tile sizing. autodetect reads the rebar-probe JSON / + // nvidia-smi; bw_gbps > 0 forces a manual override (0 = autodetect/default). + bool pcie_autodetect = true; + float pcie_bw_gbps = 0.0f; int32_t n_batch = 2048; // logical batch size for prompt processing (must be >=32 to use BLAS) int32_t n_ubatch = 512; // physical batch size for prompt processing (must be >=32 to use BLAS) int32_t n_keep = 0; // number of tokens to keep from initial prompt diff --git a/llama.cpp/common/pcie-profile.cpp b/llama.cpp/common/pcie-profile.cpp new file mode 100644 --- /dev/null +++ b/llama.cpp/common/pcie-profile.cpp @@ -0,0 +1,201 @@ +// opencoti F5-opt W1 (#293) — PCIe link + host<->device bandwidth profile. +// See pcie-profile.h for the contract. CPU-side only; never throws. + +#include "pcie-profile.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +// PCIe generation -> per-lane raw transfer rate (GT/s). +float gen_to_gtps(int gen) { + switch (gen) { + case 1: return 2.5f; + case 2: return 5.0f; + case 3: return 8.0f; + case 4: return 16.0f; + case 5: return 32.0f; + case 6: return 64.0f; + default: return 0.0f; + } +} + +// Effective usable H2D bandwidth from link geometry. The 0.8 factor folds in +// 128b/130b (gen3+) encoding overhead + DMA/protocol inefficiency; matches the +// rolling_kv.md auto-adapt formula. width lanes * GT/s / 8 bits-per-byte * 0.8. +float geometry_bw_gbps(int width, float gtps) { + if (width <= 0 || gtps <= 0.0f) return 0.0f; + return (float) width * gtps * 0.8f / 8.0f; +} + +// "8.0 GT/s PCIe" / "16.0 GT/s PCIe" -> leading float. Empty/garbage -> 0. +float parse_leading_float(const std::string & s) { + return s.empty() ? 0.0f : (float) atof(s.c_str()); +} + +// Minimal value-after-key extraction for our OWN well-formed probe JSON. +// Returns the substring between key's ':' and the next ',' or '}' / newline. +bool json_raw_after(const std::string & body, const std::string & key, std::string & out) { + const size_t k = body.find(key); + if (k == std::string::npos) return false; + size_t c = body.find(':', k + key.size()); + if (c == std::string::npos) return false; + ++c; + while (c < body.size() && (body[c] == ' ' || body[c] == '\t')) ++c; + size_t e = c; + while (e < body.size() && body[e] != ',' && body[e] != '\n' && body[e] != '}') ++e; + out = body.substr(c, e - c); + return true; +} + +float json_float_after(const std::string & body, const std::string & key) { + std::string raw; + if (!json_raw_after(body, key, raw)) return 0.0f; + return (float) atof(raw.c_str()); +} + +// Quoted-string value: trims surrounding quotes/spaces. +std::string json_string_after(const std::string & body, const std::string & key) { + std::string raw; + if (!json_raw_after(body, key, raw)) return {}; + size_t a = raw.find('"'); + if (a == std::string::npos) return {}; + size_t b = raw.find('"', a + 1); + if (b == std::string::npos) return {}; + return raw.substr(a + 1, b - a - 1); +} + +std::string read_file(const std::string & path) { + std::ifstream f(path, std::ios::binary); + if (!f.is_open()) return {}; + std::ostringstream ss; + ss << f.rdbuf(); + return ss.str(); +} + +// Newest ./.opencoti/rebar-probe-*.json. Filenames embed a YYYYMMDD-HHMMSS +// stamp, so lexicographic max == newest. Empty if none. +std::string newest_probe_json() { + const char * env = getenv("OPENCOTI_PCIE_PROFILE"); + if (env && env[0]) return std::string(env); + + const char * dirs[] = {"./.opencoti", ".opencoti"}; + std::string best; + for (const char * d : dirs) { + DIR * dp = opendir(d); + if (!dp) continue; + struct dirent * ent; + while ((ent = readdir(dp)) != nullptr) { + const std::string name = ent->d_name; + if (name.rfind("rebar-probe-", 0) != 0) continue; + if (name.size() < 5 || name.compare(name.size() - 5, 5, ".json") != 0) continue; + const std::string full = std::string(d) + "/" + name; + if (best.empty() || name > best.substr(best.find_last_of('/') + 1)) best = full; + } + closedir(dp); + if (!best.empty()) break; + } + return best; +} + +// Parse a probe JSON into a profile. Prefers the MEASURED pinned H2D bandwidth +// over the geometry formula; still records link width/speed for logging. +bool profile_from_probe_json(const std::string & path, pcie_profile & out) { + const std::string body = read_file(path); + if (body.empty()) return false; + + // measured pinned H2D: locate the "pinned" object, then its "h2d". + float pinned_h2d = 0.0f; + const size_t p = body.find("\"pinned\""); + if (p != std::string::npos) { + pinned_h2d = json_float_after(body.substr(p), "\"h2d\""); + } + + const int width = atoi(json_string_after(body, "\"pci_link_width_current\"").c_str()); + const float gtps = parse_leading_float(json_string_after(body, "\"pci_link_speed_current\"")); + + if (pinned_h2d <= 0.0f && width <= 0) return false; + + out.link_width = width; + out.link_gtps = gtps; + out.effective_bw_gbps = pinned_h2d > 0.0f ? pinned_h2d : geometry_bw_gbps(width, gtps); + out.source = "probe-json"; + out.detail = "from " + path + (pinned_h2d > 0.0f ? " (measured pinned H2D)" : " (link geometry)"); + return out.effective_bw_gbps > 0.0f; +} + +// One-shot nvidia-smi query for link gen + width. CPU-side, no CUDA link dep. +bool profile_from_nvidia_smi(pcie_profile & out) { + FILE * pp = popen("nvidia-smi --query-gpu=pcie.link.gen.current,pcie.link.width.current " + "--format=csv,noheader,nounits -i 0 2>/dev/null", "r"); + if (!pp) return false; + char buf[256] = {}; + const bool got = fgets(buf, sizeof(buf), pp) != nullptr; + pclose(pp); + if (!got) return false; + + int gen = 0, width = 0; + // format: ", " + if (sscanf(buf, " %d , %d", &gen, &width) < 2) return false; + const float gtps = gen_to_gtps(gen); + const float bw = geometry_bw_gbps(width, gtps); + if (bw <= 0.0f) return false; + + out.link_width = width; + out.link_gtps = gtps; + out.effective_bw_gbps = bw; + out.source = "nvidia-smi"; + out.detail = "gen" + std::to_string(gen) + " x" + std::to_string(width) + " (link geometry)"; + return true; +} + +pcie_profile g_pcie_profile; +bool g_pcie_inited = false; + +} // namespace + +pcie_profile pcie_profile_detect(bool autodetect, float manual_gbps) { + pcie_profile prof; // conservative default below + + if (manual_gbps > 0.0f) { + prof.effective_bw_gbps = manual_gbps; + prof.source = "manual"; + prof.detail = "--pcie-bw-gbps override"; + return prof; + } + + if (autodetect) { + const std::string js = newest_probe_json(); + if (!js.empty() && profile_from_probe_json(js, prof)) return prof; + if (profile_from_nvidia_smi(prof)) return prof; + } + + // Conservative default: PCIe 3.0 x8 (common on Ryzen 5600G/5700G hosts where + // the iGPU absorbs 8 lanes — see rolling_kv.md). ~6.4 GB/s. + prof.link_width = 8; + prof.link_gtps = 8.0f; + prof.effective_bw_gbps = geometry_bw_gbps(8, 8.0f); + prof.source = "default"; + prof.detail = autodetect ? "no probe JSON / nvidia-smi; assuming PCIe3 x8" + : "autodetect disabled; assuming PCIe3 x8"; + return prof; +} + +const pcie_profile & pcie_profile_init(bool autodetect, float manual_gbps) { + g_pcie_profile = pcie_profile_detect(autodetect, manual_gbps); + g_pcie_inited = true; + return g_pcie_profile; +} + +const pcie_profile & pcie_profile_get() { + return g_pcie_profile; +} diff --git a/llama.cpp/common/pcie-profile.h b/llama.cpp/common/pcie-profile.h new file mode 100644 --- /dev/null +++ b/llama.cpp/common/pcie-profile.h @@ -0,0 +1,38 @@ +#pragma once + +// opencoti F5-opt W1 (#293) — PCIe link + host<->device bandwidth profile. +// +// Boot-time detection of the GPU's effective host-to-device transfer bandwidth +// and PCIe link geometry. This is a foundation diagnostic and, more +// importantly, the input M7 Rolling KV (#296) uses to size streaming KV tiles: +// tile_bytes_max = compute_ms_per_layer * effective_bw_gbps * 0.8 +// (see docs/features/rolling_kv.md, "Auto-adapt loop", step 1). +// +// Detection is CPU-side only (no CUDA dependency in common/). Sources, in +// preference order: +// 1. manual override (--pcie-bw-gbps F, F > 0) +// 2. explicit probe JSON via $OPENCOTI_PCIE_PROFILE +// 3. newest ./.opencoti/rebar-probe-*.json (measured pinned H2D + link) +// 4. one-shot `nvidia-smi` query (pcie.link.gen.current + width.current) +// 5. conservative default (PCIe 3.0 x8 ~= 6.4 GB/s) +// Never throws; never fails server boot. + +#include + +struct pcie_profile { + float effective_bw_gbps = 0.0f; // usable H2D bandwidth estimate (GB/s) + int link_width = 0; // current PCIe lane count (e.g. 8, 16) + float link_gtps = 0.0f; // current per-lane transfer rate (GT/s) + std::string source = "default"; // manual|probe-json|nvidia-smi|default + std::string detail; // human-readable note for logging +}; + +// Run detection once. manual_gbps > 0 forces source="manual"; autodetect=false +// skips probe/nvidia-smi discovery and returns the conservative default. +pcie_profile pcie_profile_detect(bool autodetect, float manual_gbps); + +// Process-global: pcie_profile_init() runs detection, caches, and returns the +// result; pcie_profile_get() returns the cached value (defaults until init +// runs). Consumed by M7 Rolling KV (#296) at server boot. +const pcie_profile & pcie_profile_init(bool autodetect, float manual_gbps); +const pcie_profile & pcie_profile_get(); diff --git a/llama.cpp/tools/server/server.cpp b/llama.cpp/tools/server/server.cpp --- a/llama.cpp/tools/server/server.cpp +++ b/llama.cpp/tools/server/server.cpp @@ -7,6 +7,7 @@ #include "arg.h" #include "build-info.h" #include "common.h" +#include "pcie-profile.h" // opencoti F5-opt W1 (#293) #include "fit.h" #include "llama.h" #include "log.h" @@ -135,6 +136,17 @@ int server_main(int argc, char ** argv, // struct that contains llama context and inference server_context ctx_server; + // opencoti F5-opt W1 (#293): detect host<->device bandwidth + PCIe link + // once at boot; logged here and cached (pcie_profile_get) for M7 Rolling KV + // (#296) streaming-tile sizing. Never fails boot — falls back to a + // conservative PCIe3 x8 default on any detection miss. + { + const pcie_profile & pp = pcie_profile_init(params.pcie_autodetect, params.pcie_bw_gbps); + LOG_INF("%s: pcie profile: %.1f GB/s effective (x%d @ %.1f GT/s, source=%s) — %s\n", + __func__, (double) pp.effective_bw_gbps, pp.link_width, (double) pp.link_gtps, + pp.source.c_str(), pp.detail.c_str()); + } + server_http_context ctx_http; if (!ctx_http.init(params)) { SRV_ERR("%s", "failed to initialize HTTP server\n");