Overlay: build the n-gram table parameter on the meta device when VLLM_PLE_QUANT_DIR is set, and swap the stub Parameter instead of set_data — removes the 102 GB virtual reservation that the kernel's overcommit heuristic refuses on hosts with less RAM+swap than the table (field report, 64 GB host); validated under an emulated 67/99 GiB commit limit, sanity PASS, tool-calling 77.0 (n=3), 81 tok/s c1

worker_image_quant.py

@@ -436,9 +436,16 @@ def _ple_quant_attach(layer_name: str, layer: torch.nn.Module,
     for p in parts[:-1]:
         owner = getattr(owner, p)
     owner._ple_quant = _PleQuantTable(quant_dir, rows, width)
-    # Stub before anything writes the parameter: the original 95 GB allocation
-    # is lazy virtual memory and stays unmaterialized.
-    getattr(owner, parts[-1]).data = torch.empty(0, width, dtype=param.dtype)
+    # Stub before anything writes the parameter. Replace the Parameter object rather
+    # than assigning .data: with a quantized sidecar the table is built on the meta
+    # device (no 95 GB virtual reservation for the kernel's overcommit heuristic to
+    # refuse), and set_data() rejects a meta -> cpu swap. vLLM's weight attributes
+    # (weight_loader, output_dim, ...) are carried over to the stub.
+    old = getattr(owner, parts[-1])
+    stub = torch.nn.Parameter(torch.empty(0, width, dtype=param.dtype), requires_grad=False)
+    for k, v in vars(old).items():
+        setattr(stub, k, v)
+    setattr(owner, parts[-1], stub)
     logger.info("PLE quant: %s.%s stubbed, gathers served from sidecar.",
                 layer_name, pname)
     return pname