Enable native 8192-token Julia-1 inference context

README.md

@@ -63,7 +63,7 @@ Julia 1 starts from [JHU CLSP's mmBERT-small](https://huggingface.co/jhu-clsp/mm
 | --- | --- | --- |
 | Main interface | Encoder representations / masked-token modeling | `state` + `question` + 2–20 `options` → one selected option |
 | Parameters | About 140M | 144.3M including the decision components |
-| Context | Upstream architecture supports up to 8,192 tokens | Evaluated Julia decision path uses up to 1,024 tokens |
+| Context | Upstream architecture supports up to 8,192 tokens | Runtime supports up to 8,192 combined tokens; historical benchmarks used 1,024 |
 | Output | Token or encoder features for a downstream task | Scores and a selected answer in the caller's option order |
 | Usage | General multilingual encoder foundation | Specialized finite-choice decisions |
 
@@ -92,7 +92,7 @@ engine = load_model(
     "Julia-1",
     device="cpu",
     strict_encoding=True,
-    max_length=1024,
+    max_length=8192,
     head_length=512,
 )
 
@@ -121,7 +121,7 @@ print(result["probabilities"])
 | `options` | 2–20 nonempty answer descriptions, in the order you want returned. |
 | `type` | `choice` (default), `score` for ordered options, or `noul` for Boolean decisions. |
 
-For `noul`, provide exactly two options: **false first, true second**. For `score`, supply options in their intended order. Strict encoding rejects inputs that exceed the model's limits instead of silently truncating them. The evaluated configuration uses a 1,024-token total sequence and a 512-token question-and-options budget; each option may use at most 48 tokens.
+For `noul`, provide exactly two options: **false first, true second**. For `score`, supply options in their intended order. Strict encoding rejects inputs that exceed the model's limits instead of silently truncating them. The runtime defaults to the checkpoint’s native **8,192-token** total sequence limit, including state, question and options. Historical benchmarks above used 1,024 tokens; An [exact 8,192-token CPU inference smoke test](metrics/context-8k-smoke.json) passed with finite outputs; 8k task accuracy has not been established. The example keeps a 512-token question-and-options budget; each option may use at most 48 tokens.
 
 The returned percentages are **display values, not calibrated certainty**. If the raw top probability exceeds 95% and every other option is below 4.5%, the result displays 100% for the winner. Values below 1% display as 0%, with their mass redistributed proportionally among the remaining options. The selected index still comes from raw logits. For analysis or your own calibration, use `logits()`.
 

inference-policy.json

@@ -1,10 +1,11 @@
 {
   "single_model": true,
-  "max_length": 1024,
+  "max_length": 8192,
   "head_length": 512,
   "strict_encoding": true,
   "calibration": null,
   "replacement_qualified": false,
   "step": 500,
-  "weights_sha256": "df853bf7fe424420011f3d0c47a05d7341aa9eefa7fb9f203ea4aada4ad95b72"
+  "weights_sha256": "df853bf7fe424420011f3d0c47a05d7341aa9eefa7fb9f203ea4aada4ad95b72",
+  "context_basis": "Native mmBERT positional limit; long-context task accuracy not established"
 }

julia/data.py

@@ -69,7 +69,7 @@ class Decisions(Dataset):
         return self.rows[index]
 
 
-def sequence(tokenizer, row, max_length=1024, head_length=256, *, strict=False):
+def sequence(tokenizer, row, max_length=8192, head_length=256, *, strict=False):
     if head_length + 4 >= max_length:
         raise ValueError('max_length must leave room beyond the question head')
     if any(x is None for x in (tokenizer.mask_token_id, tokenizer.cls_token_id, tokenizer.sep_token_id)):
@@ -111,7 +111,7 @@ def sequence(tokenizer, row, max_length=1024, head_length=256, *, strict=False):
 
 
 class Collator:
-    def __init__(self, tokenizer, max_length=512, head_length=256):
+    def __init__(self, tokenizer, max_length=8192, head_length=256):
         self.tokenizer, self.max_length, self.head_length = tokenizer, max_length, head_length
 
     def __call__(self, rows, *, include_targets=True):

julia/inference.py

@@ -8,12 +8,22 @@ from .data import Collator, validate_row
 from .probabilities import display_probabilities
 
 
+def context_length(checkpoint, requested):
+    config = json.loads((Path(checkpoint) / 'encoder/config.json').read_text())
+    limit = config['max_position_embeddings']
+    value = limit if requested is None else requested
+    if type(value) is not int or not 1 <= value <= limit:
+        raise ValueError(f'max_length must be an integer between 1 and {limit}')
+    return value
+
+
 class TransformerEngine:
-    def __init__(self, checkpoint, device='cuda', max_length=1024, head_length=256, *, memory_map=True):
+    def __init__(self, checkpoint, device='cuda', max_length=None, head_length=256, *, memory_map=True):
         from transformers import AutoModel, AutoTokenizer
         from .model import JuliaDecisionModel
         self.device = configure(device)
         root = Path(checkpoint)
+        max_length = context_length(root, max_length)
         if (root / 'INCOMPLETE').exists():
             raise ValueError('Refusing to load an incomplete INT8 export')
         self.tokenizer = AutoTokenizer.from_pretrained(root / 'tokenizer', trust_remote_code=False)
@@ -60,7 +70,7 @@ class TransformerEngine:
         return result
 
 
-def load_model(checkpoint, device='cpu', max_length=1024, head_length=256, *, backend=None, **kwargs):
+def load_model(checkpoint, device='cpu', max_length=None, head_length=256, *, backend=None, **kwargs):
     """Load a Julia checkpoint through the supported resident inference runtime."""
     from .router.engine import FastEngine
     if checkpoint is None:

julia/router/engine.py

@@ -38,7 +38,7 @@ class FastEngine(Engine):
     Optional torch.compile specializes the transformer, while Bend handles CPU
     softmax/selection via ctypes. CUDA softmax stays on-device to avoid a roundtrip.
     """
-    def __init__(self, checkpoint, device='cpu', max_length=1024, head_length=256,
+    def __init__(self, checkpoint, device='cpu', max_length=None, head_length=256,
                  batch_size=16, encoding_cache=2048, token_cache=8192,
                  compile_model=False, library=None, bend_postprocess=False, transformer_backend=None,
                  strict_encoding=False, marker_only_head=None, memory_map=True, padding_ratio=1.25):
@@ -60,6 +60,7 @@ class FastEngine(Engine):
         self.model.marker_only_head = (self.device.type == 'cpu' if marker_only_head is None else marker_only_head)
         self.strict_encoding = strict_encoding
         self.batch_size = batch_size
+        max_length = self.collate.max_length
         self.max_length, self.head_length = max_length, head_length
         self.encoding_cache = encoding_cache
         self._encoded = OrderedDict()

metrics/context-8k-smoke.json

@@ -0,0 +1,8 @@
+{
+  "tokens": 8192,
+  "device": "cpu",
+  "elapsed_seconds": 24.356244013994,
+  "finite_logits": true,
+  "max_length": 8192,
+  "scope": "Runtime execution smoke; not long-context accuracy evaluation"
+}

tests/test_context.py

@@ -0,0 +1,21 @@
+import unittest
+from pathlib import Path
+from julia.inference import context_length
+from julia.data import sequence
+
+class Tokenizer:
+    mask_token='[MASK]';mask_token_id=4;cls_token_id=1;sep_token_id=2
+    def __call__(self,text,**kwargs):return {'input_ids':[5]*len(text.split())}
+
+class ContextTests(unittest.TestCase):
+    def test_default_uses_native_checkpoint_limit(self):
+        self.assertEqual(context_length(Path(__file__).resolve().parents[1],None),8192)
+    def test_rejects_beyond_native_limit(self):
+        with self.assertRaises(ValueError):context_length(Path(__file__).resolve().parents[1],8193)
+    def test_full_budget_and_strict_overflow(self):
+        row=dict(state='',question='choose',options=['a','b'])
+        overhead=len(sequence(Tokenizer(),row,strict=True)['ids'])
+        row['state']=' '.join(['x']*(8192-overhead))
+        self.assertEqual(len(sequence(Tokenizer(),row,strict=True)['ids']),8192)
+        row['state']+=' x'
+        with self.assertRaises(ValueError):sequence(Tokenizer(),row,strict=True)