"""Oracle BIO logits prove window mechanics independently of trained predictions.""" import json import sys from pathlib import Path import numpy as np from transformers import AutoTokenizer sys.path.insert(0,str(Path(__file__).resolve().parents[1])) from pii.core import encode_windows, LABELS, byte_positions, decode_window, result root=Path(__file__).resolve().parents[1] tokenizer=AutoTokenizer.from_pretrained(root/'artifacts/backbone',local_files_only=True) max_length,stride=512,42 cases=[] def make_case(name,prefix,entities,suffix): text=prefix; spans=[] for label,value in entities: start=len(text);text+=value;spans.append({'label':label,'start':start,'end':len(text)});text+=' ' text+=suffix bp=byte_positions(text) expected=[{'label':s['label'],'start':bp[s['start']],'end':bp[s['end']]} for s in spans] encoded=encode_windows(tokenizer,text,max_length,stride) raw=tokenizer(text,add_special_tokens=False,truncation=False,return_offsets_mapping=True,verbose=False) covered=[] for wi,ids in enumerate(encoded['input_ids']):covered.extend(ids[1 if wi==0 else stride+1:-1]) assert covered==raw['input_ids'], 'manual windows lost or duplicated tokens outside overlap' windows=[]; findings=[]; covers=[0]*len(spans); clips=0 for wi,offsets in enumerate(encoded['offset_mapping']): content=[(a,b) for a,b in offsets if a!=b] complete=[content[0][0]<=s['start'] and content[-1][1]>=s['end'] for s in spans] covers[:]=[v+int(c) for v,c in zip(covers,complete)] logits=np.full((len(offsets),15),-8,dtype=np.float32) for i,(a,b) in enumerate(offsets): label_id=0; peak=12. if a!=b: for si,s in enumerate(spans): if a0,wi+1