#!/usr/bin/env python3
"""Run development first, then evaluate the same cases without training a model."""
import argparse
import hashlib
import json
from pathlib import Path

import torch
import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer
import safety_research as base


def sha(raw):
    return hashlib.sha256(raw).hexdigest()


def measure(prepared, model, tokenizer):
    rows=[]
    for case,target,encoded_pair in prepared:
        metrics=[]
        for encoded in encoded_pair:
            with torch.inference_mode():
                logits=model(**encoded,use_cache=False).logits[0,-1].detach()
            if logits.ndim!=1 or not torch.isfinite(logits).all() or not 0<=target<logits.numel():
                raise ValueError('Ugyldige logits eller svar-token.')
            probabilities=logits.softmax(-1);top=int(logits.argmax());prob=float(probabilities[target])
            if not torch.isfinite(probabilities).all():raise ValueError('Ugyldig sandsynlighed.')
            metrics.append({'token_count':int(encoded['input_ids'].shape[-1]),'top_token_id':top,'top_token':tokenizer.decode([top]),'top_is_answer':top==target,'answer_probability':prob,'answer_rank':1+int((logits>logits[target]).sum())})
        rows.append({**case,'answer_id':target,'a':metrics[0],'b':metrics[1]})
    return rows


def main():
    parser=argparse.ArgumentParser(description='Mål udvikling først og evaluering senere på samme fastlåste casefil.')
    parser.add_argument('cases',type=Path);parser.add_argument('--stage',required=True,choices=('development','evaluation'))
    parser.add_argument('--development',type=Path,help='Udviklingsresultatet, kræves ved evaluering')
    parser.add_argument('--output',type=Path,required=True);parser.add_argument('--model',choices=tuple(base.MODELS),default='pythia');parser.add_argument('--offline',action='store_true')
    args=parser.parse_args()
    if args.output.exists():parser.error('Outputfilen findes allerede.')
    if (args.stage=='evaluation')!=(args.development is not None):parser.error('Angiv --development kun ved evaluation.')
    try:
        raw=args.cases.read_bytes()
        if len(raw)>131072:raise ValueError('Casefilen må højst fylde 128 KB.')
        data=base.validate_cases(json.loads(raw.decode('utf-8')))
        model_name,revision=base.MODELS[args.model]
        metadata={'cases_sha256':sha(raw),'source_sha256':sha(Path(__file__).read_bytes()),'evaluation_source_sha256':sha(Path(base.__file__).read_bytes()),'model':model_name,'revision':revision,'torch_version':str(torch.__version__),'transformers_version':transformers.__version__}
        previous=None
        if args.development:
            if args.development.stat().st_size>131072:raise ValueError('Udviklingsresultatet er for stort.')
            previous=json.loads(args.development.read_text(encoding='utf-8'))
            if not isinstance(previous,dict) or previous.get('format')!='learning-lab-safety-staged-development-v1' or any(previous.get(k)!=v for k,v in metadata.items()):raise ValueError('Udviklingsresultatet passer ikke til denne casefil, model eller programversion.')
            expected=[c for c in data['cases'] if c['split']=='development']
            rows=previous.get('rows')
            if not isinstance(rows,list) or len(rows)!=len(expected) or any(any(r.get(k)!=v for k,v in c.items()) for r,c in zip(rows,expected)):raise ValueError('Udviklingsresultatets cases stemmer ikke med casefilen.')
        torch.set_num_threads(1)
        tokenizer=AutoTokenizer.from_pretrained(model_name,revision=revision,local_files_only=args.offline,trust_remote_code=False)
        ready=base.prepare(data,tokenizer)
        selected=[(case,target,[tokenizer(case[k],return_tensors='pt',add_special_tokens=False) for k in ('prompt_a','prompt_b')]) for case,target in ready if case['split']==args.stage]
        model=AutoModelForCausalLM.from_pretrained(model_name,revision=revision,local_files_only=args.offline,trust_remote_code=False,use_safetensors=True).eval()
        measured=measure(selected,model,tokenizer)
        if previous:
            result={'format':'learning-lab-safety-research-v1',**metadata,'hypothesis':data['hypothesis'],**({'threat_model':data['threat_model']} if 'threat_model' in data else {}),'rows':previous['rows']+measured}
        else:
            result={'format':'learning-lab-safety-staged-development-v1',**metadata,'rows':measured}
        with args.output.open('x',encoding='utf-8') as f:json.dump(result,f,ensure_ascii=False,indent=2,allow_nan=False);f.write('\n')
    except (OSError,ValueError,UnicodeError,RuntimeError,TypeError,AttributeError) as exc:
        raise SystemExit(f'Forsøget kunne ikke gennemføres: {exc}') from exc
    print(f"Målt {len(measured)} {args.stage}-cases. Gemt: {args.output}")

if __name__=='__main__':main()
