#!/usr/bin/env python3
"""Compare next-token scores for two fixed prompt templates on one checkpoint."""
import argparse
import csv
import json
from datetime import datetime, timezone
from pathlib import Path

import torch
import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer

MODEL = 'EleutherAI/pythia-70m'
REVISION = 'a39f36b100fe8a5377810d56c3f4789b9c53ac42'
EXAMPLES = [('France','Paris'),('Italy','Rome'),('Germany','Berlin'),('Spain','Madrid'),('Japan','Tokyo'),('China','Beijing')]
TEMPLATES = [('A','The capital of {country} is'),('B','The capital city of {country} is')]


def score(logits, target_id, tokenizer):
    if logits.ndim != 1 or not torch.isfinite(logits).all():
        raise ValueError('Modellens logits er ikke en endelig vektor.')
    if not 0 <= target_id < logits.numel():
        raise ValueError('Svar-token ligger uden for modellens ordforråd.')
    p = logits.softmax(-1)
    top = int(logits.argmax())
    return {'answer_probability':float(p[target_id]),
            'answer_rank':1+int((logits>logits[target_id]).sum()),
            'top_token':tokenizer.decode([top]), 'top_token_id':top, 'top_is_answer':top==target_id}


def score_choices(logits, candidates, target_id):
    ids=[c['token_id'] for c in candidates]
    if len(set(ids))!=len(ids) or target_id not in ids or any(i<0 or i>=logits.numel() for i in ids):
        raise ValueError('Svarmulighederne skal være forskellige tokens i ordforrådet.')
    probabilities=logits[ids].softmax(-1)
    choice=int(probabilities.argmax())
    return {'choice_token_id':ids[choice], 'choice_token':candidates[choice]['text'],
            'choice_is_answer':ids[choice]==target_id,
            'choice_probability':float(probabilities[ids.index(target_id)]),
            'choice_distribution':[float(x) for x in probabilities]}


def prepare_inputs(tokenizer):
    candidates=[]
    for _, capital in EXAMPLES:
        answer=' '+capital
        ids=tokenizer(answer,add_special_tokens=False).input_ids
        if len(ids)!=1:
            raise ValueError(f'Svaret {answer!r} er ikke præcis ét token.')
        candidates.append({'text':answer,'token_id':int(ids[0])})
    if len({c['token_id'] for c in candidates})!=len(candidates):
        raise ValueError('Svarmulighederne skal være forskellige tokens.')
    prepared=[]
    for country, capital in EXAMPLES:
        variants=[]
        for name, template in TEMPLATES:
            prompt=template.format(country=country)
            inputs=tokenizer(prompt,return_tensors='pt',add_special_tokens=False)
            prompt_ids=inputs['input_ids'][0].tolist()
            for candidate in candidates:
                full_ids=tokenizer(prompt+candidate['text'],add_special_tokens=False).input_ids
                if full_ids!=prompt_ids+[candidate['token_id']]:
                    raise ValueError(f'Svaret {candidate["text"]!r} står ikke som et selvstændigt næste token efter {prompt!r}.')
            variants.append((name,prompt,inputs))
        prepared.append((country,' '+capital,variants))
    return candidates,prepared


def evaluate(model, tokenizer):
    candidates,prepared=prepare_inputs(tokenizer)
    answer_ids={c['text']:c['token_id'] for c in candidates}
    rows=[]
    for country, answer, inputs_by_variant in prepared:
        target_id=answer_ids[answer]
        variants=[]
        for name, prompt, inputs in inputs_by_variant:
            with torch.no_grad():logits=model(**inputs,use_cache=False).logits[0,-1].detach()
            variants.append({'template':name,'prompt':prompt,'token_count':int(inputs['input_ids'].shape[-1]),**score(logits,target_id,tokenizer),**score_choices(logits,candidates,target_id)})
        rows.append({'country':country,'answer':answer,'answer_id':target_id,'variants':variants})
    return {'format':'learning-lab-prompt-eval-v2','model':MODEL,'revision':REVISION,
            'torch_version':str(torch.__version__),'transformers_version':transformers.__version__,
            'boundary_checked':True,
            'templates':[{'name':name,'text':text} for name,text in TEMPLATES], 'candidates':candidates, 'rows':rows}


def main():
    parser=argparse.ArgumentParser(description='Sammenlign to promptformuleringer på Pythia-70M.')
    parser.add_argument('--offline',action='store_true',help='Brug kun allerede hentede modeldata.')
    parser.add_argument('--output-dir',type=Path)
    args=parser.parse_args()
    if args.output_dir is not None and args.output_dir.exists():parser.error(f'Outputmappen findes allerede: {args.output_dir}')
    torch.set_num_threads(1)
    try:
        tokenizer=AutoTokenizer.from_pretrained(MODEL,revision=REVISION,local_files_only=args.offline,trust_remote_code=False)
        prepare_inputs(tokenizer)
        model=AutoModelForCausalLM.from_pretrained(MODEL,revision=REVISION,local_files_only=args.offline,trust_remote_code=False,use_safetensors=True).eval()
        result=evaluate(model,tokenizer)
    except (OSError,ValueError,RuntimeError) as exc:raise SystemExit(f'Evalueringen kunne ikke gennemføres: {exc}') from exc
    output=args.output_dir or Path('prompt-eval-runs')/datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S_%fZ')
    try:output.mkdir(parents=True,exist_ok=False)
    except FileExistsError:parser.error(f'Outputmappen findes allerede: {output}')
    (output/'prompt_eval.json').write_text(json.dumps(result,ensure_ascii=False,indent=2,allow_nan=False),encoding='utf-8')
    with (output/'examples.csv').open('w',encoding='utf-8',newline='') as f:
        writer=csv.DictWriter(f,fieldnames=['country','answer','answer_id','template','prompt','token_count','answer_probability','answer_rank','top_token','top_token_id','top_is_answer','choice_token_id','choice_token','choice_is_answer','choice_probability','choice_distribution'])
        writer.writeheader()
        for row in result['rows']:
            for variant in row['variants']:writer.writerow({'country':row['country'],'answer':row['answer'],'answer_id':row['answer_id'],**variant})
    print(f"{len(result['rows'])} spørgsmål, {len(TEMPLATES)} formuleringer. Gemt i {output}: prompt_eval.json og examples.csv.")

if __name__=='__main__':main()
