#!/usr/bin/env python3
"""Compare two demonstration mappings on a fixed Pythia checkpoint."""
import argparse
import hashlib
import json
from pathlib import Path

import torch
import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer

MODEL = 'EleutherAI/pythia-70m'
REVISION = 'a39f36b100fe8a5377810d56c3f4789b9c53ac42'
KEYS = ('A', 'B', 'C', 'D')
ANSWERS = (' 1', ' 2')
MAP_A = ('1', '2', '1', '2')
MAP_B = ('2', '1', '2', '1')


def prompt(mapping, query, order='forward'):
    if tuple(mapping) not in (MAP_A, MAP_B) or query not in KEYS:
        raise ValueError('Ukendt demonstration eller nøgle.')
    if order not in ('forward', 'reverse'):
        raise ValueError('Ukendt rækkefølge.')
    pairs = list(zip(KEYS, mapping))
    if order == 'reverse':
        pairs.reverse()
    return ''.join(f'{key} -> {value}\n' for key, value in pairs) + f'{query} ->'


def answer_ids(tokenizer, text):
    prefix = tokenizer(text, add_special_tokens=False).input_ids
    ids = []
    for answer in ANSWERS:
        token = tokenizer(answer, add_special_tokens=False).input_ids
        combined = tokenizer(text + answer, add_special_tokens=False).input_ids
        if len(token) != 1 or combined != prefix + token:
            raise ValueError('Svarene skal hver være ét selvstændigt token efter prompten.')
        ids.append(token[0])
    if ids[0] == ids[1]:
        raise ValueError('Svarene skal have forskellige token-IDer.')
    return ids


def run(model, tokenizer, order):
    cases = []
    for i, key in enumerate(KEYS):
        pair = []
        for mapping in (MAP_A, MAP_B):
            text = prompt(mapping, key, order)
            first, second = answer_ids(tokenizer, text)
            inputs = tokenizer(text, return_tensors='pt', add_special_tokens=False)
            with torch.no_grad():
                logits = model(**inputs, use_cache=False).logits[0, -1]
                probs = logits.softmax(-1)
            pair.append({'prompt': text, 'shown_answer': mapping[i],
                         'p_one': float(probs[first]), 'p_two': float(probs[second]),
                         'logit_gap': float(logits[first] - logits[second]),
                         'top_token': tokenizer.decode([int(logits.argmax())])})
        cases.append({'query': key, 'mapping_a': pair[0], 'mapping_b': pair[1]})
    return {'format': 'learning-lab-fewshot-lookup-v1', 'model': MODEL, 'revision': REVISION,
            'torch_version': str(torch.__version__), 'transformers_version': transformers.__version__,
            'script_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
            'order': order, 'answer_tokens': list(ANSWERS), 'cases': cases}


def main():
    parser = argparse.ArgumentParser(description='Undersøg næste-token-fordelingen ved to demonstrationssæt.')
    parser.add_argument('--offline', action='store_true')
    parser.add_argument('--order', choices=('forward', 'reverse'), default='forward')
    parser.add_argument('--output', type=Path, default=Path('fewshot-lookup.json'))
    args = parser.parse_args()
    if args.output.exists():
        parser.error('Outputfilen findes allerede; vælg et nyt filnavn.')
    torch.set_num_threads(1)
    try:
        tokenizer = AutoTokenizer.from_pretrained(MODEL, revision=REVISION, local_files_only=args.offline, trust_remote_code=False)
        model = AutoModelForCausalLM.from_pretrained(MODEL, revision=REVISION, local_files_only=args.offline, trust_remote_code=False, use_safetensors=True)
        model.eval()
        result = run(model, tokenizer, args.order)
    except (OSError, ValueError, RuntimeError) as exc:
        raise SystemExit(f'Forsøget kunne ikke gennemføres: {exc}') from exc
    try:
        with args.output.open('x', encoding='utf-8') as output:
            json.dump(result, output, ensure_ascii=False, indent=2)
    except FileExistsError:
        parser.error('Outputfilen findes allerede; vælg et nyt filnavn.')
    print(f"Gemt {len(result['cases'])} parrede målinger i {args.output}.")


if __name__ == '__main__':
    main()
