#!/usr/bin/env python3
"""Measure every ordering of four fixed demonstrations on a pinned model."""
import argparse
import hashlib
import itertools
import json
from pathlib import Path

import torch
import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer
from fewshot_lookup import MODEL, REVISION, KEYS, ANSWERS, MAP_A, MAP_B, answer_ids


def ordered_prompt(mapping, query, ordering):
    if query not in KEYS or sorted(ordering) != list(KEYS) or tuple(mapping) not in (MAP_A, MAP_B):
        raise ValueError('Ugyldigt spørgsmål, mapping eller rækkefølge.')
    lookup = dict(zip(KEYS, mapping))
    return ''.join(f'{key} -> {lookup[key]}\n' for key in ordering) + f'{query} ->'


def run(model, tokenizer):
    results = []
    for ordering in itertools.permutations(KEYS):
        cases = []
        for i, key in enumerate(KEYS):
            pair = []
            for mapping in (MAP_A, MAP_B):
                text = ordered_prompt(mapping, key, ordering)
                one, two = answer_ids(tokenizer, text)
                inputs = tokenizer(text, return_tensors='pt', add_special_tokens=False)
                with torch.no_grad():
                    logits = model(**inputs, use_cache=False).logits[0, -1]
                    probabilities = logits.softmax(-1)
                pair.append({'shown_answer': mapping[i], 'p_one': float(probabilities[one]),
                             'p_two': float(probabilities[two]), 'logit_gap': float(logits[one] - logits[two]),
                             'top_token': tokenizer.decode([int(logits.argmax())])})
            cases.append({'query': key, 'mapping_a': pair[0], 'mapping_b': pair[1]})
        results.append({'order': ''.join(ordering), 'cases': cases})
    files = [Path(__file__), Path(__file__).with_name('fewshot_lookup.py')]
    return {'format': 'learning-lab-fewshot-order-sweep-v1', 'model': MODEL, 'revision': REVISION,
            'torch_version': str(torch.__version__), 'transformers_version': transformers.__version__,
            'source_sha256': {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in files},
            'answer_tokens': list(ANSWERS), 'orders': results}


def main():
    parser = argparse.ArgumentParser(description='Mål alle 24 rækkefølger af fire faste demonstrationer.')
    parser.add_argument('--offline', action='store_true')
    parser.add_argument('--output', type=Path, default=Path('fewshot-order-sweep.json'))
    args = parser.parse_args()
    if args.output.exists():
        parser.error('Outputfilen findes allerede; vælg et nyt filnavn.')
    torch.set_num_threads(1)
    try:
        tokenizer = AutoTokenizer.from_pretrained(MODEL, revision=REVISION, local_files_only=args.offline, trust_remote_code=False)
        model = AutoModelForCausalLM.from_pretrained(MODEL, revision=REVISION, local_files_only=args.offline, trust_remote_code=False, use_safetensors=True)
        model.eval()
        result = run(model, tokenizer)
    except (OSError, ValueError, RuntimeError) as exc:
        raise SystemExit(f'Forsøget kunne ikke gennemføres: {exc}') from exc
    try:
        with args.output.open('x', encoding='utf-8') as output:
            json.dump(result, output, ensure_ascii=False, indent=2)
    except FileExistsError:
        parser.error('Outputfilen findes allerede; vælg et nyt filnavn.')
    print(f"Gemt {len(result['orders'])} rækkefølger i {args.output}.")


if __name__ == '__main__':
    main()
