#!/usr/bin/env python3
"""Compare zero and calibration-mean replacements of one Pythia attention head."""
import argparse
import hashlib
import json
from pathlib import Path

import torch
import torch.nn.functional as F
import transformers
from transformers import AutoModelForCausalLM
import induction_heads as induction


def validate(head, calibration, tests):
    if len(head) != 2 or any(type(x) is not int for x in head) or not 0 <= head[0] < 6 or not 0 <= head[1] < 8:
        raise ValueError('Vælg lag 0–5 og head 0–7.')
    seeds = [*calibration, *tests]
    if not 2 <= len(calibration) <= 10 or not 2 <= len(tests) <= 10 or any(type(s) is not int or not 0 <= s < 2**32 for s in seeds) or len(set(seeds)) != len(seeds):
        raise ValueError('Vælg 2–10 forskellige kalibreringsseeds og 2–10 andre testseeds.')


def head_input(model, tokens, head):
    layer, index = head
    width = model.config.hidden_size // model.config.num_attention_heads
    captured = []
    def observe(_module, inputs):
        captured.append(inputs[0][..., index * width:(index + 1) * width].detach().clone())
    handle = model.gpt_neox.layers[layer].attention.dense.register_forward_pre_hook(observe)
    try:
        with torch.inference_mode():
            model(torch.cat((tokens, tokens)).unsqueeze(0), use_cache=False)
    finally:
        handle.remove()
    if len(captured) != 1:
        raise RuntimeError('Forventede ét headoutput pr. følge.')
    return captured[0]


def measure(model, tokens, head, replacement=None):
    ids = torch.cat((tokens, tokens)).unsqueeze(0)
    layer, index = head
    width = model.config.hidden_size // model.config.num_attention_heads
    handle = None
    if replacement is not None:
        def replace(_module, inputs):
            combined = inputs[0].clone()
            combined[..., index * width:(index + 1) * width] = replacement.to(combined)
            return (combined, *inputs[1:])
        handle = model.gpt_neox.layers[layer].attention.dense.register_forward_pre_hook(replace)
    try:
        with torch.inference_mode():
            logits = model(ids, use_cache=False).logits
        losses = F.cross_entropy(logits[0, :-1], ids[0, 1:], reduction='none')
        return losses[induction.LENGTH:2 * induction.LENGTH - 1].tolist()
    finally:
        if handle is not None:
            handle.remove()


def experiment(model, head, calibration, tests):
    validate(head, calibration, tests)
    calibrations = [induction.sequence(seed) for seed in calibration]
    # One constant vector averaged over calibration sequences and all positions.
    means = torch.cat([head_input(model, tokens, head).reshape(-1, model.config.hidden_size // model.config.num_attention_heads) for tokens in calibrations]).mean(0)
    rows = []
    for seed in tests:
        tokens = induction.sequence(seed)
        # Replacing a head with its own original vector at every position is an identity control.
        identity = head_input(model, tokens, head)
        rows.append({'seed': seed, 'token_ids': tokens.tolist(),
                     'baseline': measure(model, tokens, head),
                     'identity': measure(model, tokens, head, identity),
                     'zero': measure(model, tokens, head, torch.zeros_like(means)),
                     'mean': measure(model, tokens, head, means)})
    return {'format': 'learning-lab-induction-replacements-v1', 'model': induction.MODEL,
            'revision': induction.REVISION, 'head': head,
            'calibration': [{'seed': seed, 'token_ids': tokens.tolist()} for seed, tokens in zip(calibration, calibrations)],
            'mean_vector': means.tolist(), 'rows': rows,
            'torch_version': str(torch.__version__), 'transformers_version': transformers.__version__,
            'script_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
            'sequence_sha256': hashlib.sha256(Path(induction.__file__).read_bytes()).hexdigest()}


def main():
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument('--head', type=int, nargs=2, default=[3, 6], metavar=('LAG', 'HEAD'))
    p.add_argument('--calibration-seeds', type=int, nargs='+', default=[101, 103, 107, 109, 113])
    p.add_argument('--test-seeds', type=int, nargs='+', default=[17, 31, 41, 53, 67])
    p.add_argument('--output', type=Path, default=Path('induction-replacements.json'))
    p.add_argument('--offline', action='store_true')
    args = p.parse_args()
    try:
        validate(args.head, args.calibration_seeds, args.test_seeds)
    except ValueError as exc:
        p.error(str(exc))
    if args.output.exists():
        p.error('Outputfilen findes allerede. Vælg et andet navn.')
    torch.set_num_threads(1)
    model = AutoModelForCausalLM.from_pretrained(induction.MODEL, revision=induction.REVISION,
        local_files_only=args.offline, attn_implementation='eager').eval()
    result = experiment(model, args.head, args.calibration_seeds, args.test_seeds)
    with args.output.open('x', encoding='utf-8') as out:
        json.dump(result, out, ensure_ascii=False, indent=2, allow_nan=False)
        out.write('\n')
    print(f"Sammenlignede nul og kalibreringsgennemsnit for {args.head}; gemt: {args.output}")


if __name__ == '__main__':
    main()
