#!/usr/bin/env python3
"""Inspect a repeated-token pattern in a fixed Pythia-70M checkpoint."""
import argparse
import hashlib
import json
from pathlib import Path

import torch
import torch.nn.functional as F
import transformers
from transformers import AutoModelForCausalLM

MODEL = 'EleutherAI/pythia-70m'
REVISION = 'a39f36b100fe8a5377810d56c3f4789b9c53ac42'
LENGTH = 32


def sequence(seed):
    generator = torch.Generator(device='cpu').manual_seed(seed)
    # Fixed, distinct ordinary vocabulary IDs. No tokenizer or special tokens are added.
    return torch.randperm(47000, generator=generator)[:LENGTH] + 1000


def measure(model, tokens, ablate=None, attention=False):
    ids = torch.cat((tokens, tokens)).unsqueeze(0)
    handle = None
    if ablate is not None:
        layer, head = ablate
        attention_module = model.gpt_neox.layers[layer].attention
        width = model.config.hidden_size // model.config.num_attention_heads

        def remove_head(_module, inputs):
            combined = inputs[0].clone()
            combined[..., head * width:(head + 1) * width] = 0
            return (combined, *inputs[1:])

        # The dense projection receives concatenated heads; zero one head before projection.
        handle = attention_module.dense.register_forward_pre_hook(remove_head)
    try:
        with torch.no_grad():
            result = model(ids, output_attentions=attention, use_cache=False)
    finally:
        if handle is not None:
            handle.remove()
    losses = F.cross_entropy(result.logits[0, :-1], ids[0, 1:], reduction='none')
    output = {'first_loss': losses[:LENGTH - 1].tolist(),
              'repeated_loss': losses[LENGTH:2 * LENGTH - 1].tolist()}
    if attention:
        matrices = torch.stack(result.attentions)[:, 0]
        j = torch.arange(LENGTH - 1)
        match = matrices[:, :, LENGTH + j, j + 1]
        foil = matrices[:, :, LENGTH + j, (j + 2) % LENGTH]
        output.update(scores=match.mean(-1).tolist(), foil_scores=foil.mean(-1).tolist(),
                      match=match.tolist(), foil=foil.tolist())
    return output


def make_result(model, discover_seed, test_seed):
    discover_tokens, test_tokens = sequence(discover_seed), sequence(test_seed)
    discover = measure(model, discover_tokens, attention=True)
    test = measure(model, test_tokens, attention=True)
    scores = torch.tensor(discover['scores'])
    flat_index = int(scores.flatten().argmax())
    heads = model.config.num_attention_heads
    selected = [flat_index // heads, flat_index % heads]
    control = [0, 0] if selected != [0, 0] else [0, 1]
    selected_ablation = measure(model, test_tokens, ablate=selected)
    control_ablation = measure(model, test_tokens, ablate=control)
    layer, head = selected
    return {
        'format': 'learning-lab-induction-v1', 'model': MODEL, 'revision': REVISION,
        'torch_version': str(torch.__version__), 'transformers_version': transformers.__version__,
        'script_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), 'length': LENGTH,
        'discover_seed': discover_seed, 'test_seed': test_seed,
        'selected': selected, 'control': control,
        'discover': {'token_ids': discover_tokens.tolist(), 'scores': discover['scores'],
                     'foil_scores': discover['foil_scores']},
        'test': {'token_ids': test_tokens.tolist(), 'scores': test['scores'],
                 'foil_scores': test['foil_scores'],
                 'match': test['match'][layer][head], 'foil': test['foil'][layer][head],
                 'first_loss': test['first_loss'], 'repeated_loss': test['repeated_loss'],
                 'selected_ablation_loss': selected_ablation['repeated_loss'],
                 'control_ablation_loss': control_ablation['repeated_loss']}
    }


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--discover-seed', type=int, default=7)
    parser.add_argument('--test-seed', type=int, default=17)
    parser.add_argument('--output', type=Path, default=Path('induction-result.json'))
    parser.add_argument('--offline', action='store_true')
    args = parser.parse_args()
    if any(not 0 <= seed < 2**32 for seed in (args.discover_seed, args.test_seed)):
        parser.error('Seeds skal være heltal fra 0 til 2³²−1.')
    if args.discover_seed == args.test_seed:
        parser.error('Brug forskellige seeds til udvælgelse og test.')
    if args.output.exists():
        parser.error('Outputfilen findes allerede. Vælg et andet navn.')
    torch.set_num_threads(1)
    model = AutoModelForCausalLM.from_pretrained(MODEL, revision=REVISION,
        local_files_only=args.offline, attn_implementation='eager').eval()
    result = make_result(model, args.discover_seed, args.test_seed)
    # Exclusive creation also protects a file created while the model was running.
    with args.output.open('x', encoding='utf-8') as file:
        json.dump(result, file, ensure_ascii=False, indent=2, allow_nan=False)
        file.write('\n')
    print(f"Valgt lag {result['selected'][0]}, head {result['selected'][1]}; gemt: {args.output}")


if __name__ == '__main__':
    main()
