#!/usr/bin/env python3
"""Tiny causal transformer trained to copy the first of two symbols, using PyTorch."""
import argparse
import json
import math
from datetime import datetime, timezone
from pathlib import Path

try:
    import torch
    from torch import nn
    from torch.nn import functional as F
except ImportError as exc:
    raise SystemExit('PyTorch mangler. Følg Python & PyTorch-vejledningen i Learning Lab.') from exc

VOCAB = 6
QUERY = 6
DIM = 24


class TinyTransformer(nn.Module):
    def __init__(self):
        super().__init__()
        self.embedding = nn.Embedding(VOCAB + 1, DIM)
        self.position = nn.Embedding(3, DIM)
        self.qkv = nn.Linear(DIM, DIM * 3, bias=False)
        self.proj = nn.Linear(DIM, DIM)
        self.norm1 = nn.LayerNorm(DIM)
        self.norm2 = nn.LayerNorm(DIM)
        self.mlp = nn.Sequential(nn.Linear(DIM, DIM * 2), nn.GELU(), nn.Linear(DIM * 2, DIM))
        self.output = nn.Linear(DIM, VOCAB)

    def forward(self, tokens):
        length = tokens.shape[1]
        x = self.embedding(tokens) + self.position(torch.arange(length, device=tokens.device))
        q, k, v = self.qkv(self.norm1(x)).chunk(3, dim=-1)
        scores = q @ k.transpose(-2, -1) / math.sqrt(DIM)
        future = torch.ones((length, length), dtype=torch.bool, device=tokens.device).triu(1)
        weights = scores.masked_fill(future, float('-inf')).softmax(dim=-1)
        x = x + self.proj(weights @ v)
        x = x + self.mlp(self.norm2(x))
        return self.output(x[:, -1, :]), weights


def data():
    train, validation = [], []
    for first in range(VOCAB):
        for second in range(VOCAB):
            # This deliberately holds out combinations, not individual symbols.
            (validation if (first + second) % 4 == 0 else train).append((first, second))
    def tensors(pairs):
        x = torch.tensor([[a, b, QUERY] for a, b in pairs], dtype=torch.long)
        y = torch.tensor([a for a, _ in pairs], dtype=torch.long)
        return x, y
    return tensors(train), tensors(validation)


def evaluate(model, split):
    model.eval()
    with torch.no_grad():
        logits, _ = model(split[0]); loss = F.cross_entropy(logits, split[1]).item()
        acc = (logits.argmax(dim=-1) == split[1]).float().mean().item()
    return loss, acc


def main():
    parser = argparse.ArgumentParser(description='Træn en lille kausal transformer til kopiering.')
    parser.add_argument('--steps', type=int, default=400)
    parser.add_argument('--seed', type=int, default=7)
    parser.add_argument('--first', type=int, default=0)
    parser.add_argument('--second', type=int, default=0)
    parser.add_argument('--output-dir', type=Path)
    args = parser.parse_args()
    if not 1 <= args.steps <= 10000 or not 0 <= args.seed < 2**32 or not 0 <= args.first < VOCAB or not 0 <= args.second < VOCAB:
        parser.error('Vælg 1–10000 skridt, et seed fra 0 til 2³²−1 og symboler fra 0 til 5.')
    output = args.output_dir or Path('transformer-runs') / datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S_%fZ')
    try: output.mkdir(parents=True, exist_ok=False)
    except FileExistsError: parser.error(f'Outputmappen findes allerede: {output}')
    torch.set_num_threads(1); torch.manual_seed(args.seed)
    model = TinyTransformer(); train, validation = data()
    optimizer = torch.optim.AdamW(model.parameters(), lr=.01)
    for step in range(args.steps + 1):
        if step % 50 == 0 or step == args.steps:
            train_loss, train_acc = evaluate(model, train); val_loss, val_acc = evaluate(model, validation)
            print(f'{step:4d} | tab træning {train_loss:.3f}, validering {val_loss:.3f} | nøjagtighed {train_acc:.1%} / {val_acc:.1%}')
        if step == args.steps: break
        model.train(); optimizer.zero_grad(); logits, _ = model(train[0]); loss = F.cross_entropy(logits, train[1]); loss.backward(); optimizer.step()
    model.eval(); tokens = torch.tensor([[args.first, args.second, QUERY]], dtype=torch.long)
    with torch.no_grad(): logits, weights = model(tokens); probabilities = logits.softmax(-1)[0]
    trace = {'format':'learning-lab-attention-v1','tokens':[str(args.first),str(args.second),'?'], 'attention':weights[0].tolist(), 'probabilities':probabilities.tolist(), 'prediction':int(probabilities.argmax()), 'target':args.first, 'steps':args.steps,'seed':args.seed, 'validation_accuracy':val_acc, 'torch_version':str(torch.__version__), 'split':'validering' if (args.first+args.second)%4==0 else 'træning'}
    (output / 'attention.json').write_text(json.dumps(trace,ensure_ascii=False,indent=2),encoding='utf-8')
    torch.save(model.state_dict(),output/'model.pt')
    print(f'Kopier første symbol: {args.first}, {args.second}, ? → model: {trace["prediction"]}. Eksemplet tilhører {trace["split"]}.')
    print(f'Gemt i {output}: attention.json og model.pt. Åbn attention.json i Learning Lab.')


if __name__ == '__main__': main()
