Train a causal byte language model

// Native document-preserving byte language-model research workflow.
// Use new checkpoint/tokenizer paths for each experiment.
import aner.token
import aner.gen
import aner.dev

let codec = token.byte_tokenizer()
let training = token.corpus(codec, List(
    "abababababababab", "babababababababa"), role: "training")
let validation = token.corpus(codec, List(
    "abababababababa", "bababababababab"), role: "validation")
let initial = gen.decoder(codec, context: 24, width: 8,
    heads: 2, layers: 1, seed: 63, dropout: 0.0)
print(gen.baseline())
print("Initial validation loss")
print(initial.loss(validation))
let partial = initial.train(training, steps: 90, batch: 2, rate: 0.01, seed: 99,
    clip_norm: 1.0, warmup_steps: 0, weight_decay: 0.0)
codec.save("byte.anertoken")
partial.save("byte-partial.anergen")
let restored = gen.load("byte-partial.anergen")
let model = restored.resume(training, steps: 90)
print(model.summary())
print("Training loss")
print(model.loss(training))
print("Validation loss")
print(model.loss(validation))
print("Validation perplexity")
print(model.perplexity(validation))
dev.capture("Native language model", model)
let prompt = codec.encode("ab", bos: true, eos: false)
let continuation = model.generate(prompt, max_new_tokens: 8,
    temperature: 0.0, top_k: 0, top_p: 1.0, seed: 7)
// Byte generation can end inside a UTF-8 sequence; Bytes is always reversible.
print(codec.decode_bytes(continuation, skip_special: true).hex())
model.save("byte-trained.anergen")