Train source aligned named entity recognition

// Fictional collaboration briefs: strict BIO2 NER with original source spans.
// Run from a fresh writable directory: the checkpoint is created exclusively.
import aner.nlp

let fitting_source = nlp.corpus(
    List("Alice works at Acme in Paris .", "Bob works at Delta Labs in Rome .",
         "Clara joined North Labs in Madrid .", "David joined Acme in Paris .",
         "Alice joined Delta Labs in Madrid .", "Bob visits New York .",
         "Clara visits Paris .", "David works at North Labs in New York .",
         "Zoë joined Acme in Rome .", "North Labs hired Alice .",
         "Acme hired Bob .", "Delta Labs hired Clara ."),
    List("ner-train-01", "ner-train-02", "ner-train-03", "ner-train-04",
         "ner-train-05", "ner-train-06", "ner-train-07", "ner-train-08",
         "ner-train-09", "ner-train-10", "ner-train-11", "ner-train-12"), role: "training")
let fitting_tokens = nlp.tokenize(fitting_source)
let training = nlp.annotate(fitting_tokens,
    List(List("B-PER", "O", "O", "B-ORG", "O", "B-LOC", "O"),
         List("B-PER", "O", "O", "B-ORG", "I-ORG", "O", "B-LOC", "O"),
         List("B-PER", "O", "B-ORG", "I-ORG", "O", "B-LOC", "O"),
         List("B-PER", "O", "B-ORG", "O", "B-LOC", "O"),
         List("B-PER", "O", "B-ORG", "I-ORG", "O", "B-LOC", "O"),
         List("B-PER", "O", "B-LOC", "I-LOC", "O"),
         List("B-PER", "O", "B-LOC", "O"),
         List("B-PER", "O", "O", "B-ORG", "I-ORG", "O", "B-LOC", "I-LOC", "O"),
         List("B-PER", "O", "B-ORG", "O", "B-LOC", "O"),
         List("B-ORG", "I-ORG", "O", "B-PER", "O"),
         List("B-ORG", "O", "B-PER", "O"),
         List("B-ORG", "I-ORG", "O", "B-PER", "O")), task: "ner")

let validation_source = nlp.corpus(
    List("Zoë works at Delta Labs in Paris .", "Alice visits New York .",
         "North Labs hired David ."),
    List("ner-validation-1", "ner-validation-2", "ner-validation-3"), role: "validation")
let validation_tokens = nlp.tokenize(validation_source)
let validation = nlp.annotate(validation_tokens,
    List(List("B-PER", "O", "O", "B-ORG", "I-ORG", "O", "B-LOC", "O"),
         List("B-PER", "O", "B-LOC", "I-LOC", "O"),
         List("B-ORG", "I-ORG", "O", "B-PER", "O")), task: "ner")

let complete = nlp.fit_tagger(training, epochs: 10, seed: 911,
    shuffle: true, normalization: "nfc", casefold: true)
let prediction = complete.tag(validation_tokens)
let report = complete.evaluate_tagger(validation)
print("NER exact entity span metrics on three synthetic sequences")
print(report.entity_metric())
print("Entity metrics by type and token accuracy")
print(report.entity_metrics())
print(report.accuracy())
print("Source tokens and BIO2 tags")
print(validation_tokens.token_texts(0))
print(prediction.tags(0))
print("Entities retain source ID, original text, token, byte and scalar spans")
print(prediction.entities(0))
print("Original UTF8 byte starts and Unicode scalar starts are different")
print(validation_tokens.token_starts(0))
print(validation_tokens.token_scalar_starts(0))
print("Validation sequence 2 for error analysis")
print(validation_tokens.token_texts(2))
print("Gold and predicted BIO2 tags")
print(validation.tags(2))
print(prediction.tags(2))
print("Gold and predicted original entity spans")
print(validation.entities(2))
print(prediction.entities(2))
print("Prediction model and evaluated gold identities")
print(prediction.model_fingerprint())
print(report.gold_fingerprint())

let first = nlp.fit_tagger(training, epochs: 4, seed: 911,
    shuffle: true, normalization: "nfc", casefold: true)
first.save("ner-stage.anertagger")
let continued = nlp.load_tagger("ner-stage.anertagger").resume_tagger(training, epochs: 6)
print("Checkpoint continuation reproduces complete model and prediction identities")
print(continued.fingerprint() == complete.fingerprint())
print(continued.tag(validation_tokens).fingerprint() == prediction.fingerprint())
// These tiny invented sentences demonstrate contracts, not deployed NER quality.