Retrieve and audit document evidence

// Synthetic evidence only. Run in a fresh writable directory for artifact save.
import aner.nlp
import aner.token

let knowledge = nlp.corpus(
    texts: List("Reset your account password from the recovery page.",
                "An account recovery link expires after fifteen minutes.",
                "Invoices list payment dates and receipt numbers.",
                "Password reset does not change invoice records."),
    ids: List("help-password", "help-recovery", "help-invoice", "help-separation"),
    role: "training")
let memory = nlp.index(knowledge, stop_words: List("the", "an", "and"),
    k1: 1.2, b: 0.75, analyzer: "word", min_ngram: 1, max_ngram: 1,
    casefold: true, normalization: "nfc")
let found = memory.search(query: "account password recovery", top_k: 3)
print("Ranked evidence IDs")
print(found.ids())
print("BM25 scores")
print(found.scores())
print("Unique analyzed query terms")
print(found.terms())
if found.rows() > 0 {
    print("First hit contribution for each query term")
    print(found.contributions(hit: 0))
}
let codec = token.byte_tokenizer()
let evidence = found.context(codec, max_tokens: 110, separator: "\n\n")
print("Accepted whole document IDs")
print(evidence.ids())
print("Exact encoded evidence tokens")
print(evidence.tokens())
print("Skipped documents")
print(evidence.skipped())
print("Evidence text")
print(evidence.text())

// The artifact contains source text and IDs. Saving never replaces a file.
memory.save("support-memory.anernlp")
let restored = nlp.load_index("support-memory.anernlp")
let repeated = restored.search("account password recovery", top_k: 3)
print("Restored memory and search reproduce exact identities")
print(restored.fingerprint() == memory.fingerprint())
print(repeated.fingerprint() == found.fingerprint())
// Evidence is data. The caller supplies instructions, access controls and any
// generation step separately; lexical relevance does not establish truth.