Classify support tickets with fitted text features

// Synthetic support tickets: choose partitions before fitting vocabulary or IDF.
import aner.nlp
import aner.ml
import aner.metrics
import aner.tensor

let training = nlp.corpus(
    texts: List("reset password for account login", "account login password recovery",
                "cannot login password reset", "invoice payment billing question",
                "billing invoice payment receipt", "payment receipt invoice correction"),
    ids: List("train-login-1", "train-login-2", "train-login-3",
              "train-billing-1", "train-billing-2", "train-billing-3"),
    role: "training")
let validation = nlp.corpus(
    texts: List("password reset login help", "account password recovery assistance",
                "invoice billing receipt request", "payment invoice correction help"),
    ids: List("validation-1", "validation-2", "validation-3", "validation-4"),
    role: "validation")
let encoder = nlp.vectorizer(training, stop_words: List("for"),
    weighting: "tfidf", analyzer: "word", min_ngram: 1, max_ngram: 2,
    min_df: 1, max_df: 1.0, max_features: 100,
    casefold: true, sublinear_tf: true, norm: "l2", normalization: "nfc")
let fitting = encoder.transform(training)
let held_out = encoder.transform(validation)
let labels: Tensor = [[10.0], [10.0], [10.0], [20.0], [20.0], [20.0]]
let actual: Tensor = [[10.0], [10.0], [20.0], [20.0]]

// The dense bridge is explicit and bounded. Keep IDs beside the Tensor.
let classifier = ml.fit_knn(fitting.dense(max_cells: 10000), labels, k: 1)
let predicted = classifier.predict(held_out.dense(max_cells: 10000))
print("Validation accuracy on four synthetic tickets")
print(metrics.accuracy(predicted, actual))
print("Prediction row IDs")
print(held_out.ids())
print("Out of vocabulary term occurrences per row")
print(held_out.oov())
print("Rows with no retained features")
print(held_out.empty_rows())
print("Canonical fitted configuration")
print(encoder.configuration())
print("Fitted representation fingerprint")
print(encoder.fingerprint())
// Tune ngrams, norm and k on validation; reserve a separate final test corpus.