// Audit the completed saved study without making inference requests.
// Run from the project root: aner run analysis/audit_llm.aner --max-steps 10000000
// Input CSV is the lossless selected-field projection of the raw request trace.
import aner.data

fn audit_require(condition: Bool, message: String) -> Unit {
    if !condition {
        print("FAILED: LLM evidence audit: " + message)
        let failure: Result<Bool, String> = Result.err(message)
        print(failure.unwrap())
    }
}

fn audit_model_index(name: String) -> Int64 {
    if name == "llama3.1:latest" { return 0 }
    if name == "llama3.2:latest" { return 1 }
    if name == "mistral:latest" { return 2 }
    if name == "qwen2.5:14b-instruct" { return 3 }
    if name == "qwen3.8:latest" { return 4 }
    return -1
}

fn audit_integer_choice(text: String) -> Int64 {
    for value in Range(1, 11) { if text.trim() == String(value) { return value } }
    return 0
}

let audit_columns = List("request_id", "round", "position", "request", "model", "returned_model_hex",
    "finish_hex", "text_hex", "value", "elapsed_ns", "input_tokens", "output_tokens", "total_tokens",
    "cached_input_tokens", "cache_write_input_tokens", "reasoning_tokens")
var audit_schema = data.schema()
for column in Range(audit_columns.len()) {
    var kind = "Int64"
    if column >= 4 && column <= 7 { kind = "String" }
    audit_schema = data.column(audit_schema, cast<String>(audit_columns.get(column)), kind, false)
}
// Empty visible replies have empty text_hex. They are present Strings, not null.
let audit_rows = data.collect(data.scan_csv("results/llm-requests.csv", audit_schema,
    missing: "__TNC_NO_NULL__"), max_rows: 2_000, max_bytes: 67_108_864, batch_rows: 256)
audit_require(data.rows(audit_rows) == 1_250, "expected exactly 1250 recorded requests")

let audit_models = List("llama3.1:latest", "llama3.2:latest", "mistral:latest", "qwen2.5:14b-instruct", "qwen3.8:latest")
let audit_usage_names = List("input_tokens", "output_tokens", "total_tokens", "cached_input_tokens", "cache_write_input_tokens", "reasoning_tokens")
let audit_seen = Array<Bool>(1_250, false)
let audit_attempted = Array<Int64>(5, 0)
let audit_accepted = Array<Int64>(5, 0)
let audit_counts = Array<Int64>(50, 0)
let audit_finish_not_stop = Array<Int64>(5, 0)
let audit_empty = Array<Int64>(5, 0)
let audit_noncanonical = Array<Int64>(5, 0)
let audit_usage_known = Array<Int64>(30, 0)
let audit_usage_missing = Array<Int64>(30, 0)
let audit_elapsed = Array<Int64>(1_250, 0)

for row in Range(data.rows(audit_rows)) {
    let request_id = data.integer(audit_rows, row, "request_id")
    let round = data.integer(audit_rows, row, "round")
    let position = data.integer(audit_rows, row, "position")
    let request = data.integer(audit_rows, row, "request")
    let model_index = audit_model_index(data.text(audit_rows, row, "model"))
    audit_require(request_id >= 1 && request_id <= 1_250, "request ID outside 1..1250")
    audit_require(round >= 1 && round <= 5 && position >= 1 && position <= 5,
        "round and position must be 1..5")
    audit_require(request >= 1 && request <= 50, "request within model block must be 1..50")
    audit_require(model_index >= 0, "unknown installed-model label")
    audit_require(model_index == (position + round - 2) % 5, "model does not match cyclic block order")
    audit_require(request_id == ((round - 1) * 5 + position - 1) * 50 + request,
        "request ID does not match its round/block/request identity")
    audit_require(!audit_seen[request_id - 1], "duplicate request ID")
    audit_seen[request_id - 1] = true
    let elapsed = data.integer(audit_rows, row, "elapsed_ns")
    audit_require(elapsed > 0, "generate-call elapsed nanoseconds must be positive")
    audit_elapsed[model_index * 250 + (round - 1) * 50 + request - 1] = elapsed
    audit_attempted[model_index] += 1

    // Strict native hex and UTF-8 decoding; never repair text or extract a digit.
    let returned_model = Bytes.from_hex(data.text(audit_rows, row, "returned_model_hex")).decode_utf8()
    let finish = Bytes.from_hex(data.text(audit_rows, row, "finish_hex")).decode_utf8()
    let raw_text = Bytes.from_hex(data.text(audit_rows, row, "text_hex")).decode_utf8()
    var expected_value = 0
    if finish != "stop" { audit_finish_not_stop[model_index] += 1 }
    else if raw_text.trim() == "" { audit_empty[model_index] += 1 }
    else {
        expected_value = audit_integer_choice(raw_text)
        if expected_value == 0 { audit_noncanonical[model_index] += 1 }
        else {
            audit_accepted[model_index] += 1
            audit_counts[model_index * 10 + expected_value - 1] += 1
        }
    }
    audit_require(data.integer(audit_rows, row, "value") == expected_value,
        "stored value differs from raw finish/text at request " + String(request_id))
    for category in Range(6) {
        let tokens = data.integer(audit_rows, row, cast<String>(audit_usage_names.get(category)))
        audit_require(tokens >= -1, "token field must be reported nonnegative usage or unknown -1")
        let slot = model_index * 6 + category
        if tokens == -1 { audit_usage_missing[slot] += 1 }
        else { audit_usage_known[slot] += tokens }
    }
}
for request_id in Range(1_250) { audit_require(audit_seen[request_id], "missing request ID") }
for model_index in Range(5) {
    audit_require(audit_attempted[model_index] == 250, "expected 250 calls for every model")
    var counted = 0
    for face in Range(10) { counted += audit_counts[model_index * 10 + face] }
    let rejected = audit_finish_not_stop[model_index] + audit_empty[model_index] + audit_noncanonical[model_index]
    audit_require(counted == audit_accepted[model_index] && counted + rejected == 250,
        "accepted histogram and exclusive rejections must account for every request")
}

// These five summaries can be compared with the original native study output.
// Timing includes all calls, accepted and rejected, without an excluded warmup.
for model_index in Range(5) {
    var histogram_json = "["
    for face in Range(10) {
        if face > 0 { histogram_json = histogram_json + "," }
        histogram_json = histogram_json + String(audit_counts[model_index * 10 + face])
    }
    histogram_json = histogram_json + "]"
    var usage_json = "{"
    for category in Range(6) {
        if category > 0 { usage_json = usage_json + "," }
        let slot = model_index * 6 + category
        usage_json = usage_json + "\"" + cast<String>(audit_usage_names.get(category))
            + "\":{\"known_total\":" + String(audit_usage_known[slot])
            + ",\"missing_count\":" + String(audit_usage_missing[slot]) + "}"
    }
    usage_json = usage_json + "}"
    let sorted = Array<Int64>(250, 0)
    for request in Range(250) { sorted[request] = audit_elapsed[model_index * 250 + request] }
    sorted.sort()
    let median_seconds = (Float64(sorted[124]) + Float64(sorted[125])) / 2_000_000_000.0
    let rejected = audit_finish_not_stop[model_index] + audit_empty[model_index] + audit_noncanonical[model_index]
    print("TNC_LLM_AUDIT_V1 {\"schema\":\"traceable-number-choices/llm-audit/1\",\"case\":\"B\","
        + "\"source\":\"results/llm-requests.csv\",\"model\":\"" + cast<String>(audit_models.get(model_index))
        + "\",\"attempted\":" + String(audit_attempted[model_index]) + ",\"accepted\":" + String(audit_accepted[model_index])
        + ",\"rejected\":" + String(rejected) + ",\"counts\":" + histogram_json
        + ",\"rejections\":{\"finish_not_stop\":" + String(audit_finish_not_stop[model_index])
        + ",\"empty\":" + String(audit_empty[model_index]) + ",\"noncanonical\":" + String(audit_noncanonical[model_index])
        + "},\"usage\":" + usage_json + ",\"generate_call_seconds\":{\"n\":250,\"median\":" + String(median_seconds)
        + ",\"min\":" + String(Float64(sorted[0]) / 1_000_000_000.0)
        + ",\"max\":" + String(Float64(sorted[249]) / 1_000_000_000.0) + "}}")
}