Research with unit aware sensor CSV data

// Complete synthetic sensor experiment, preserving text identifiers and source order.
// Requires the updated native development runtime with CSV unit columns.
// Run from the directory containing examples/fixtures/units/sensor_runs.csv.
// Population spread is not measurement uncertainty or a calibration-source model.
import aner.data
import aner.units
import aner.math
import aner.tensor

fn close(actual: Float64, expected: Float64, tolerance: Float64 = 0.000000001) -> Bool {
    return math.abs(actual - expected) <= tolerance
}
fn verify(label: String, condition: Bool) -> Unit {
    print(label)
    print(condition)
}

var schema = data.schema()
schema = data.column(schema, "sensor_id", "String", false)
schema = data.column(schema, "group_id", "String", false)
schema = schema.unit_column("temperature_f", "degF", true)
schema = schema.unit_column("distance_cm", "cm", true)
schema = schema.unit_column("elapsed_s", "s", false)
schema = data.column(schema, "quality", "String", false)
let original = data.scan_csv("examples/fixtures/units/sensor_runs.csv", schema, missing: "NA")
// Normalize at ingestion: source numbers still mean degF/cm; target columns are degC/m.
let source = original.convert_unit(column: "temperature_f", target_unit: "degC")
    .convert_unit(column: "distance_cm", target_unit: "m")
let all = data.summarize(source, max_bytes: 1048576, batch_rows: 2)
verify("All eight source rows were validated", data.scanned_rows(all) == 8 && data.rows(all) == 8)
verify("Whole-source temperature has six present and two missing", data.valid(all, "temperature_f") == 6 && data.missing(all, "temperature_f") == 2)
verify("Whole-source distance has seven present and one missing", data.valid(all, "distance_cm") == 7 && data.missing(all, "distance_cm") == 1)
let accepted = data.where_eq(source, "quality", "accepted")
let accepted_report = data.summarize(accepted, max_bytes: 1048576, batch_rows: 2)
verify("Six accepted rows remain after quality selection", data.rows(accepted_report) == 6 && data.scanned_rows(accepted_report) == 8)
verify("Accepted temperature has five present and one missing", data.valid(accepted_report, "temperature_f") == 5 && data.missing(accepted_report, "temperature_f") == 1)
let accepted_mean: Point<degC> = accepted_report.unit_mean("temperature_f", "degC")
verify("Accepted observed mean is 24 degC", close(accepted_mean.value(), 24.0))

// Eligibility combines a text quality rule, a compatible Point threshold and presence.
let present = data.where_present(data.where_present(accepted, "temperature_f"), "distance_cm")
let warm = data.where_ge(present, "temperature_f", value: 21.0@degC)
let traveled = data.where_ge(warm, "distance_cm", value: 1.0@m)
let eligible = data.where_ge(traveled, "elapsed_s", value: 1.0@s)
let final_report = data.summarize(eligible, max_bytes: 1048576, batch_rows: 2)
verify("Four eligible rows and no missing measurements", data.rows(final_report) == 4 && data.missing(final_report, "temperature_f") == 0 && data.missing(final_report, "distance_cm") == 0)
let table = data.collect(eligible, max_rows: 100, max_bytes: 1048576, batch_rows: 2)
verify("Original sensor IDs and source order are retained", data.text(table, 0, "sensor_id") == "0002" && data.text(table, 3, "sensor_id") == "0005")
verify("Group labels remain available beside measurements", data.text(table, 0, "group_id") == "lab-a" && data.text(table, 3, "group_id") == "lab-c")
verify("Column inspection retains normalized units and kind", data.column_type(table, 2) == "Point<degC>" && data.column_unit(table, 2) == "degC" && data.column_unit(table, 3) == "m" && data.column_unit(table, 0) == "")

let temperatures: Array<Point<degC>> = table.unit_values("temperature_f", "degC")
let distances: Array<Quantity<m>> = table.unit_values("distance_cm", "m")
let times: Array<Quantity<s>> = table.unit_values("elapsed_s", "s")
verify("Normalized temperatures have exact Point<degC> elements", type(temperatures) == "Array<Point<degC>>" && dtype(temperatures) == "Float64")
verify("Normalized temperature mean is 25 degC", close(temperatures.mean().value(), 25.0))
verify("Population spread is sqrt(5) delta_degC", temperatures.std().unit() == "delta_degC" && close(temperatures.std().value(), math.sqrt(5.0)))

let correction = Array<Quantity<delta_degC>>(size: temperatures.len(), fill: 0.5@delta_degC)
let calibrated = temperatures.add(correction)
let reference = Array<Point<degC>>(size: temperatures.len(), fill: 25.5@degC)
let anomalies: Array<Quantity<delta_degC>> = calibrated.sub(reference)
verify("Declared offset changes mean to 25.5 degC", close(calibrated.mean().value(), 25.5))
verify("Interval anomalies retain type and sum to zero", type(anomalies) == "Array<Quantity<delta_degC>>" && close(anomalies.sum().value(), 0.0))
verify("Uniform correction preserves observed population spread", close(calibrated.std().value(), temperatures.std().value()))

let speeds: Array<Quantity<"m/s">> = distances.div(times)
verify("Per-row speed is canonical m/s", speeds.unit() == "m/s" && close(speeds.get(0).value(), 0.3) && close(speeds.get(3).value(), 0.6))
verify("Mean per-run speed is 0.395 m/s", close(speeds.mean().value(), 0.395))
let overall: Quantity<"m/s"> = distances.sum() / times.sum()
verify("Total distance divided by total time is 0.375 m/s", close(overall.value(), 0.375))

// Raw model input is deliberate: retain its m/s descriptor with the IDs above.
let export_unit = speeds.unit()
let raw = speeds.values()
let model_input = raw.to_tensor(rows: speeds.len(), cols: 1)
verify("Explicit numerical export has four rows and retains its unit label", export_unit == "m/s" && tensor.rows(model_input) == 4 && tensor.cols(model_input) == 1 && close(tensor.value(model_input, 3, 0), 0.6))
temperatures[0] = -10.0@degC
verify("Column extraction is detached from immutable table storage", close(table.unit_value(0, "temperature_f", "degC").value(), 22.0))