{ "schema": "aner.native-training-qualification/1", "status": "pass", "qualification_date_utc": "2026-10-31", "scope": "One native CUDA V100 machine; explicit short-data execution and exact recovery qualification. This is not an evaluation of model quality or a cross-hardware performance guarantee.", "source": { "base_commit": "982e56104ad03acf77977a2fa89d8178ee1e9b12", "compiled_snapshot_manifest_sha256": "2231a69bb6a79d232c25ddbf77cb5d56efd3edee2e74a9fefc94c30c4eaf2c41", "snapshot_file_count": 6843, "selected_file_count": 33, "local_selected_files_match_remote_verified_snapshot": true, "files": [ { "path": "CMakeLists.txt", "sha256": "f68f0e22e2a2923123cdc282d233a4d7902c897bbed0790cc2847bcdbbedadac", "bytes": 143026 }, { "path": "cmake/AnerLLMTraining.cmake", "sha256": "e759865da24dccba81192e2762448742987f7b9c406339a94ef9c0038f33325f", "bytes": 9376 }, { "path": "include/aner/gen_modern.hpp", "sha256": "0e0e88b99ac1ffb57d1c1cc5d89e41b89fa5e64eb71905226de34660c33509ad", "bytes": 5572 }, { "path": "include/aner/gen_modern_jobs.hpp", "sha256": "5805cf9603133a128d47c658e03918eef729cbd7ab1bc04d42ef8dd06bf642c7", "bytes": 3050 }, { "path": "src/gen_modern_adapters.cpp", "sha256": "6d0ba57a502f49061e0e114d86a1fbd53968f4382507cd0ca0c77ee8bfeec864", "bytes": 13343 }, { "path": "src/gen_modern_adapters.hpp", "sha256": "d9145f53838af197d5b1301c6615196ff485c81d72a72d966ad35e517758298e", "bytes": 2658 }, { "path": "src/gen_modern_device.cpp", "sha256": "c6ed88437989053775149559c1ea6266907fdff82a89823c4cbf78cee9d0dc10", "bytes": 2872 }, { "path": "src/gen_modern_device.hpp", "sha256": "5f2d2b582fcf1f8cb40385faff1410b11a00af1df39f114a1f443c9639895609", "bytes": 524 }, { "path": "src/gen_modern_io.cpp", "sha256": "f64007323869adc8bb2108cd78b351e4cdde537b064c45445127295915289e8e", "bytes": 62841 }, { "path": "src/gen_modern_io.hpp", "sha256": "78af547daa77992949c69b9d9aec46aebc4cd52eb7f7c1372285f8f2e2d3804c", "bytes": 4636 }, { "path": "src/gen_modern_jobs.cpp", "sha256": "736f2a6cea9bcfbf95aca677c33f90c409965ff4b4ffb5fe70c2d074d9b3aac6", "bytes": 16294 }, { "path": "src/gen_modern_nf4.cpp", "sha256": "efce56062e5018a6325336549abc495c874e1bed6a3d2f11040191459667a851", "bytes": 21467 }, { "path": "src/gen_modern_nf4.hpp", "sha256": "b805531eb6d028d9b394040113a1dab65a53ce0ec7264cfc7834cfa1c305a78a", "bytes": 3099 }, { "path": "src/gen_modern_nf4_cuda.cpp", "sha256": "694a4fef267c6dfe4f3bd27f35df3f05b36986b2df8bd3b4f3ed32e5a720cdf6", "bytes": 27344 }, { "path": "src/gen_modern_nf4_cuda.hpp", "sha256": "55e885803e2dec223f097fa71a2a049b27aad50400daafef344d0ae7b3ff65ad", "bytes": 2100 }, { "path": "src/gen_modern_nf4_kernels.cuh", "sha256": "88d53fff47a6cdb73f8c2f18afafcf6c97c18e6a28c4013324c46ecc1b2fa521", "bytes": 8576 }, { "path": "src/gen_modern_paged.cpp", "sha256": "190683d2c6c6a65d53245171a413c40326c49d065c9ebaaadceaa8de731fd111", "bytes": 12699 }, { "path": "src/gen_modern_paged.hpp", "sha256": "45c8ff8f7b43eac0c659da5c28b3a245614333384bd6d666775818ed4f29c165", "bytes": 2858 }, { "path": "src/gen_modern_stub.cpp", "sha256": "9b7853a63c59e5f03cffa6f34d3d179d7304e6014d2f4b8d613c9e01749a8697", "bytes": 3253 }, { "path": "src/gen_modern_torch.cpp", "sha256": "d508aba32d2f737f60e94a111f15264b16000806f79bc1de60c836cbf3d44e89", "bytes": 68322 }, { "path": "src/gen_modern_torch_internal.hpp", "sha256": "cb6e6d89824f45af2dde812181c9c22f32a19a9847b2c252d8d0cdb0cf9f4d7b", "bytes": 603 }, { "path": "tests/gen_modern_adapters_test.cpp", "sha256": "af903f78264b2ec49c6e6cf1441093b0d839c99ed4352220a88acac3fb10b1c2", "bytes": 18839 }, { "path": "tests/gen_modern_jobs_test.cpp", "sha256": "c9961c2cb670a0965b99231818ecff7f92ae7c434623b0c566da219684c570c1", "bytes": 7642 }, { "path": "tests/gen_modern_nf4_cuda_test.cpp", "sha256": "f60e6253f6011cf699637f4aaaf72f85db85b2f49f176681466e040edf330751", "bytes": 13345 }, { "path": "tests/gen_modern_nf4_test.cpp", "sha256": "d242e2ff36948a3b831a2f35fd814860630a406d565b08fce77935ffe7ffe66e", "bytes": 16365 }, { "path": "tests/gen_modern_paged_test.cpp", "sha256": "51ec990fa195ba95f88163e120170cee308df8bb95557bc1d24de32b20446397", "bytes": 12094 }, { "path": "tests/modern_accelerated_probe.cpp", "sha256": "2fadae2645ed247ec1388538293f3075f0ddb7b8a083b021104454dc81b2b175", "bytes": 18912 }, { "path": "tests/modern_accelerated_tests.cpp", "sha256": "09e7244ef0b3dfec0cc849044018eefb4a7711c00f7ac0c5b61479727e1c3f43", "bytes": 9545 }, { "path": "tests/modern_adaptation_tests.cpp", "sha256": "8a7b8f9148f18f7f4dd0b86680e177dd6ecad880772d8d18eda5263f5c528909", "bytes": 6456 }, { "path": "tests/modern_artifact_tests.cpp", "sha256": "94a2bc6c04dff523b5cc58f3d168e55c5be212a869271c168528ec30a3e32b2a", "bytes": 25771 }, { "path": "tests/modern_cuda_tests.cpp", "sha256": "22a4f58fd9ce9d1380162a3ab8a9b8945f365a6f795fb34f4d52230eaa1aa6c4", "bytes": 12274 }, { "path": "tests/modern_training_tests.cpp", "sha256": "e6bf0bc1245739cec2666c7786367c88dc21844ed01fe1bf34ea5ecfe37ca474", "bytes": 10820 }, { "path": "tools/create_modern_llm_fixture.py", "sha256": "1a9ba521b54ea935c49c7aadf821a05ad31c704118548e0208bda20ced9ce273", "bytes": 7392 } ] }, "backend_identity": "aner.llama-trainer/3;libtorch=2.9.0;fp32-masters;threads=1;build=2df9843c2ff20c066c1a74df3982ce59d98bbcc28831871caf5ef8f8e03a15ee", "toolchain": { "cxx_compiler": "g++ (Ubuntu 15.2.0-16ubuntu1) 15.2.0", "cxx_standard": 20, "libtorch_build": "2.9.0+cu126", "profile": "prebuilt CUDA libraries linked from C++; fixed trusted CUDA kernels compiled with bundled native NVRTC at runtime", "nvcc_used": false, "python_training_runtime_used": false, "new_global_installs": false, "native_aner_cuda_kernels_enabled": false, "cublas_workspace_config": ":4096:8", "tf32_enabled": false, "fp16_cublas_accumulation": "float32; reduced-precision reduction and accumulation disabled", "pinned_llama_commit": "5047748f114564fcf271e6a648f60e2d2eae6264", "dependencies": { "original_libtorch_archive_sha256": "eae606920e69a6cfe8848bac8d55b26f5e1dbb5627e8f1c4c3b5963b01ec4561", "original_libtorch_archive_matches_prior_qualification": true, "installed_dependency_files": [ { "component": "build-version", "sha256": "9bbc314cf0c1fe821a6119fdf07df33d5ef3ec1e9efc7ed00d9ca0ecadbcb729", "bytes": 12, "matches_pinned_archive": true }, { "component": "libtorch.so", "sha256": "024ae7b7d8eae378c36c992e8a69e16a027794ca2d6b13a9700873bb73ba20f0", "bytes": 15704, "matches_pinned_archive": true }, { "component": "libtorch_cpu.so", "sha256": "fc869199da810e6532f2f03a413fdf341d638862810487a018ee93adbcab9809", "bytes": 480549585, "matches_pinned_archive": true }, { "component": "libtorch_cuda.so", "sha256": "dca12b10cc034d36a6a8cf65289f983031da9e2ebf460c5738d7d800b816b4ce", "bytes": 1831783201, "matches_pinned_archive": true }, { "component": "libc10.so", "sha256": "048f9814b4275278c8a05815899c4e658636b9bf3844f5a89d1f5343dca68d16", "bytes": 1461784, "matches_pinned_archive": true }, { "component": "libc10_cuda.so", "sha256": "0391906632605b1a71f746fb2a24332767637bf8533f25476ca075e3a12563ce", "bytes": 1729248, "matches_pinned_archive": true }, { "component": "libcudart-09529672.so.12", "sha256": "2843f62f6f29b6f0e3fb130015c335bd25c643420f27f0727bff369736526bce", "bytes": 733281, "matches_pinned_archive": true }, { "component": "libnvrtc-a1f1e7d6.so.12", "sha256": "6b745cdbd452c820052d0a6a8e6c26c13a4644ecb6498f57389c5fab81cf5dac", "bytes": 58066865, "matches_pinned_archive": true }, { "component": "libnvrtc-builtins.so", "sha256": "c58cdc18ed35f3143c3bf7d0bed69f9e899ee5e91e11822c59415cd2e26fc369", "bytes": 5326401, "matches_pinned_archive": true } ], "immutable_header_files_checked": 1666, "immutable_header_bytes_checked": 16722546, "all_header_contents_and_readonly_modes_unchanged": true, "old_llama_commit": "5047748f114564fcf271e6a648f60e2d2eae6264", "old_llama_tracked_clean": true, "old_qualified_source_and_build_not_modified_by_this_task": true, "global_installs_performed": false } }, "gpu": { "name": "Tesla V100-PCIE-32GB", "compute_capability": "7.0", "multiprocessors": 80, "global_memory_bytes": 34072559616, "cuda_runtime_version": 12060, "cuda_driver_api_version": 13000, "installed_driver_version": "580.178.04", "driver_attributes": { "unified_addressing": 1, "managed_memory": 1, "pageable_memory_access": 0, "concurrent_managed_access": 1 }, "idle_before_tests_and_real_probe": true, "after_qualification": { "used_memory_mib": 0, "utilization_percent": 0, "compute_processes": 0 } }, "jit": { "kernel_identity": "aner.nf4.cuda.fused/1;source_sha256=9c1724750613370c09122ca06b383f639a98b5177e12a36daf612538fd385421;cxx17:fmad-off:decode-separate-rn:fp32-explicit-fma-rn:half-wmma-fp32acc-half-output:tile16x16x32:shared-f32-4096:shared-half-3072:global-decode-scratch-0:no-atomics", "capabilities": "compiled=true;available=true;nvrtc=12.6;target=compute_70;toolchain_sha256=23ee8a82b5901a4e119659382dd4edc5b4d526e84efa400123710bec0c2b508c;toolchain_files=1668;toolchain_bytes=80115812;dependencies=immutable-for-process;arithmetic=cxx17:fmad-off:decode-separate-rn:fp32-explicit-fma-rn:half-wmma-fp32acc-half-output:tile16x16x32:shared-f32-4096:shared-half-3072:global-decode-scratch-0:no-atomics", "toolchain_sha256": "23ee8a82b5901a4e119659382dd4edc5b4d526e84efa400123710bec0c2b508c", "toolchain_files": 1668, "toolchain_bytes": 80115812, "header_inventory": { "files": 1666, "bytes": 16722546, "directory_symlinks": false, "readonly": true }, "nvrtc_version": [ 12, 6 ], "target": "compute_70", "standalone_wmma_feasibility": { "passed": true, "compile_seconds": 0.055530390003696084, "ptx_sha256": "fd3b4ace82251537db2aaf21bb3d25e9c5647dc6aebd2fb1f24a81b3c87a1abe", "wmma_instruction_present": true, "wmma_output_cells": 256, "max_abs_error": 0.0, "scope": "native NVRTC compute70 compilation and actual FP16 WMMA with FP32 accumulation; feasibility, not a performance benchmark" } }, "unit_suites": [ { "name": "nf4-fused", "status": "pass", "checks": 36565, "source_file": "tests/gen_modern_nf4_cuda_test.cpp", "binary": "aner_modern_nf4_cuda_tests", "binary_sha256": "39ab032e41f8432b41af062188045056af4b38e27f089655a85550300a34e447", "seconds": 3.776412422012072, "log_sha256": "de421877d34d8ebd9f418f27e0aa9f03082a8338200749e427a678c174f66c87" }, { "name": "paged", "status": "pass", "checks": 235, "source_file": "tests/gen_modern_paged_test.cpp", "binary": "aner_modern_paged_tests", "binary_sha256": "9bbf39c472fb81dedede5fade6f555a9adf0a29bb177a260aa5411b1c63fc596", "seconds": 1.0192984929890372, "log_sha256": "6ef6c8c1a93bc1a73d278156316c277f2890ee74708f1c045753d57d7c6da985" }, { "name": "accelerated", "status": "pass", "checks": 15861, "source_file": "tests/modern_accelerated_tests.cpp", "binary": "aner_modern_accelerated_tests", "binary_sha256": "ca6bd164d050e83879988db748478b51bb6df527f11c1e9455ed5e99557f458e", "seconds": 14.078225389996078, "log_sha256": "ae75d94c3895cd649454635d13d47ae6d1316e4110e9d49d864fd77096cc6544" }, { "name": "cuda-legacy", "status": "pass", "checks": 518517, "source_file": "tests/modern_cuda_tests.cpp", "binary": "aner_modern_cuda_tests", "binary_sha256": "bb52b2fcbc498190056c3c8c846afd81a550000580873bc83e7f3da3e132743f", "seconds": 3.6254540410009213, "log_sha256": "017280847787f49763206d79415b8ad0e6f2fadf4d934a49fbe6e7c06651c1a4" }, { "name": "nf4", "status": "pass", "checks": 536, "source_file": "tests/gen_modern_nf4_test.cpp", "binary": "aner_modern_nf4_tests", "binary_sha256": "0443eba2aa464b45e4fe88871f6cd7ed28b80884f6e58ac049493e8b6fd1efb5", "seconds": 1.2706011550035328, "log_sha256": "ff0867c8fa9a036a43cd7219bbfdb880e2ea54bfaa8779e2fc65200ab8c172ca" }, { "name": "adapters", "status": "pass", "checks": 240, "source_file": "tests/gen_modern_adapters_test.cpp", "binary": "aner_modern_adapters_tests", "binary_sha256": "662e3b667e4f3fc3bb21f2dc13f1f098b4d53e462115b76662be49a3e9f0a702", "seconds": 0.4669223059900105, "log_sha256": "5a083a8169ae923974863a21fa07e070963f061f8ed8677f661ed82623f2bf24" }, { "name": "artifacts", "status": "pass", "checks": 736, "source_file": "tests/modern_artifact_tests.cpp", "binary": "aner_modern_artifact_tests", "binary_sha256": "facd1b31eaddffdcfa130d1a1dbb4d460a1e0aefd11632d1573f601ea41162aa", "seconds": 0.16530489298747852, "log_sha256": "81d370a58a4ac628cbc19a30aa0beed71f51c6c8c1368d7096d9748abcdd2f3d" }, { "name": "training", "status": "pass", "checks": 110039, "source_file": "tests/modern_training_tests.cpp", "binary": "aner_modern_training_tests", "binary_sha256": "72f7ec71d9ac42001f07c9fdfe67ba447cd86731779dd752e5b81dfd42899215", "seconds": 0.8183713479957078, "log_sha256": "6793ade5a3fec358460738b285d645a1e7f80dda1c0bc2ac8d7e5b16188299c9" }, { "name": "jobs", "status": "pass", "checks": 92, "source_file": "tests/gen_modern_jobs_test.cpp", "binary": "aner_modern_jobs_tests", "binary_sha256": "2632e9730544f3dfc22fc221df67f0e8c07baf58723b7a40d84972715ac04277", "seconds": 0.6674294059921522, "log_sha256": "988bf2d58a80a24dd9b94a5a2a72fc28f14cf88e5375f243abe86e17ed309dc6" }, { "name": "adaptation", "status": "pass", "checks": 450115, "source_file": "tests/modern_adaptation_tests.cpp", "binary": "aner_modern_adaptation_tests", "binary_sha256": "2f0371c9e6e7b17a79107a56024ecf059d9c62fb2e05141c2997fa6d48f47be5", "seconds": 1.6719825780019164, "log_sha256": "3c393824b0d8d96921db004c82e391e6cf74b7d560a49e459d0c5dc811ba4913" } ], "unit_check_total": 1132936, "component_benchmarks": [ { "event": "fused_nf4_benchmark", "rows": 576, "columns": 576, "tokens": 32, "iterations": 5, "host_tiled_fp32_median_seconds": 0.001045756, "fused_fp32_median_seconds": 0.000134633, "fused_wmma_half_median_seconds": 0.000226512, "packed_gpu_bytes": 171156, "dense_float32_bytes": 1327104, "global_decode_workspace_bytes": 0, "scope": "warm synchronized linear calls; excludes upload/JIT; Half is a separate arithmetic profile" }, { "event": "fused_nf4_benchmark", "rows": 1536, "columns": 576, "tokens": 32, "iterations": 5, "host_tiled_fp32_median_seconds": 0.002525482, "fused_fp32_median_seconds": 0.000192595, "fused_wmma_half_median_seconds": 0.000224518, "packed_gpu_bytes": 456408, "dense_float32_bytes": 3538944, "global_decode_workspace_bytes": 0, "scope": "warm synchronized linear calls; excludes upload/JIT; Half is a separate arithmetic profile" }, { "event": "fused_nf4_benchmark", "rows": 49152, "columns": 576, "tokens": 32, "iterations": 5, "host_tiled_fp32_median_seconds": 0.083968013, "fused_fp32_median_seconds": 0.002759654, "fused_wmma_half_median_seconds": 0.000836664, "packed_gpu_bytes": 14605056, "dense_float32_bytes": 113246208, "global_decode_workspace_bytes": 0, "scope": "warm synchronized linear calls; excludes upload/JIT; Half is a separate arithmetic profile" }, { "event": "fused_nf4_benchmark", "rows": 4096, "columns": 4096, "tokens": 128, "iterations": 5, "host_tiled_fp32_median_seconds": 0.048438938, "fused_fp32_median_seconds": 0.006846047, "fused_wmma_half_median_seconds": 0.001728379, "packed_gpu_bytes": 8654848, "dense_float32_bytes": 67108864, "global_decode_workspace_bytes": 0, "scope": "warm synchronized linear calls; excludes upload/JIT; Half is a separate arithmetic profile" } ], "real_model": { "name": "SmolLM2-135M-Instruct", "source_filename": "smollm2-135m-instruct-f16.gguf", "source_sha256": "c953b337d61834a459884f51250e8242c7a4d2d7f74e465e84eb30e0e51d6d59", "source_bytes": 270886400, "parameters": 134515008, "input_contract": { "event": "inputs", "base": "Native pretrained Llama; backend=aner.llama-trainer/3;libtorch=2.9.0;fp32-masters;threads=1;build=2df9843c2ff20c066c1a74df3982ce59d98bbcc28831871caf5ef8f8e03a15ee; device=cuda:0; parameters=134515008; trainable=0; method=full; storage=nf4_double_quant; precision=float32; optimizer=adamw; nf4_kernel=host_tiled; lora_rank=0; updates=0; scored_tokens=0; source_sha256=c953b337d61834a459884f51250e8242c7a4d2d7f74e465e84eb30e0e51d6d59", "memory": { "weight_bytes": 69515644, "floating_base_bytes": 140544, "packed_base_bytes": 69375100, "gpu_packed_base_bytes": 0, "adapter_bytes": 0, "frozen_adapter_bytes": 0, "trainable_bytes": 0, "optimizer_bytes": 0, "managed_optimizer_bytes": 0, "optimizer_prefetch_window_bytes": 0, "dequant_tile_bytes": 8388608, "estimated_budget_bytes": 4294967296, "compute_dtype": "float32", "paged_optimizer": false, "storage": "nf4_double_quant", "scope": "owned CPU and GPU buffers; managed bytes count once and do not specify residency; prefetch window is not a VRAM cap; excludes allocator/BLAS, JIT modules and other retained handles; not process RSS" }, "training": "Pretrained data; role=training; examples=1; scored_tokens=9; sha256=5708999878e6c6ba0b4beb59b89b16587d6359a6c9f8d12dc4d7b6f88a34391c", "validation": "Pretrained data; role=validation; examples=1; scored_tokens=10; sha256=16c0efe3b05e82fdd39273fcbd02bd3c44a434417b2d2b7472e776af08dd55ca", "qv_targets": 60 }, "training_context": 32, "training_text": "Aner makes model adaptation explicit and reproducible.", "validation_text": "The quiet mountain lake reflects the clear morning sky.", "adapter_options": { "rank": 2, "alpha": 4, "seed": 71, "tiny_parameters": 13, "tiny_sharing": "global" }, "update_options": { "micro_batch": 1, "accumulation": 1, "seed": 11, "clip_norm": 1, "adapter_learning_rate": 0.0002, "full_learning_rate": 1e-05 }, "initial_logit_agreement_against_host_tiled_fp32": [ { "event": "logit_agreement", "label": "host_tiled_fp32_adamw_initial", "values": 196608, "max_abs_error": 0, "max_scaled_error": 0, "scaled_bound": 0.001 }, { "event": "logit_agreement", "label": "fused_fp32_adamw_initial", "values": 196608, "max_abs_error": 0.000148773193359375, "max_scaled_error": 9.77713044316843e-05, "scaled_bound": 0.001 }, { "event": "logit_agreement", "label": "fused_fp16_adamw_initial", "values": 196608, "max_abs_error": 0.06687355041503906, "max_scaled_error": 0.03325160883955577, "scaled_bound": 0.05 } ], "same_host_update_benchmarks": [ { "event": "benchmark", "label": "host_tiled_fp32_adamw", "profile": { "precision": "float32", "optimizer": "adamw", "nf4_kernel": "host_tiled", "page_bytes": 2097152, "loss_scale": 1, "scale_good_updates": 0, "overflow_retries": 0, "growth_interval": 2, "master_dtype": "float32", "kernel_identity": "aner.nf4.host_tiled/1" }, "warmup_updates": 1, "measured_updates": 3, "seconds": [ 1.41589597, 1.443905514, 1.424155677 ], "mean_seconds": 1.4279857203333333, "median_seconds": 1.424155677, "target_count": 60, "scored_tokens_total": 36, "memory": { "weight_bytes": 69515644, "floating_base_bytes": 140544, "packed_base_bytes": 69375100, "gpu_packed_base_bytes": 0, "adapter_bytes": 460800, "frozen_adapter_bytes": 0, "trainable_bytes": 460800, "optimizer_bytes": 921600, "managed_optimizer_bytes": 0, "optimizer_prefetch_window_bytes": 0, "dequant_tile_bytes": 8388608, "estimated_budget_bytes": 4294967296, "compute_dtype": "float32", "paged_optimizer": false, "storage": "nf4_double_quant", "scope": "owned CPU and GPU buffers; managed bytes count once and do not specify residency; prefetch window is not a VRAM cap; excludes allocator/BLAS, JIT modules and other retained handles; not process RSS" }, "scope": "same-host one-update resume wall time including private optimizer clone and held-out evaluation; synchronized CUDA, one warmup; optimizer=adamw held constant. FP16 changes numerical precision. No general throughput or quality claim." }, { "event": "benchmark", "label": "fused_fp32_adamw", "profile": { "precision": "float32", "optimizer": "adamw", "nf4_kernel": "fused", "page_bytes": 2097152, "loss_scale": 1, "scale_good_updates": 0, "overflow_retries": 0, "growth_interval": 2, "master_dtype": "float32", "kernel_identity": "aner.nf4.cuda.fused/1;source_sha256=9c1724750613370c09122ca06b383f639a98b5177e12a36daf612538fd385421;cxx17:fmad-off:decode-separate-rn:fp32-explicit-fma-rn:half-wmma-fp32acc-half-output:tile16x16x32:shared-f32-4096:shared-half-3072:global-decode-scratch-0:no-atomics" }, "warmup_updates": 1, "measured_updates": 3, "seconds": [ 0.25521944, 0.241986806, 0.250038384 ], "mean_seconds": 0.24908154333333332, "median_seconds": 0.250038384, "target_count": 60, "scored_tokens_total": 36, "memory": { "weight_bytes": 138889900, "floating_base_bytes": 140544, "packed_base_bytes": 69375100, "gpu_packed_base_bytes": 69374256, "adapter_bytes": 460800, "frozen_adapter_bytes": 0, "trainable_bytes": 460800, "optimizer_bytes": 921600, "managed_optimizer_bytes": 0, "optimizer_prefetch_window_bytes": 0, "dequant_tile_bytes": 0, "estimated_budget_bytes": 4294967296, "compute_dtype": "float32", "paged_optimizer": false, "storage": "nf4_double_quant", "scope": "owned CPU and GPU buffers; managed bytes count once and do not specify residency; prefetch window is not a VRAM cap; excludes allocator/BLAS, JIT modules and other retained handles; not process RSS" }, "scope": "same-host one-update resume wall time including private optimizer clone and held-out evaluation; synchronized CUDA, one warmup; optimizer=adamw held constant. FP16 changes numerical precision. No general throughput or quality claim." }, { "event": "benchmark", "label": "fused_fp16_adamw", "profile": { "precision": "float16", "optimizer": "adamw", "nf4_kernel": "fused", "page_bytes": 2097152, "loss_scale": 512, "scale_good_updates": 0, "overflow_retries": 0, "growth_interval": 2, "master_dtype": "float32", "kernel_identity": "aner.nf4.cuda.fused/1;source_sha256=9c1724750613370c09122ca06b383f639a98b5177e12a36daf612538fd385421;cxx17:fmad-off:decode-separate-rn:fp32-explicit-fma-rn:half-wmma-fp32acc-half-output:tile16x16x32:shared-f32-4096:shared-half-3072:global-decode-scratch-0:no-atomics" }, "warmup_updates": 1, "measured_updates": 3, "seconds": [ 0.288477136, 0.281035665, 0.27820717 ], "mean_seconds": 0.2825733236666667, "median_seconds": 0.281035665, "target_count": 60, "scored_tokens_total": 36, "memory": { "weight_bytes": 138889900, "floating_base_bytes": 140544, "packed_base_bytes": 69375100, "gpu_packed_base_bytes": 69374256, "adapter_bytes": 460800, "frozen_adapter_bytes": 0, "trainable_bytes": 460800, "optimizer_bytes": 921600, "managed_optimizer_bytes": 0, "optimizer_prefetch_window_bytes": 0, "dequant_tile_bytes": 0, "estimated_budget_bytes": 4294967296, "compute_dtype": "float16", "paged_optimizer": false, "storage": "nf4_double_quant", "scope": "owned CPU and GPU buffers; managed bytes count once and do not specify residency; prefetch window is not a VRAM cap; excludes allocator/BLAS, JIT modules and other retained handles; not process RSS" }, "scope": "same-host one-update resume wall time including private optimizer clone and held-out evaluation; synchronized CUDA, one warmup; optimizer=adamw held constant. FP16 changes numerical precision. No general throughput or quality claim." } ], "qualification": [ { "method": "qlora", "backend_adapter_method": "lora", "status": "pass", "base_storage": "nf4_double_quant", "parameters": 134515008, "trainable_parameters": 115200, "target_tensor_count": 60, "first_update_profile": { "precision": "float16", "optimizer": "paged_adamw", "nf4_kernel": "fused", "page_bytes": 2097152, "loss_scale": 128, "scale_good_updates": 1, "overflow_retries": 0, "growth_interval": 2, "master_dtype": "float32", "kernel_identity": "aner.nf4.cuda.fused/1;source_sha256=9c1724750613370c09122ca06b383f639a98b5177e12a36daf612538fd385421;cxx17:fmad-off:decode-separate-rn:fp32-explicit-fma-rn:half-wmma-fp32acc-half-output:tile16x16x32:shared-f32-4096:shared-half-3072:global-decode-scratch-0:no-atomics" }, "first_update_scored_tokens": 9, "first_update_validation_loss": 5.604807281494141, "single_model_owned_buffers": { "weight_bytes": 138889900, "floating_base_bytes": 140544, "packed_base_bytes": 69375100, "gpu_packed_base_bytes": 69374256, "adapter_bytes": 460800, "frozen_adapter_bytes": 0, "trainable_bytes": 460800, "optimizer_bytes": 921600, "managed_optimizer_bytes": 921600, "optimizer_prefetch_window_bytes": 2097152, "dequant_tile_bytes": 0, "estimated_budget_bytes": 4294967296, "compute_dtype": "float16", "paged_optimizer": true, "storage": "nf4_double_quant", "scope": "owned CPU and GPU buffers; managed bytes count once and do not specify residency; prefetch window is not a VRAM cap; excludes allocator/BLAS, JIT modules and other retained handles; not process RSS" }, "first_update_global_managed_counters": { "managed_live_bytes": 921600, "global_peak_managed_bytes": 921600, "live_allocations": 240, "cumulative_allocated_bytes": 921600, "prefetch_to_cpu_requested_bytes": 921600, "prefetch_to_cuda_requested_bytes": 921600, "prefetch_calls": 480, "free_failures": 0 }, "exact_checkpoint_reload_parameter_moments_scaler": true, "exact_resumed_scored_token_counters": true, "exact_one_plus_checkpoint_plus_one_equals_uninterrupted_two": { "event": "exact_resume", "method": "lora", "equal_parameters": true, "equal_optimizer": true, "equal_profile_state": true, "updates": 2, "profile": { "precision": "float16", "optimizer": "paged_adamw", "nf4_kernel": "fused", "page_bytes": 2097152, "loss_scale": 256, "scale_good_updates": 0, "overflow_retries": 0, "growth_interval": 2, "master_dtype": "float32", "kernel_identity": "aner.nf4.cuda.fused/1;source_sha256=9c1724750613370c09122ca06b383f639a98b5177e12a36daf612538fd385421;cxx17:fmad-off:decode-separate-rn:fp32-explicit-fma-rn:half-wmma-fp32acc-half-output:tile16x16x32:shared-f32-4096:shared-half-3072:global-decode-scratch-0:no-atomics" } }, "source_model_and_profile_immutable": true, "actual_parameter_update_and_finite_loss": true, "managed_ownership_released_after_handles": true, "timings_seconds": { "first_update": 0.345591518, "checkpoint_write": 4.423811457, "checkpoint_reload": 11.324260175, "resume_one": 0.396832095, "uninterrupted_two": 0.593445509, "fa_first_update": 0.308053614, "fa_checkpoint_write": 4.349019409, "fa_checkpoint_reload": 11.32580899, "fa_resume_one": 0.316969875, "fa_uninterrupted_two": 0.509728633 }, "retained_comparison_handles_memory": { "event": "memory", "label": "lora_whole_qualification_with_retained_comparison_handles", "torch_allocated_peak_bytes": 222590976, "torch_reserved_peak_bytes": 226492416, "cuda_free_bytes_point_sample": 33252704256, "cuda_total_bytes": 34072559616, "managed": { "managed_live_bytes": 3686400, "global_peak_managed_bytes": 3686400, "live_allocations": 960, "cumulative_allocated_bytes": 3686400, "prefetch_to_cpu_requested_bytes": 6451200, "prefetch_to_cuda_requested_bytes": 6451200, "prefetch_calls": 3360, "free_failures": 0 }, "scope": "Torch allocator peaks exclude managed allocations, JIT modules and external allocations. Managed counters are process-wide allocation/request counters, never current GPU residency. Multiple retained comparison handles are included; free bytes are not a peak." } }, { "method": "lora_fa", "backend_adapter_method": "lora_fa", "status": "pass", "base_storage": "nf4_double_quant", "parameters": 134515008, "trainable_parameters": 46080, "target_tensor_count": 60, "first_update_profile": { "precision": "float16", "optimizer": "paged_adamw", "nf4_kernel": "fused", "page_bytes": 2097152, "loss_scale": 128, "scale_good_updates": 1, "overflow_retries": 0, "growth_interval": 2, "master_dtype": "float32", "kernel_identity": "aner.nf4.cuda.fused/1;source_sha256=9c1724750613370c09122ca06b383f639a98b5177e12a36daf612538fd385421;cxx17:fmad-off:decode-separate-rn:fp32-explicit-fma-rn:half-wmma-fp32acc-half-output:tile16x16x32:shared-f32-4096:shared-half-3072:global-decode-scratch-0:no-atomics" }, "first_update_scored_tokens": 9, "first_update_validation_loss": 5.601944351196289, "single_model_owned_buffers": { "weight_bytes": 138889900, "floating_base_bytes": 140544, "packed_base_bytes": 69375100, "gpu_packed_base_bytes": 69374256, "adapter_bytes": 184320, "frozen_adapter_bytes": 277440, "trainable_bytes": 184320, "optimizer_bytes": 368640, "managed_optimizer_bytes": 368640, "optimizer_prefetch_window_bytes": 2097152, "dequant_tile_bytes": 0, "estimated_budget_bytes": 4294967296, "compute_dtype": "float16", "paged_optimizer": true, "storage": "nf4_double_quant", "scope": "owned CPU and GPU buffers; managed bytes count once and do not specify residency; prefetch window is not a VRAM cap; excludes allocator/BLAS, JIT modules and other retained handles; not process RSS" }, "first_update_global_managed_counters": { "managed_live_bytes": 368640, "global_peak_managed_bytes": 3686400, "live_allocations": 120, "cumulative_allocated_bytes": 4055040, "prefetch_to_cpu_requested_bytes": 6819840, "prefetch_to_cuda_requested_bytes": 6819840, "prefetch_calls": 3600, "free_failures": 0 }, "exact_checkpoint_reload_parameter_moments_scaler": true, "exact_resumed_scored_token_counters": true, "exact_one_plus_checkpoint_plus_one_equals_uninterrupted_two": { "event": "exact_resume", "method": "lora_fa", "equal_parameters": true, "equal_optimizer": true, "equal_profile_state": true, "updates": 2, "profile": { "precision": "float16", "optimizer": "paged_adamw", "nf4_kernel": "fused", "page_bytes": 2097152, "loss_scale": 256, "scale_good_updates": 0, "overflow_retries": 0, "growth_interval": 2, "master_dtype": "float32", "kernel_identity": "aner.nf4.cuda.fused/1;source_sha256=9c1724750613370c09122ca06b383f639a98b5177e12a36daf612538fd385421;cxx17:fmad-off:decode-separate-rn:fp32-explicit-fma-rn:half-wmma-fp32acc-half-output:tile16x16x32:shared-f32-4096:shared-half-3072:global-decode-scratch-0:no-atomics" } }, "source_model_and_profile_immutable": true, "actual_parameter_update_and_finite_loss": true, "managed_ownership_released_after_handles": true, "timings_seconds": { "first_update": 0.308053614, "checkpoint_write": 4.349019409, "checkpoint_reload": 11.32580899, "resume_one": 0.316969875, "uninterrupted_two": 0.509728633 }, "retained_comparison_handles_memory": { "event": "memory", "label": "lora_fa_whole_qualification_with_retained_comparison_handles", "torch_allocated_peak_bytes": 221193216, "torch_reserved_peak_bytes": 226492416, "cuda_free_bytes_point_sample": 33372241920, "cuda_total_bytes": 34072559616, "managed": { "managed_live_bytes": 1474560, "global_peak_managed_bytes": 3686400, "live_allocations": 480, "cumulative_allocated_bytes": 5160960, "prefetch_to_cpu_requested_bytes": 9031680, "prefetch_to_cuda_requested_bytes": 9031680, "prefetch_calls": 5040, "free_failures": 0 }, "scope": "Torch allocator peaks exclude managed allocations, JIT modules and external allocations. Managed counters are process-wide allocation/request counters, never current GPU residency. Multiple retained comparison handles are included; free bytes are not a peak." } }, { "method": "tinylora", "backend_adapter_method": "tinylora", "status": "pass", "base_storage": "nf4_double_quant", "parameters": 134515008, "trainable_parameters": 13, "target_tensor_count": 60, "first_update_profile": { "precision": "float16", "optimizer": "paged_adamw", "nf4_kernel": "fused", "page_bytes": 2097152, "loss_scale": 128, "scale_good_updates": 1, "overflow_retries": 0, "growth_interval": 2, "master_dtype": "float32", "kernel_identity": "aner.nf4.cuda.fused/1;source_sha256=9c1724750613370c09122ca06b383f639a98b5177e12a36daf612538fd385421;cxx17:fmad-off:decode-separate-rn:fp32-explicit-fma-rn:half-wmma-fp32acc-half-output:tile16x16x32:shared-f32-4096:shared-half-3072:global-decode-scratch-0:no-atomics" }, "first_update_scored_tokens": 9, "first_update_validation_loss": 5.602960586547852, "single_model_owned_buffers": { "weight_bytes": 138889900, "floating_base_bytes": 140544, "packed_base_bytes": 69375100, "gpu_packed_base_bytes": 69374256, "adapter_bytes": 52, "frozen_adapter_bytes": 473280, "trainable_bytes": 52, "optimizer_bytes": 104, "managed_optimizer_bytes": 104, "optimizer_prefetch_window_bytes": 2097152, "dequant_tile_bytes": 0, "estimated_budget_bytes": 4294967296, "compute_dtype": "float16", "paged_optimizer": true, "storage": "nf4_double_quant", "scope": "owned CPU and GPU buffers; managed bytes count once and do not specify residency; prefetch window is not a VRAM cap; excludes allocator/BLAS, JIT modules and other retained handles; not process RSS" }, "first_update_global_managed_counters": { "managed_live_bytes": 104, "global_peak_managed_bytes": 3686400, "live_allocations": 2, "cumulative_allocated_bytes": 5161064, "prefetch_to_cpu_requested_bytes": 9031784, "prefetch_to_cuda_requested_bytes": 9031784, "prefetch_calls": 5044, "free_failures": 0 }, "exact_checkpoint_reload_parameter_moments_scaler": true, "exact_resumed_scored_token_counters": true, "exact_one_plus_checkpoint_plus_one_equals_uninterrupted_two": { "event": "exact_resume", "method": "tinylora", "equal_parameters": true, "equal_optimizer": true, "equal_profile_state": true, "updates": 2, "profile": { "precision": "float16", "optimizer": "paged_adamw", "nf4_kernel": "fused", "page_bytes": 2097152, "loss_scale": 256, "scale_good_updates": 0, "overflow_retries": 0, "growth_interval": 2, "master_dtype": "float32", "kernel_identity": "aner.nf4.cuda.fused/1;source_sha256=9c1724750613370c09122ca06b383f639a98b5177e12a36daf612538fd385421;cxx17:fmad-off:decode-separate-rn:fp32-explicit-fma-rn:half-wmma-fp32acc-half-output:tile16x16x32:shared-f32-4096:shared-half-3072:global-decode-scratch-0:no-atomics" } }, "source_model_and_profile_immutable": true, "actual_parameter_update_and_finite_loss": true, "managed_ownership_released_after_handles": true, "timings_seconds": { "first_update": 0.248696382, "checkpoint_write": 4.443490618, "checkpoint_reload": 11.390726042, "resume_one": 0.263534186, "uninterrupted_two": 0.433084223 }, "retained_comparison_handles_memory": { "event": "memory", "label": "tinylora_whole_qualification_with_retained_comparison_handles", "torch_allocated_peak_bytes": 220672512, "torch_reserved_peak_bytes": 226492416, "cuda_free_bytes_point_sample": 33252704256, "cuda_total_bytes": 34072559616, "managed": { "managed_live_bytes": 416, "global_peak_managed_bytes": 3686400, "live_allocations": 8, "cumulative_allocated_bytes": 5161376, "prefetch_to_cpu_requested_bytes": 9032408, "prefetch_to_cuda_requested_bytes": 9032408, "prefetch_calls": 5068, "free_failures": 0 }, "scope": "Torch allocator peaks exclude managed allocations, JIT modules and external allocations. Managed counters are process-wide allocation/request counters, never current GPU residency. Multiple retained comparison handles are included; free bytes are not a peak." } }, { "method": "full", "backend_adapter_method": "full", "status": "pass", "base_storage": "float32", "parameters": 134515008, "trainable_parameters": 134515008, "target_tensor_count": 272, "first_update_profile": { "precision": "float16", "optimizer": "paged_adamw", "nf4_kernel": "host_tiled", "page_bytes": 2097152, "loss_scale": 128, "scale_good_updates": 1, "overflow_retries": 0, "growth_interval": 2, "master_dtype": "float32", "kernel_identity": "aner.nf4.host_tiled/1" }, "first_update_scored_tokens": 9, "first_update_validation_loss": 5.011479187011719, "single_model_owned_buffers": { "weight_bytes": 538060032, "floating_base_bytes": 538060032, "packed_base_bytes": 0, "gpu_packed_base_bytes": 0, "adapter_bytes": 0, "frozen_adapter_bytes": 0, "trainable_bytes": 538060032, "optimizer_bytes": 1076120064, "managed_optimizer_bytes": 1076120064, "optimizer_prefetch_window_bytes": 2097152, "dequant_tile_bytes": 0, "estimated_budget_bytes": 4294967296, "compute_dtype": "float16", "paged_optimizer": true, "storage": "float32", "scope": "owned CPU and GPU buffers; managed bytes count once and do not specify residency; prefetch window is not a VRAM cap; excludes allocator/BLAS, JIT modules and other retained handles; not process RSS" }, "first_update_global_managed_counters": { "managed_live_bytes": 1076120064, "global_peak_managed_bytes": 1076120064, "live_allocations": 544, "cumulative_allocated_bytes": 1081281440, "prefetch_to_cpu_requested_bytes": 1085152472, "prefetch_to_cuda_requested_bytes": 1085152472, "prefetch_calls": 6728, "free_failures": 0 }, "exact_checkpoint_reload_parameter_moments_scaler": true, "exact_resumed_scored_token_counters": true, "exact_one_plus_checkpoint_plus_one_equals_uninterrupted_two": { "event": "exact_resume", "method": "full", "equal_parameters": true, "equal_optimizer": true, "equal_profile_state": true, "updates": 2, "profile": { "precision": "float16", "optimizer": "paged_adamw", "nf4_kernel": "host_tiled", "page_bytes": 2097152, "loss_scale": 256, "scale_good_updates": 0, "overflow_retries": 0, "growth_interval": 2, "master_dtype": "float32", "kernel_identity": "aner.nf4.host_tiled/1" } }, "source_model_and_profile_immutable": true, "actual_parameter_update_and_finite_loss": true, "managed_ownership_released_after_handles": true, "timings_seconds": { "first_update": 2.058114982, "checkpoint_write": 21.694343663, "checkpoint_reload": 29.227792813, "resume_one": 2.341639228, "uninterrupted_two": 3.439132429 }, "retained_comparison_handles_memory": { "event": "memory", "label": "full_whole_qualification_with_retained_comparison_handles", "torch_allocated_peak_bytes": 3554389504, "torch_reserved_peak_bytes": 3611295744, "cuda_free_bytes_point_sample": 25564545024, "cuda_total_bytes": 34072559616, "managed": { "managed_live_bytes": 4304480256, "global_peak_managed_bytes": 4304480256, "live_allocations": 2176, "cumulative_allocated_bytes": 4309641632, "prefetch_to_cpu_requested_bytes": 7541872856, "prefetch_to_cuda_requested_bytes": 7541872856, "prefetch_calls": 16688, "free_failures": 0 }, "scope": "Torch allocator peaks exclude managed allocations, JIT modules and external allocations. Managed counters are process-wide allocation/request counters, never current GPU residency. Multiple retained comparison handles are included; free bytes are not a peak." } } ], "final": { "event": "passed", "checks": 590039, "managed": { "managed_live_bytes": 0, "global_peak_managed_bytes": 4304480256, "live_allocations": 0, "cumulative_allocated_bytes": 4309641632, "prefetch_to_cpu_requested_bytes": 7541872856, "prefetch_to_cuda_requested_bytes": 7541872856, "prefetch_calls": 16688, "free_failures": 0 }, "quality_claim": false, "oversubscription_claim": false }, "total_process_seconds": 237.8172182580165, "probe_binary_sha256": "26a7ecd4b213ae9160f70aeb3bac89d8d04b4a63ecdeff8ad136dcb82748efac", "log_sha256": "6a3909e3210aeeb0970ff3d24a6e6bfa5f352372bcaeab663b6022f83394482e", "jsonl_sha256": "91760451052ff419dd7e41c7ff36cccfef7f9c0056a2d1137a5baa8df9f91e74" }, "limits": { "quality_improvement_claim": false, "managed_oversubscription_claim": false, "vram_cap_claim": false, "other_gpus_qualified": false, "benchmark_repetitions": "one warmup plus three synchronized measured resume updates for real model; five warm synchronized calls per component shape", "memory": "Owned model buffers, Torch allocator peaks, process-wide managed allocation counters and point CUDA free-memory samples are separate measurements. They exclude different storage and cannot be added into a single-model VRAM claim. Multiple retained comparison handles appear in qualification memory events. Prefetch request windows and last-prefetch metadata are not residency limits.", "performance": "Component benchmarks exclude upload and JIT. Real update timings include immutable optimizer-state cloning and held-out evaluation. FP16 changes arithmetic and was slower than fused FP32 for this short real-model workload. No general speedup is claimed.", "remaining": [ "Activation checkpointing", "BF16 and additional hardware/architecture qualification", "Real managed-memory oversubscription under measured pressure", "Long-run dataset quality and stability evaluation" ] }, "primary_references": [ "https://docs.nvidia.com/cuda/archive/12.6.3/nvrtc/index.html", "https://docs.nvidia.com/cuda/archive/12.6.3/volta-tuning-guide/index.html", "https://docs.nvidia.com/cuda/archive/12.6.3/cuda-c-programming-guide/index.html", "https://docs.nvidia.com/cuda/archive/12.6.3/cuda-driver-api/group__CUDA__UNIFIED.html" ], "local_qualification": { "mac_native": { "tests": 13, "passed": 12, "gpu_only_skipped": 1, "log_sha256": "fdfcf9057cf8db425600200027f7737cdaabec8819cd9b6bb1227891d4f8162d", "scope": "CPU native training, artifacts, language, NF4, adapters, jobs, profile validation; genuine GPU paging is separately required on V100" }, "mac_latest_test_only_updates": { "tests": 3, "passed": 3, "log_sha256": "a8992910ee45ec67ac9abd39da0bd58ef89282d2068842fd207ddc4b0da9e382" }, "mac_service_only": { "tests": 3, "passed": 3, "log_sha256": "4557735a3f54e3ab857d5df2c577629af7f0657082c8c68533c73e89c84140a8", "scope": "Existing service/Ollama/cloud mock connections and explicit native-unavailable language behavior; local loopback server only" }, "mac_existing_work_integration": { "tests": 5, "passed": 5, "log_sha256": "99e9114eac24ad2299d6091866e652cb77e2b144f4433698513a9702b5d2494a", "backend_identity_matches_clean_feature": true, "scope": "Three-way integration with existing local language changes, preserving unrelated work" }, "editor": { "native_signature_entries": 1249, "special_overloads": 148, "editor_signatures": 5419, "grammar_checks": 1957, "new_profile_method_help": "pass" }, "documentation": { "aner_fenced_blocks_typechecked": 7, "new_complete_example_typechecks_native_and_service": true }, "cpu_backend_identity": "aner.llama-trainer/3;libtorch=2.9.0;fp32-masters;threads=1;build=47b77db4db6868a83949cff41b321e3edce38e9b898b8ee30fcc69be62e628b2" } }