diff --git a/Tutorials/ML/CMakeLists.txt b/Tutorials/ML/CMakeLists.txt index db28093cd7d..ad8a5477b88 100644 --- a/Tutorials/ML/CMakeLists.txt +++ b/Tutorials/ML/CMakeLists.txt @@ -19,3 +19,14 @@ o2physics_add_dpl_workflow(apply-ml-selection SOURCES applyMlSelection.cxx PUBLIC_LINK_LIBRARIES O2Physics::AnalysisCore O2Physics::MLCore COMPONENT_NAME AnalysisTutorial) + +o2physics_add_dpl_workflow(benchmark-onnx-model + SOURCES benchmarkOnnxModel.cxx + PUBLIC_LINK_LIBRARIES O2::Framework O2::ML O2::GPUCommon ONNXRuntime::ONNXRuntime + TARGETVARNAME benchmarkOnnxModelTarget + COMPONENT_NAME AnalysisTutorial) + +# System libraries are linked directly: the workflow helper requires namespaced targets. +if(CMAKE_DL_LIBS) + target_link_libraries(${benchmarkOnnxModelTarget} PRIVATE ${CMAKE_DL_LIBS}) +endif() diff --git a/Tutorials/ML/README-benchmark.md b/Tutorials/ML/README-benchmark.md new file mode 100644 index 00000000000..65702fc5ede --- /dev/null +++ b/Tutorials/ML/README-benchmark.md @@ -0,0 +1,123 @@ +# ONNX inference benchmark tutorial + +`benchmarkOnnxModel.cxx` registers `o2-analysis-benchmark-onnx-model`. +It runs once with deterministic synthetic data, writes one CSV row, and exits. +It accepts one dense floating-point input of shape `[batch, features]` and one +or more outputs. Static dimensions must match the options; symbolic dimensions +are filled from the options. Output types are left to ONNX Runtime. + +## Build and GPU prerequisites + +Build O2Physics normally after loading its O2 environment. This target links +`O2::ML`, `O2::GPUCommon`, `ONNXRuntime::ONNXRuntime`, and the platform dynamic +loader. No HIP or CUDA headers are required for building the task. The O2 installation +must contain `ML/3rdparty/GPUORTFloat16.h` and its GPUCommon headers. + +For AMD GPU runs, ONNX Runtime must contain **MIGraphXExecutionProvider**, and its +MIGraphX/ROCm shared libraries, including `libamdhip64.so`, must be discoverable +at runtime. The task checks provider availability and disables CPU fallback: +unsupported graphs fail rather than producing a misleading GPU benchmark. +No changes to O2's OrtInterface are needed for this task; the current checked-out +OrtInterface does not attach an AMD execution provider, so this example uses +ORT directly. + +For NVIDIA runs, select `--provider cuda`. ONNX Runtime must contain +**CUDAExecutionProvider**, with matching CUDA/cuDNN libraries and an NVIDIA +driver. Telemetry loads `libcudart.so` (also tries `.so.13`, `.so.12`, `.so.11.0`) +from the library search path. Keep this runtime consistent with the ORT CUDA +build. CUDA provider options require ORT 1.18 or newer: `use_tf32=0` disables +TF32 so the FP32 benchmark does not silently use TF32 matrix multiplication. +Both GPU backends disable CPU fallback and use the same timing/CSV path. + +Check the actual hydra driver, ROCm, MIGraphX and ORT combination first. MI100 +(gfx908) and MI50 (gfx906) do not have identical software support. MI50 is absent +from current supported ROCm hardware lists, so it needs a compatible older +stack or a separately validated build. The old ROCm execution provider was +removed from ONNX Runtime starting with 1.23; this task uses MIGraphX. + +References: +- +- +- +- + +## Run the three comparisons + +Use equivalent FP32 and FP16 exports of the same network. Replace the example +feature count with the actual model input width. Use separate output files; +each invocation overwrites its output CSV. + +```bash +o2-analysis-benchmark-onnx-model -b --model-path model-fp32.onnx \ + --provider cpu --fp16 false --batch-size 1024 --features 5 \ + --cpu-threads 1 --warmup 20 --iterations 200 --output-csv cpu-fp32.csv + +o2-analysis-benchmark-onnx-model -b --model-path model-fp32.onnx \ + --provider migraphx --gpu-id 0 --fp16 false --batch-size 1024 --features 5 \ + --warmup 20 --iterations 200 --output-csv gpu-fp32.csv + +o2-analysis-benchmark-onnx-model -b --model-path model-fp16.onnx \ + --provider migraphx --gpu-id 0 --fp16 true --batch-size 1024 --features 5 \ + --warmup 20 --iterations 200 --output-csv gpu-fp16.csv +``` + +For NVIDIA, replace `--provider migraphx` with `--provider cuda`, for example: + +```bash +o2-analysis-benchmark-onnx-model -b --model-path model-fp32.onnx \ + --provider cuda --gpu-id 0 --fp16 false --batch-size 1024 --features 5 \ + --warmup 20 --iterations 200 --output-csv cuda-fp32.csv + +o2-analysis-benchmark-onnx-model -b --model-path model-fp16.onnx \ + --provider cuda --gpu-id 0 --fp16 true --batch-size 1024 --features 5 \ + --warmup 20 --iterations 200 --output-csv cuda-fp16.csv +``` + +`gpu-id` is the logical ID after `HIP_VISIBLE_DEVICES` / `ROCR_VISIBLE_DEVICES` +masking on AMD or `CUDA_VISIBLE_DEVICES` masking on NVIDIA. Run one benchmark process at a time on an idle +GPU. Repeat with the same batch sizes and CPU thread count, and compare several +runs. Small batches can be dominated by host overhead and transfers. + +`--fp16 true` selects O2's `o2::OrtDataType::Float16_t` input storage and checks +that the model input is FLOAT16. It does not convert model weights or prove that +every internal operation is FP16. Export/inspect the model accordingly. FP16 +models retaining FP32 input interfaces need an adapted input path; this small +tutorial intentionally rejects them. CPU runs require FP32 input. + +## What the CSV measures + +- `samples_per_second = batch_size * iterations / seconds`. +- Timing covers synchronous `Session::Run`, input/output transfers, and output + allocation/destruction. It excludes session creation, MIGraphX compilation, + warmup, input generation, FP16 conversion, telemetry, and CSV I/O. This is + host-to-host inference throughput, not GPU kernel-only throughput. +- GPU memory fields are **device-wide used bytes**, sampled through + `hipMemGetInfo` (AMD) or `cudaMemGetInfo` (NVIDIA) before session creation, after warmup, and after timed runs. + They include other processes and runtime allocations; they are neither + model-only memory nor a transient peak. The baseline follows GPU runtime/context + initialization. CPU runs leave these fields empty. GPU telemetry failures + fail the run rather than reporting zero usage. +- Precision labels describe the requested model/input variant, not measured + arithmetic precision. Strict CPU fallback rejection can reject graphs that + would otherwise run with some CPU operators. + +This benchmark does not check prediction accuracy. Compare real model outputs +between the exports separately before interpreting performance gains. + +## Adding the AO2D input later + +Planned input on hydra: + +```text +/lustre/alice/users/marin/R3A/TPCTrees/TPCTrees_LHC24as_250224/AO2D_merge_LHC24as_updatedBB_250224.root +``` + +The file has not been inspected here. Do not pass `--aod-file` to this standalone +synthetic task yet. First establish the AO2D tables, feature columns and order, +training normalization, selections, and missing-value policy. Then add an +analysis `process(...)` input with the appropriate table definitions, build +contiguous batches from its rows, and reuse the inference path. Keep session +creation in `init`, aggregate timing/sample counts across batches, and write +results at end of stream. Handle the final partial batch explicitly (dynamic +batch, or pad and count only valid samples). Keep data reading/preprocessing +outside inference timing or report a separate full-pipeline metric. diff --git a/Tutorials/ML/benchmarkOnnxModel.cxx b/Tutorials/ML/benchmarkOnnxModel.cxx new file mode 100644 index 00000000000..67677940346 --- /dev/null +++ b/Tutorials/ML/benchmarkOnnxModel.cxx @@ -0,0 +1,266 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// Benchmark one dense ONNX input [batch, features] using CPU, AMD MIGraphX or NVIDIA CUDA. +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace +{ +// Load GPU telemetry at runtime so CPU builds need no HIP or CUDA headers. +// Memory is device-wide, not process-local; samples are not a transient peak. +class GpuMemory +{ + public: + explicit GpuMemory(int device, bool cuda) + { + const std::vector libraries = cuda + ? std::vector{"libcudart.so", "libcudart.so.13", "libcudart.so.12", "libcudart.so.11.0"} + : std::vector{"libamdhip64.so"}; + for (auto name : libraries) { + library = dlopen(name, RTLD_LAZY | RTLD_LOCAL); + if (library) { + break; + } + } + if (!library) { + throw std::runtime_error(cuda ? "GPU telemetry requires libcudart.so (CUDA 11/12/13) on LD_LIBRARY_PATH" + : "GPU telemetry requires libamdhip64.so on LD_LIBRARY_PATH"); + } + auto setDevice = reinterpret_cast(dlsym(library, cuda ? "cudaSetDevice" : "hipSetDevice")); + memoryFunction = cuda ? "cudaMemGetInfo" : "hipMemGetInfo"; + getMemory = reinterpret_cast(dlsym(library, memoryFunction)); + if (!setDevice || !getMemory || setDevice(device) != 0) { + dlclose(library); + library = nullptr; + throw std::runtime_error(cuda ? "Cannot select CUDA device or resolve CUDA memory API" + : "Cannot select HIP device or resolve HIP memory API"); + } + } + ~GpuMemory() + { + if (library) { + dlclose(library); + } + } + GpuMemory(GpuMemory const&) = delete; + GpuMemory& operator=(GpuMemory const&) = delete; + uint64_t used() const + { + size_t free = 0, total = 0; + if (getMemory(&free, &total) != 0 || free > total) { + throw std::runtime_error(std::string(memoryFunction) + " failed"); + } + return total - free; + } + + private: + const char* memoryFunction = nullptr; + void* library = nullptr; + int (*getMemory)(size_t*, size_t*) = nullptr; +}; + +std::string csvQuote(std::string const& value) +{ + std::string result = "\""; + for (char c : value) { + result += c; + if (c == '"') { + result += '"'; + } + } + return result + '"'; +} +} // namespace + +using namespace o2::framework; + +struct BenchmarkOnnxModel { + Configurable modelPath{"model-path", "model.onnx", "Local ONNX model"}; + Configurable provider{"provider", "cpu", "cpu, migraphx or cuda"}; + Configurable gpuId{"gpu-id", 0, "HIP/CUDA logical device ID (after visibility masking)"}; + Configurable fp16{"fp16", false, "Require FP16 input; does not convert model weights"}; + Configurable batchSize{"batch-size", 1024, "Samples per inference"}; + Configurable features{"features", 5, "Features per sample, in model order"}; + Configurable warmup{"warmup", 10, "Untimed warm-up calls"}; + Configurable iterations{"iterations", 100, "Timed inference calls"}; + Configurable cpuThreads{"cpu-threads", 1, "ORT intra-op CPU threads"}; + Configurable outputCsv{"output-csv", "onnx-benchmark.csv", "CSV output (overwritten)"}; + + void run(ProcessingContext& pc) + { + benchmark(); + pc.services().get().endOfStream(); + pc.services().get().readyToQuit(QuitRequest::Me); + } + + void benchmark() + { + if (batchSize.value <= 0 || features.value <= 0 || iterations.value <= 0 || warmup.value < 1 || cpuThreads.value <= 0 || gpuId.value < 0) { + throw std::runtime_error("Require positive batch, features, iterations, warmup, threads and nonnegative GPU ID"); + } + bool cuda = provider.value == "cuda"; + bool gpu = cuda || provider.value == "migraphx"; + if (!gpu && provider.value != "cpu") { + throw std::runtime_error("provider must be cpu, migraphx or cuda"); + } + if (!gpu && fp16.value) { + throw std::runtime_error("Use FP32 for the CPU baseline"); + } + std::unique_ptr memory; + uint64_t before = 0, afterWarmup = 0, afterRun = 0; + Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "tutorial-benchmark"); + Ort::SessionOptions options; + options.SetIntraOpNumThreads(cpuThreads.value); + options.SetInterOpNumThreads(1); + options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); + if (gpu) { + const std::string providerName = cuda ? "CUDAExecutionProvider" : "MIGraphXExecutionProvider"; + auto providers = Ort::GetAvailableProviders(); + if (std::find(providers.begin(), providers.end(), providerName) == providers.end()) { + throw std::runtime_error("This ONNX Runtime has no " + providerName); + } + memory = std::make_unique(gpuId.value, cuda); + before = memory->used(); + if (cuda) { + // V2 options allow disabling TF32 for the FP32 comparison. + OrtCUDAProviderOptionsV2* raw = nullptr; + Ort::ThrowOnError(Ort::GetApi().CreateCUDAProviderOptions(&raw)); + std::unique_ptr nvidia(raw, Ort::GetApi().ReleaseCUDAProviderOptions); + const std::string device = std::to_string(gpuId.value); + const char* keys[] = {"device_id", "use_tf32"}; + const char* values[] = {device.c_str(), "0"}; + Ort::ThrowOnError(Ort::GetApi().UpdateCUDAProviderOptions(nvidia.get(), keys, values, 2)); + options.AppendExecutionProvider_CUDA_V2(*nvidia); + } else { + OrtMIGraphXProviderOptions amd{}; + amd.device_id = gpuId.value; + amd.migraphx_mem_limit = std::numeric_limits::max(); + // Keep the supplied model precision; do not automatically quantize FP32. + amd.migraphx_fp16_enable = 0; + options.AppendExecutionProvider_MIGraphX(amd); + } + options.AddConfigEntry("session.disable_cpu_ep_fallback", "1"); + } + Ort::Session session(env, modelPath.value.c_str(), options); + if (session.GetInputCount() != 1 || session.GetOutputCount() < 1) { + throw std::runtime_error("Tutorial expects exactly one dense input and at least one output"); + } + auto inputInfo = session.GetInputTypeInfo(0); + auto tensorInfo = inputInfo.GetTensorTypeAndShapeInfo(); + auto shape = tensorInfo.GetShape(); + auto type = fp16.value ? ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16 : ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; + if (tensorInfo.GetElementType() != type || shape.size() != 2 || + (shape[0] >= 0 && shape[0] != batchSize.value) || (shape[1] >= 0 && shape[1] != features.value)) { + throw std::runtime_error("Input must match fp16 flag and [batch-size, features]; only dynamic dimensions may be overridden"); + } + shape = {batchSize.value, features.value}; + const size_t count = static_cast(batchSize.value) * features.value; + if (count > std::numeric_limits::max() / sizeof(float)) { + throw std::runtime_error("Input size overflow"); + } + // Replace this feature filling with AO2D extraction and training preprocessing later. + std::vector floats(fp16.value ? 0 : count); + std::vector halves(fp16.value ? count : 0); + static_assert(sizeof(o2::OrtDataType::Float16_t) == 2); + for (size_t i = 0; i < count; ++i) { + float value = static_cast(i % 101) / 100.f - 0.5f; + if (fp16.value) { + halves[i] = o2::OrtDataType::Float16_t(value); + } else { + floats[i] = value; + } + } + auto host = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + void* data = fp16.value ? static_cast(halves.data()) : static_cast(floats.data()); + size_t bytes = count * (fp16.value ? sizeof(halves[0]) : sizeof(floats[0])); + auto input = Ort::Value::CreateTensor(host, data, bytes, shape.data(), shape.size(), type); + Ort::AllocatorWithDefaultOptions allocator; + auto inputName = session.GetInputNameAllocated(0, allocator); + const char* inputNames[] = {inputName.get()}; + std::vector names; + names.reserve(session.GetOutputCount()); + for (size_t i = 0; i < session.GetOutputCount(); ++i) { + names.emplace_back(session.GetOutputNameAllocated(i, allocator).get()); + } + std::vector outputNames; + outputNames.reserve(names.size()); + for (auto const& name : names) { + outputNames.push_back(name.c_str()); + } + Ort::RunOptions runOptions; + auto infer = [&]() { + // Default synchronous Run with CPU input/output includes host/device transfers + // and output allocation/destruction. No asynchronous Run option is enabled. + auto outputs = session.Run(runOptions, inputNames, &input, 1, outputNames.data(), outputNames.size()); + }; + for (int i = 0; i < warmup.value; ++i) { + infer(); + } + if (memory) { + afterWarmup = memory->used(); + } + auto start = std::chrono::steady_clock::now(); + for (int i = 0; i < iterations.value; ++i) { + infer(); + } + double seconds = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + if (memory) { + afterRun = memory->used(); + } + std::ofstream csv(outputCsv.value, std::ios::trunc); + if (!csv) { + throw std::runtime_error("Cannot open output-csv"); + } + csv << "model,ort_version,provider,precision,gpu_id,batch_size,features,warmup,iterations,cpu_threads,seconds,samples_per_second,gpu_used_before_bytes,gpu_used_after_warmup_bytes,gpu_used_after_run_bytes\n"; + csv << csvQuote(modelPath.value) << ',' << csvQuote(Ort::GetVersionString()) << ',' << provider.value << ',' + << (fp16.value ? "fp16" : "fp32") << ',' << (gpu ? gpuId.value : -1) << ',' + << batchSize.value << ',' << features.value << ',' << warmup.value << ',' << iterations.value << ',' << cpuThreads.value << ',' + << std::setprecision(12) << seconds << ',' << (static_cast(batchSize.value) * iterations.value / seconds) << ','; + if (memory) { + csv << before << ',' << afterWarmup << ',' << afterRun; + } else { + csv << ",,"; + } + csv << '\n'; + csv.close(); + if (!csv) { + throw std::runtime_error("Failed writing output-csv"); + } + } +}; + +WorkflowSpec defineDataProcessing(ConfigContext const& cfgc) +{ + return WorkflowSpec{adaptAnalysisTask(cfgc)}; +}