From 4d6acac7721ea9bc9513ddb200a5b0cafd20771a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 22 Oct 2026 17:51:02 -0700 Subject: [PATCH 1/2] Vulkan: decide direct convolution auto from the output check, not a timing race --- docs/backend.md | 8 ++++---- docs/backend_probe.md | 5 +++-- src/core/probe/conv_direct.cpp | 21 +++++++++++++++++---- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/docs/backend.md b/docs/backend.md index 057d3f20e..b1775470b 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -233,10 +233,10 @@ sd-cli -m model.safetensors -p "a cat" --conv-direct off,diffusion=on - `on` requests direct 2D and 3D convolution. - `off` uses the existing im2col-based convolution paths. - `auto` first checks support for a representative F16-weight/F32-input direct - convolution graph. Vulkan then benchmarks direct convolution against im2col - using two small shapes per dimension, with warmup, output validation, and five - interleaved measurements per shape. Direct convolution must improve the sum of - median times by at least 5%; a failed probe defaults to `off`. + convolution graph. On Vulkan, direct 2D convolution is then enabled when its + output on two small shapes matches im2col (a failed probe defaults to `off`), + and direct 3D convolution stays `off`. The decision does not depend on timing, + so the same seed renders the same pixels on a busy and on an idle machine. Other backends retain their implicit GEMM checks without benchmarking: CUDA 2D requires a compiled Turing-or-newer MMA architecture; HIP 2D requires the supported CDNA MFMA or RDNA WMMA architectures described below for flash diff --git a/docs/backend_probe.md b/docs/backend_probe.md index c420326d4..e51a9b96a 100644 --- a/docs/backend_probe.md +++ b/docs/backend_probe.md @@ -5,8 +5,9 @@ policy stays separate: `src/core/probe/flash_attn.cpp`, for example, defines att shapes and its CPU/CUDA/HIP/Vulkan shortcuts, then calls `sd_run_backend_probe` for unknown configurations. `src/core/probe/conv_direct.cpp` uses metadata-only probe graphs to check direct 2D/3D convolution support, then enables known implicit GEMM -paths on non-Vulkan backends. Vulkan compares direct convolution with im2col -using the shared measurement machinery. +paths on non-Vulkan backends. Vulkan runs direct 2D convolution against im2col +through the shared measurement machinery for its output check only (the timing +is logged, not used), and keeps direct 3D convolution off. For each representative shape, create a `BackendProbeCase`: diff --git a/src/core/probe/conv_direct.cpp b/src/core/probe/conv_direct.cpp index 7f4666a41..60091b802 100644 --- a/src/core/probe/conv_direct.cpp +++ b/src/core/probe/conv_direct.cpp @@ -65,6 +65,15 @@ bool sd_probe_conv_direct(ggml_backend_t backend, bool conv3d) { return false; } if (sd_backend_is(backend, "Vulkan")) { + // The decision must not depend on how busy the machine is. The two probe shapes run in well under a + // millisecond, so on a loaded host both paths measure the same submit overhead and the old 5% race + // flipped from run to run, changing the pixels of a fixed seed. Direct 3D convolution stays off: it is + // a tie on NVIDIA in the probe and fails its output check on RADV. Direct 2D convolution is on whenever its + // output matches im2col: measured faster and far smaller in VAE decodes on RADV (gfx1151) and NVIDIA. + if (conv3d) { + LOG_INFO("conv3d direct auto: %s -> off (Vulkan)", name); + return false; + } std::vector cases; for (const auto& shape : {std::pair{32, 64}, {16, 128}}) { BackendProbeCase test; @@ -74,14 +83,18 @@ bool sd_probe_conv_direct(ggml_backend_t backend, bool conv3d) { test.initialize = [conv3d](BackendProbeGraph& graph) { return initialize_conv_probe(graph, conv3d); }; cases.push_back(std::move(test)); } - const auto result = sd_run_backend_probe(backend, 0, cases); + // Only the output check decides, so one timed run each and no time budget a busy host could exhaust. + BackendProbeOptions options; + options.measurement_runs = 1; + options.time_budget_ms = 1e9; + const auto result = sd_run_backend_probe(backend, 0, cases, options); if (!result.completed) { LOG_WARN("conv%dd direct auto: %s -> off (%s)", dims, name, result.error.c_str()); return false; } - LOG_INFO("conv%dd direct auto: %s -> %s (probe: im2col %.3f ms, direct %.3f ms)", - dims, name, result.candidate_faster ? "on" : "off", result.reference_ms, result.candidate_ms); - return result.candidate_faster; + LOG_INFO("conv%dd direct auto: %s -> on (output check passed; probe: im2col %.3f ms, direct %.3f ms)", + dims, name, result.reference_ms, result.candidate_ms); + return true; } if (sd_backend_supports_conv_implicit_gemm(backend, conv3d)) { LOG_INFO("conv%dd direct auto: %s -> on (implicit GEMM)", dims, name); From a2c3b975a261d745d7a5ff9f5020c602c9a2df70 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 22 Oct 2026 22:10:54 -0700 Subject: [PATCH 2/2] Vulkan conv direct auto: no free-memory floor on the output-check probe The probe graphs need a few MB; the default 128 MiB floor still turned direct convolution off whenever another workload held the VRAM, so load could change the path and the pixels. A device too full to allocate the probe still fails it (off). Also update docs/performance.md, which still described the Vulkan rule as a benchmark. --- docs/performance.md | 5 +++-- src/core/probe/conv_direct.cpp | 8 ++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/performance.md b/docs/performance.md index 18491039e..eb23ca542 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -32,8 +32,9 @@ segment count change. path. Performance depends on GPU architecture, backend, driver, model, and tensor shapes, so enabling direct convolution can either improve or reduce speed. -Automatic selection uses small benchmarks on Vulkan and implicit GEMM checks -on other backends. It does not guarantee the fastest end-to-end generation. +Automatic selection checks direct 2D convolution output against im2col on +Vulkan (timing is not used; direct 3D stays off there) and uses implicit GEMM +checks on other backends. It does not guarantee the fastest end-to-end generation. Compare `on` and `off` with the same model and generation settings over repeated runs, keeping flash attention unchanged and separating startup/probe time from steady-state generation time. See [backend selection](backend.md) diff --git a/src/core/probe/conv_direct.cpp b/src/core/probe/conv_direct.cpp index 60091b802..5e1688553 100644 --- a/src/core/probe/conv_direct.cpp +++ b/src/core/probe/conv_direct.cpp @@ -84,9 +84,13 @@ bool sd_probe_conv_direct(ggml_backend_t backend, bool conv3d) { cases.push_back(std::move(test)); } // Only the output check decides, so one timed run each and no time budget a busy host could exhaust. + // No free-memory cutoff either: the probe graphs need a few MB, and the default 128 MiB floor would turn + // direct convolution off whenever another workload holds the VRAM. A device too full to allocate them + // still fails the probe (allocation failure -> off). BackendProbeOptions options; - options.measurement_runs = 1; - options.time_budget_ms = 1e9; + options.measurement_runs = 1; + options.time_budget_ms = 1e9; + options.min_free_device_bytes = 0; const auto result = sd_run_backend_probe(backend, 0, cases, options); if (!result.completed) { LOG_WARN("conv%dd direct auto: %s -> off (%s)", dims, name, result.error.c_str());