From e1a55a0b2017c2fde91232335b5ac4976d9f8d0a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 22 Oct 2026 03:37:33 -0700 Subject: [PATCH 1/6] Count physical cores in the affinity mask for the default thread count on Linux The sysfs loop read /sys/devices/system/cpu/topology/thread_siblings (missing the /cpu path component), never opened a file, and always fell back to half of std::thread::hardware_concurrency(). That halves every core count on a host without SMT and ignores taskset / container cpusets (a 8-CPU cpuset on a 192-CPU host ran 96 threads). Count the distinct thread-sibling sets among the CPUs in the process affinity mask instead, falling back to the old estimate over the allowed CPUs when sysfs has no topology. --- src/core/util.cpp | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/src/core/util.cpp b/src/core/util.cpp index 7bd5c0bd1..90c91344a 100644 --- a/src/core/util.cpp +++ b/src/core/util.cpp @@ -29,6 +29,10 @@ #include #endif +#ifdef __linux__ +#include +#endif + #include "ggml-backend.h" #include "ggml.h" #include "stable-diffusion.h" @@ -340,20 +344,33 @@ bool MmapWrapper::copy_data(void* buf, size_t n, size_t offset) const { // LICENSE: https://github.com/ggerganov/llama.cpp/blob/master/LICENSE int32_t sd_get_num_physical_cores() { #ifdef __linux__ - // enumerate the set of thread siblings, num entries is num cores - std::unordered_set siblings; - for (uint32_t cpu = 0; cpu < UINT32_MAX; ++cpu) { - std::ifstream thread_siblings("/sys/devices/system/cpu" + std::to_string(cpu) + "/topology/thread_siblings"); - if (!thread_siblings.is_open()) { - break; // no more cpus + // One thread per physical core this process may run on: the distinct thread-sibling sets among the CPUs in + // the affinity mask (a taskset / container cpuset leaves fewer CPUs than the machine has). The old loop read + // "/sys/devices/system/cpu/..." (missing "/cpu"), never found a file and always fell through to half the + // logical CPUs, which on a host without SMT is half its cores. + cpu_set_t mask; + CPU_ZERO(&mask); + if (sched_getaffinity(0, sizeof(mask), &mask) == 0) { + std::unordered_set siblings; + int allowed = 0; + for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { + if (!CPU_ISSET(cpu, &mask)) { + continue; + } + allowed++; + std::ifstream thread_siblings("/sys/devices/system/cpu/cpu" + std::to_string(cpu) + "/topology/thread_siblings"); + std::string line; + if (thread_siblings.is_open() && std::getline(thread_siblings, line) && !line.empty()) { + siblings.insert(line); + } } - std::string line; - if (std::getline(thread_siblings, line)) { - siblings.insert(line); + if (!siblings.empty()) { + return static_cast(siblings.size()); + } + if (allowed > 0) { + // No topology in sysfs (some containers): assume 2-way SMT above 4 CPUs, as below. + return allowed <= 4 ? allowed : allowed / 2; } - } - if (siblings.size() > 0) { - return static_cast(siblings.size()); } #elif defined(__APPLE__) && defined(__MACH__) int32_t num_physical_cores; From f55cb1088b7fc97bdba58cbd6b11e1e297a242e5 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 22 Oct 2026 04:39:53 -0700 Subject: [PATCH 2/6] ggml-cpu: fill and scatter direct conv patches in row runs (bit-identical) The CPU CONV_2D / CONV_3D kernels (--conv-direct) gathered each patch element by element across all input planes, and wrote each GEMM output element into a different output plane. Process up to 64 consecutive patches of one output row at a time instead: each source row segment is read once per run and each output plane gets a contiguous row segment. Same values and the same fp32 -> fp16 conversions in the same slots, so results are bit-identical to the per-patch loops. --- .../0014-ggml-cpu-conv-direct-row-runs.patch | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 scripts/unsloth/ggml-patches/0014-ggml-cpu-conv-direct-row-runs.patch diff --git a/scripts/unsloth/ggml-patches/0014-ggml-cpu-conv-direct-row-runs.patch b/scripts/unsloth/ggml-patches/0014-ggml-cpu-conv-direct-row-runs.patch new file mode 100644 index 000000000..141fe1fe1 --- /dev/null +++ b/scripts/unsloth/ggml-patches/0014-ggml-cpu-conv-direct-row-runs.patch @@ -0,0 +1,217 @@ +diff --git a/src/ggml-cpu/ops.cpp b/src/ggml-cpu/ops.cpp +--- a/src/ggml-cpu/ops.cpp ++++ b/src/ggml-cpu/ops.cpp +@@ -7464,6 +7464,8 @@ + // ggml_compute_forward_conv_2d + + ++#define CONV_2D_RUN 64 ++ + static void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params, + const ggml_tensor * kernel, // [KW, KH, IC, OC] + const ggml_tensor * src, // [W, H, C, N] +@@ -7521,40 +7523,44 @@ + const int64_t patch_start = patch_start_batch + params->ith * patch_per_thread; + const int64_t patch_end = std::min(patch_start + patch_per_thread, patch_end_batch); + +- //im2col for a patch +- for (int64_t p = patch_start; p < patch_end; ++p) { +- const int64_t batch_n = p / (dst_w * dst_h); +- const int64_t src_x = (p / dst_w) % dst_h; +- const int64_t src_y = p % dst_w; ++ // im2col for this thread's patches, a run of up to CONV_2D_RUN consecutive patches of one output row at a ++ // time: each (ic, ky) source row is then read once, contiguously, for the whole run instead of once per patch ++ // (one patch touches c_in planes; one plane per patch was a cache and TLB miss per element). Same values, ++ // same conversions as the per-patch loop, so the result is bit-identical. ++ const size_t ts = traits->type_size; ++ for (int64_t p = patch_start; p < patch_end;) { ++ const int64_t n_idx = p / (dst_w * dst_h); ++ const int64_t oy = (p / dst_w) % dst_h; ++ const int64_t ox0 = p % dst_w; ++ const int64_t run = std::min({ patch_end - p, dst_w - ox0, (int64_t) CONV_2D_RUN }); + +- const float * src_base = (const float *)((const char *)src_data + batch_n * src->nb[3]); +- char * dst_row = (char *) tmp + (p % patches_per_batch) * knl_n * traits->type_size; ++ const char * src_base = (const char *) src_data + n_idx * src->nb[3]; ++ char * dst_base = (char *) tmp + (p % patches_per_batch) * knl_n * ts; + + for (int64_t ic = 0; ic < c_in; ++ic) { + for (int64_t ky = 0; ky < knl_h; ++ky) { ++ const int64_t sy = oy * stride_y + ky * dilation_y - pad_y; ++ const bool row = sy >= 0 && sy < src_h; ++ const char * src_row = src_base + sy * src->nb[1] + ic * src->nb[2]; + for (int64_t kx = 0; kx < knl_w; ++kx) { +- const int64_t sy = src_x * stride_y + ky * dilation_y - pad_y; +- const int64_t sx = src_y * stride_x + kx * dilation_x - pad_x; +- +- int64_t dst_idx = ic * (knl_h * knl_w) + ky * knl_w + kx; +- +- float src_val; +- if (sy < 0 || sy >= src_h || sx < 0 || sx >= src_w) { +- src_val = 0.0f; +- } else { +- const float * src_ptr = (const float *)((const char *)src_base + sx * src->nb[0] + sy * src->nb[1] + ic * src->nb[2]); +- src_val = *src_ptr; +- } +- +- char * element_ptr = dst_row + dst_idx * traits->type_size; +- if (kernel_type == GGML_TYPE_F32) { +- *(float *) element_ptr = src_val; +- } else if (kernel_type == GGML_TYPE_F16) { +- *(ggml_fp16_t *) element_ptr = GGML_CPU_FP32_TO_FP16(src_val); ++ char * dst_col = dst_base + (ic * (knl_h * knl_w) + ky * knl_w + kx) * ts; ++ for (int64_t j = 0; j < run; ++j) { ++ const int64_t sx = (ox0 + j) * stride_x + kx * dilation_x - pad_x; ++ float src_val = 0.0f; ++ if (row && sx >= 0 && sx < src_w) { ++ src_val = *(const float *) (src_row + sx * src->nb[0]); ++ } ++ char * element_ptr = dst_col + j * knl_n * ts; ++ if (kernel_type == GGML_TYPE_F32) { ++ *(float *) element_ptr = src_val; ++ } else { ++ *(ggml_fp16_t *) element_ptr = GGML_CPU_FP32_TO_FP16(src_val); ++ } + } + } + } + } ++ p += run; + } // patches handled by this thread + + ggml_barrier(params->threadpool); +@@ -7574,17 +7580,23 @@ + const int64_t permute_start = params->ith * permute_per_thread; + const int64_t permute_end = std::min(permute_start + permute_per_thread, patch_n); + +- for (int64_t i = permute_start; i < permute_end; ++i) { +- const int64_t p = patch_start_batch + i; +- const int64_t batch_n = p / (dst_w * dst_h); +- const int64_t dst_y = (p / dst_w) % dst_h; +- const int64_t dst_x = p % dst_w; ++ // Same run blocking for the transpose back: one output plane at a time per run, so each write is a contiguous ++ // row segment rather than one element in each of c_out planes. ++ for (int64_t i = permute_start; i < permute_end;) { ++ const int64_t p = patch_start_batch + i; ++ const int64_t n_idx = p / (dst_w * dst_h); ++ const int64_t oy = (p / dst_w) % dst_h; ++ const int64_t ox0 = p % dst_w; ++ const int64_t run = std::min({ permute_end - i, dst_w - ox0, (int64_t) CONV_2D_RUN }); + + for (int64_t oc = 0; oc < c_out; ++oc) { +- const float value = gemm_output[i * c_out + oc]; +- float * dst_ptr = (float *)((char *)dst_data + dst_x * dst->nb[0] + dst_y * dst->nb[1] + oc * dst->nb[2] + batch_n * dst->nb[3]); +- *dst_ptr = value; ++ char * dst_row = (char *) dst_data + ox0 * dst->nb[0] + oy * dst->nb[1] + oc * dst->nb[2] + n_idx * dst->nb[3]; ++ const float * col = gemm_output + i * c_out + oc; ++ for (int64_t j = 0; j < run; ++j) { ++ *(float *) (dst_row + j * dst->nb[0]) = col[j * c_out]; ++ } + } ++ i += run; + } + } + } +@@ -7662,45 +7674,48 @@ + const int64_t patch_start = patch_start_batch + params->ith * patch_per_thread; + const int64_t patch_end = std::min(patch_start + patch_per_thread, patch_end_batch); + +- for (int64_t p = patch_start; p < patch_end; ++p) { ++ // Runs of consecutive patches along one output row, as in conv_2d: each (ic, kz, ky) source row is read once ++ // per run. Same values and conversions as the per-patch loop (bit-identical). ++ const size_t ts = traits->type_size; ++ for (int64_t p = patch_start; p < patch_end;) { + const int64_t p_in_batch = p % (dst_w * dst_h * dst_d); + const int64_t p_in_depth = p_in_batch % (dst_w * dst_h); + const int64_t batch_idx = p / (dst_w * dst_h * dst_d); + const int64_t dst_z = p_in_batch / (dst_w * dst_h); + const int64_t dst_y = p_in_depth / dst_w; +- const int64_t dst_x = p_in_depth % dst_w; ++ const int64_t dst_x0 = p_in_depth % dst_w; ++ const int64_t run = std::min({ patch_end - p, dst_w - dst_x0, (int64_t) CONV_2D_RUN }); + +- char * dst_row = (char *) tmp + (p % patches_per_batch) * knl_n_total * traits->type_size; ++ char * dst_base = (char *) tmp + (p % patches_per_batch) * knl_n_total * ts; + + for (int64_t ic = 0; ic < c; ++ic) { ++ const int64_t cn_idx = batch_idx * c + ic; + for (int64_t kz = 0; kz < knl_d; ++kz) { ++ const int64_t sz = dst_z * s2 + kz * d2 - p2; + for (int64_t ky = 0; ky < knl_h; ++ky) { ++ const int64_t sy = dst_y * s1 + ky * d1 - p1; ++ const bool row = sz >= 0 && sz < src_d && sy >= 0 && sy < src_h; ++ const char * src_row = (const char *) src_data + sy * src->nb[1] + sz * src->nb[2] + cn_idx * src->nb[3]; + for (int64_t kx = 0; kx < knl_w; ++kx) { +- const int64_t sz = dst_z * s2 + kz * d2 - p2; +- const int64_t sy = dst_y * s1 + ky * d1 - p1; +- const int64_t sx = dst_x * s0 + kx * d0 - p0; +- +- int64_t dst_idx = ic * knl_n_per_channel + kz * (knl_h * knl_w) + ky * knl_w + kx; +- +- float src_val; +- if (sz < 0 || sz >= src_d || sy < 0 || sy >= src_h || sx < 0 || sx >= src_w) { +- src_val = 0.0f; +- } else { +- const int64_t cn_idx = batch_idx * c + ic; +- const float * src_ptr = (const float *)((const char *)src_data + sx*src->nb[0] + sy*src->nb[1] + sz*src->nb[2] + cn_idx*src->nb[3]); +- src_val = *src_ptr; +- } +- +- char * element_ptr = dst_row + dst_idx * traits->type_size; +- if (kernel_type == GGML_TYPE_F32) { +- *(float *)element_ptr = src_val; +- } else if (kernel_type == GGML_TYPE_F16) { +- *(ggml_fp16_t *)element_ptr = GGML_CPU_FP32_TO_FP16(src_val); ++ char * dst_col = dst_base + (ic * knl_n_per_channel + kz * (knl_h * knl_w) + ky * knl_w + kx) * ts; ++ for (int64_t j = 0; j < run; ++j) { ++ const int64_t sx = (dst_x0 + j) * s0 + kx * d0 - p0; ++ float src_val = 0.0f; ++ if (row && sx >= 0 && sx < src_w) { ++ src_val = *(const float *) (src_row + sx * src->nb[0]); ++ } ++ char * element_ptr = dst_col + j * knl_n_total * ts; ++ if (kernel_type == GGML_TYPE_F32) { ++ *(float *) element_ptr = src_val; ++ } else { ++ *(ggml_fp16_t *) element_ptr = GGML_CPU_FP32_TO_FP16(src_val); ++ } + } + } + } + } + } ++ p += run; + } + + ggml_barrier(params->threadpool); +@@ -7714,21 +7729,25 @@ + const int64_t permute_start = params->ith * permute_per_thread; + const int64_t permute_end = std::min(permute_start + permute_per_thread, patch_n_in_batch); + +- for (int64_t i = permute_start; i < permute_end; ++i) { ++ for (int64_t i = permute_start; i < permute_end;) { + const int64_t p = patch_start_batch + i; + const int64_t p_in_batch = p % (dst_w * dst_h * dst_d); + const int64_t p_in_depth = p_in_batch % (dst_w * dst_h); + const int64_t batch_idx = p / (dst_w * dst_h * dst_d); + const int64_t dst_z = p_in_batch / (dst_w * dst_h); + const int64_t dst_y = p_in_depth / dst_w; +- const int64_t dst_x = p_in_depth % dst_w; ++ const int64_t dst_x0 = p_in_depth % dst_w; ++ const int64_t run = std::min({ permute_end - i, dst_w - dst_x0, (int64_t) CONV_2D_RUN }); + + for (int64_t ioc = 0; ioc < oc; ++ioc) { +- const float value = gemm_output[i * oc + ioc]; + const int64_t ocn_idx = batch_idx * oc + ioc; +- float * dst_ptr = (float *)((char *)dst_data + dst_x*dst->nb[0] + dst_y*dst->nb[1] + dst_z*dst->nb[2] + ocn_idx*dst->nb[3]); +- *dst_ptr = value; ++ char * dst_row = (char *) dst_data + dst_x0 * dst->nb[0] + dst_y * dst->nb[1] + dst_z * dst->nb[2] + ocn_idx * dst->nb[3]; ++ const float * col = gemm_output + i * oc + ioc; ++ for (int64_t j = 0; j < run; ++j) { ++ *(float *) (dst_row + j * dst->nb[0]) = col[j * oc]; ++ } + } ++ i += run; + } + } + } From 0569826a9f72d34ea6e17aa960736589f439d9af Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 22 Oct 2026 05:13:22 -0700 Subject: [PATCH 3/6] ggml-patches/0008: keep the AVX512 CPU variants on Intel CPUs without AVX-VNNI 0008 dropped the AVX512 variants on every CPU without AVX-VNNI because Zen 4 ran them 15 to 50% slower than AVX2. That also took them from Intel Skylake-SP, Cascade Lake and Ice Lake, which run them much faster: on a 6-core Cascade Lake VM the cascadelake variant (AVX512-VNNI) cut Z-Image Turbo sampling from 50.2 s to 29.3 s and FLUX.2 klein from 63.8 s to 39.3 s against haswell. Only AMD without AVX-VNNI (Zen 4) keeps the AVX2 variant now. --- .../0008-ggml-cpu-avx512-variants-need-avx-vnni.patch | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/unsloth/ggml-patches/0008-ggml-cpu-avx512-variants-need-avx-vnni.patch b/scripts/unsloth/ggml-patches/0008-ggml-cpu-avx512-variants-need-avx-vnni.patch index b3e8b9a81..6304b90ed 100644 --- a/scripts/unsloth/ggml-patches/0008-ggml-cpu-avx512-variants-need-avx-vnni.patch +++ b/scripts/unsloth/ggml-patches/0008-ggml-cpu-avx512-variants-need-avx-vnni.patch @@ -2,14 +2,15 @@ diff --git a/src/ggml-cpu/arch/x86/cpu-feats.cpp b/src/ggml-cpu/arch/x86/cpu-fea index d775a036..a29335b9 100644 --- a/src/ggml-cpu/arch/x86/cpu-feats.cpp +++ b/src/ggml-cpu/arch/x86/cpu-feats.cpp -@@ -300,6 +300,10 @@ static int ggml_backend_cpu_x86_score() { +@@ -300,6 +300,11 @@ static int ggml_backend_cpu_x86_score() { if (!is.AVX512VL()) { return 0; } if (!is.AVX512DQ()) { return 0; } if (!is.AVX512BW()) { return 0; } -+ // Unsloth: the AVX512 variants are only offered on CPUs that also have AVX-VNNI (Zen 5, -+ // Sapphire Rapids and newer). On Zen 4, which has AVX512 but not AVX-VNNI, they ran -+ // stable-diffusion.cpp 15 to 50% slower than the AVX2 variant; the AVX2 one is picked there. -+ if (!is.AVX_VNNI()) { return 0; } ++ // Unsloth: on AMD the AVX512 variants are only offered with AVX-VNNI (Zen 5). On Zen 4, which ++ // has AVX512 but not AVX-VNNI, they ran stable-diffusion.cpp 15 to 50% slower than the AVX2 ++ // variant, so the AVX2 one is picked there. Intel AVX512 CPUs without AVX-VNNI (Skylake-SP, ++ // Cascade Lake, Ice Lake) keep them: cascadelake ran Z-Image 1.7x faster than haswell. ++ if (is.is_amd && !is.AVX_VNNI()) { return 0; } score += 1<<7; #endif #ifdef GGML_AVX512_VBMI From a162a5499eaa7c995c0d51ff6a31717bf7ef9b07 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 22 Oct 2026 05:13:22 -0700 Subject: [PATCH 4/6] CPU: direct convolution by default (bit-identical, no full im2col buffer) --conv-direct auto now picks the direct CONV_2D / CONV_3D kernels on the CPU backend. ggml-cpu computes them with the same fp16 patches and the same GEMM as im2col + mul_mat, so outputs are bit-identical, but through a fixed 16 MB patch buffer instead of the whole im2col matrix (2.8 GB for one 128-channel conv of a 1024x1024 VAE decode, 5.6 GB for a 256-channel one). With the row-run fill from the previous commit they are also faster than im2col. Direct convolution is only taken for F16 / F32 weights, which is all the direct kernels accept; any other weight type keeps im2col instead of aborting in ggml-cpu (also for an explicit --conv-direct on). --- src/core/ggml_extend.cpp | 5 +++-- src/core/probe/conv_direct.cpp | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/core/ggml_extend.cpp b/src/core/ggml_extend.cpp index 80e2121c6..46b8e7c54 100644 --- a/src/core/ggml_extend.cpp +++ b/src/core/ggml_extend.cpp @@ -460,7 +460,8 @@ ggml_tensor* ggml_ext_conv_2d(ggml_context* ctx, p1 = 0; } - if (direct) { + // The direct kernels take F16 / F32 weights only (ggml-cpu asserts on anything else); other types keep im2col. + if (direct && (w->type == GGML_TYPE_F16 || w->type == GGML_TYPE_F32)) { x = ggml_conv_2d_direct(ctx, w, x, s0, s1, p0, p1, d0, d1); } else { // ggml_conv_2d, but skips the final [OC, N] -> [N, OC] relayout when N == 1 (no data moves). @@ -503,7 +504,7 @@ ggml_tensor* ggml_ext_conv_3d(ggml_context* ctx, if (scale != 1.f) { x = ggml_ext_scale(ctx, x, scale); } - if (direct) { + if (direct && (w->type == GGML_TYPE_F16 || w->type == GGML_TYPE_F32)) { int64_t OC = w->ne[3] / IC; int64_t N = x->ne[3] / IC; x = ggml_conv_3d_direct(ctx, w, x, s0, s1, s2, p0, p1, p2, d0, d1, d2, (int)IC, (int)N, (int)OC); diff --git a/src/core/probe/conv_direct.cpp b/src/core/probe/conv_direct.cpp index 7f4666a41..ae9f46f39 100644 --- a/src/core/probe/conv_direct.cpp +++ b/src/core/probe/conv_direct.cpp @@ -83,6 +83,13 @@ bool sd_probe_conv_direct(ggml_backend_t backend, bool conv3d) { dims, name, result.candidate_faster ? "on" : "off", result.reference_ms, result.candidate_ms); return result.candidate_faster; } + if (sd_backend_is_cpu(backend)) { + // ggml-cpu's direct kernels produce the same values as im2col + mul_mat (bit-identical) and work through a + // fixed 16 MB patch buffer instead of materialising the whole im2col matrix, which in a VAE decoder at + // 1024x1024 is 2.8 GB for one 128-channel conv and 5.6 GB for a 256-channel one. + LOG_INFO("conv%dd direct auto: %s -> on (CPU)", dims, name); + return true; + } if (sd_backend_supports_conv_implicit_gemm(backend, conv3d)) { LOG_INFO("conv%dd direct auto: %s -> on (implicit GEMM)", dims, name); return true; From f38a4dad43d2ab5b8c2eed0644a164d2eee1587a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 22 Oct 2026 05:54:51 -0700 Subject: [PATCH 5/6] ggml-patches/0014: no row pointer for a source row outside the input --- .../ggml-patches/0014-ggml-cpu-conv-direct-row-runs.patch | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/unsloth/ggml-patches/0014-ggml-cpu-conv-direct-row-runs.patch b/scripts/unsloth/ggml-patches/0014-ggml-cpu-conv-direct-row-runs.patch index 141fe1fe1..820a48dc2 100644 --- a/scripts/unsloth/ggml-patches/0014-ggml-cpu-conv-direct-row-runs.patch +++ b/scripts/unsloth/ggml-patches/0014-ggml-cpu-conv-direct-row-runs.patch @@ -39,7 +39,7 @@ diff --git a/src/ggml-cpu/ops.cpp b/src/ggml-cpu/ops.cpp for (int64_t ky = 0; ky < knl_h; ++ky) { + const int64_t sy = oy * stride_y + ky * dilation_y - pad_y; + const bool row = sy >= 0 && sy < src_h; -+ const char * src_row = src_base + sy * src->nb[1] + ic * src->nb[2]; ++ const char * src_row = row ? src_base + sy * src->nb[1] + ic * src->nb[2] : nullptr; for (int64_t kx = 0; kx < knl_w; ++kx) { - const int64_t sy = src_x * stride_y + ky * dilation_y - pad_y; - const int64_t sx = src_y * stride_x + kx * dilation_x - pad_x; @@ -140,7 +140,7 @@ diff --git a/src/ggml-cpu/ops.cpp b/src/ggml-cpu/ops.cpp for (int64_t ky = 0; ky < knl_h; ++ky) { + const int64_t sy = dst_y * s1 + ky * d1 - p1; + const bool row = sz >= 0 && sz < src_d && sy >= 0 && sy < src_h; -+ const char * src_row = (const char *) src_data + sy * src->nb[1] + sz * src->nb[2] + cn_idx * src->nb[3]; ++ const char * src_row = row ? (const char *) src_data + sy * src->nb[1] + sz * src->nb[2] + cn_idx * src->nb[3] : nullptr; for (int64_t kx = 0; kx < knl_w; ++kx) { - const int64_t sz = dst_z * s2 + kz * d2 - p2; - const int64_t sy = dst_y * s1 + ky * d1 - p1; From e181faddf41c15728a08df57347dfa11b7ffd216 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 22 Oct 2026 22:12:13 -0700 Subject: [PATCH 6/6] CPU conv direct auto: only with the row-run kernels; dynamic affinity mask - ggml-patches/0014 also defines GGML_CPU_CONV_DIRECT_ROW_RUNS in ggml-cpu.h. CPU auto selects direct convolution only when it is defined, so a source build without the fork's ggml patches keeps the old off default instead of the slower per-element direct loops. - The comment and docs now say bit-identical holds for F16 conv weights (every Conv2d / Conv3d block); F32 LoRA tensors and depthwise convs keep F32 activations in the direct kernels and come out closer to exact. - sd_get_num_physical_cores sizes the affinity mask with CPU_ALLOC and grows it on EINVAL, for kernels built for more than 1024 CPUs. - docs/backend.md and docs/performance.md describe the CPU auto rule. --- docs/backend.md | 10 ++++--- docs/performance.md | 6 +++-- .../0014-ggml-cpu-conv-direct-row-runs.patch | 14 ++++++++++ src/core/probe/conv_direct.cpp | 16 +++++++++--- src/core/util.cpp | 26 +++++++++++++++---- 5 files changed, 59 insertions(+), 13 deletions(-) diff --git a/docs/backend.md b/docs/backend.md index 057d3f20e..45175135e 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -241,9 +241,13 @@ sd-cli -m model.safetensors -p "a cat" --conv-direct off,diffusion=on CUDA 2D requires a compiled Turing-or-newer MMA architecture; HIP 2D requires the supported CDNA MFMA or RDNA WMMA architectures described below for flash attention. MUSA 2D has a SIMT implementation. CUDA, HIP, and MUSA 3D also have - a SIMT implementation, so they do not require MMA. Other backends default to - `off`. Results and decision logs are cached per compute backend and dimension - in the context, as with flash attention. Each dimension is selected independently, + a SIMT implementation, so they do not require MMA. The CPU backend selects + `on` for 2D and 3D when ggml carries the row-run direct kernels + (`scripts/unsloth/ggml-patches/0014`, applied by the prebuilt workflow) and + `off` otherwise: the direct kernels use a fixed patch buffer instead of the + full im2col matrix and match im2col bit for bit with F16 weights. Other + backends default to `off`. Results and decision logs are cached per compute + backend and dimension in the context, as with flash attention. Each dimension is selected independently, and all compute devices in a module must qualify. Parameter-storage devices do not affect this choice. diff --git a/docs/performance.md b/docs/performance.md index 18491039e..2d8440241 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -32,8 +32,10 @@ segment count change. path. Performance depends on GPU architecture, backend, driver, model, and tensor shapes, so enabling direct convolution can either improve or reduce speed. -Automatic selection uses small benchmarks on Vulkan and implicit GEMM checks -on other backends. It does not guarantee the fastest end-to-end generation. +Automatic selection uses small benchmarks on Vulkan, implicit GEMM checks on +CUDA/HIP/MUSA, and the direct kernels on CPU when ggml carries the row-run +patch (see [backend selection](backend.md)). It does not guarantee the fastest +end-to-end generation. Compare `on` and `off` with the same model and generation settings over repeated runs, keeping flash attention unchanged and separating startup/probe time from steady-state generation time. See [backend selection](backend.md) diff --git a/scripts/unsloth/ggml-patches/0014-ggml-cpu-conv-direct-row-runs.patch b/scripts/unsloth/ggml-patches/0014-ggml-cpu-conv-direct-row-runs.patch index 820a48dc2..310e1feec 100644 --- a/scripts/unsloth/ggml-patches/0014-ggml-cpu-conv-direct-row-runs.patch +++ b/scripts/unsloth/ggml-patches/0014-ggml-cpu-conv-direct-row-runs.patch @@ -1,3 +1,17 @@ +diff --git a/include/ggml-cpu.h b/include/ggml-cpu.h +--- a/include/ggml-cpu.h ++++ b/include/ggml-cpu.h +@@ -3,6 +3,10 @@ + #include "ggml.h" + #include "ggml-backend.h" + ++// The direct conv_2d / conv_3d kernels fill and scatter their patches in row runs (bit-identical to the ++// per-element loops); stable-diffusion.cpp defaults CPU convolution to direct only when this is defined. ++#define GGML_CPU_CONV_DIRECT_ROW_RUNS 1 ++ + #ifdef __cplusplus + extern "C" { + #endif diff --git a/src/ggml-cpu/ops.cpp b/src/ggml-cpu/ops.cpp --- a/src/ggml-cpu/ops.cpp +++ b/src/ggml-cpu/ops.cpp diff --git a/src/core/probe/conv_direct.cpp b/src/core/probe/conv_direct.cpp index ae9f46f39..d6060856d 100644 --- a/src/core/probe/conv_direct.cpp +++ b/src/core/probe/conv_direct.cpp @@ -8,6 +8,7 @@ #include "core/ggml_extend_backend.h" #include "core/probe/backend_probe.h" #include "core/util.h" +#include "ggml-cpu.h" static void build_conv_direct_probe(BackendProbeGraph& graph, bool conv3d, bool direct, int size = 32, int channels = 64) { if (conv3d) { @@ -84,11 +85,20 @@ bool sd_probe_conv_direct(ggml_backend_t backend, bool conv3d) { return result.candidate_faster; } if (sd_backend_is_cpu(backend)) { - // ggml-cpu's direct kernels produce the same values as im2col + mul_mat (bit-identical) and work through a - // fixed 16 MB patch buffer instead of materialising the whole im2col matrix, which in a VAE decoder at - // 1024x1024 is 2.8 GB for one 128-channel conv and 5.6 GB for a 256-channel one. +#ifdef GGML_CPU_CONV_DIRECT_ROW_RUNS + // ggml-cpu's direct kernels work through a fixed 16 MB patch buffer instead of materialising the whole + // im2col matrix, which in a VAE decoder at 1024x1024 is 2.8 GB for one 128-channel conv and 5.6 GB for a + // 256-channel one. With F16 weights (every Conv2d / Conv3d block) they produce the same values as im2col + + // mul_mat, bit for bit; F32 weights (runtime LoRA tensors) and depthwise convs keep F32 activations where + // im2col rounds them to F16, so those come out closer to the exact result rather than identical. LOG_INFO("conv%dd direct auto: %s -> on (CPU)", dims, name); return true; +#else + // Without the row-run kernels (scripts/unsloth/ggml-patches/0014, applied by the prebuilt workflow) the + // per-element direct loops are slower than im2col on large convolutions. + LOG_INFO("conv%dd direct auto: %s -> off (CPU, ggml without the row-run direct kernels)", dims, name); + return false; +#endif } if (sd_backend_supports_conv_implicit_gemm(backend, conv3d)) { LOG_INFO("conv%dd direct auto: %s -> on (implicit GEMM)", dims, name); diff --git a/src/core/util.cpp b/src/core/util.cpp index 70ee57d9d..080ca5341 100644 --- a/src/core/util.cpp +++ b/src/core/util.cpp @@ -30,6 +30,7 @@ #endif #ifdef __linux__ +#include #include #endif @@ -347,13 +348,26 @@ int32_t sd_get_num_physical_cores() { // One thread per physical core this process may run on: the distinct thread-sibling sets among the CPUs in // the affinity mask (a taskset / container cpuset leaves fewer CPUs than the machine has). Upstream's loop // counts every CPU of the machine, so a 4-CPU cpuset on a 192-CPU host still gets 96 threads. - cpu_set_t mask; - CPU_ZERO(&mask); - if (sched_getaffinity(0, sizeof(mask), &mask) == 0) { + // The mask is sized dynamically: a kernel built for more than CPU_SETSIZE (1024) CPUs rejects a smaller one + // with EINVAL, so grow it until the call succeeds. + for (int ncpus = CPU_SETSIZE; ncpus <= (1 << 20); ncpus *= 2) { + cpu_set_t* mask = CPU_ALLOC(ncpus); + const size_t size = CPU_ALLOC_SIZE(ncpus); + if (mask == nullptr) { + break; + } + CPU_ZERO_S(size, mask); + if (sched_getaffinity(0, size, mask) != 0) { + CPU_FREE(mask); + if (errno == EINVAL) { + continue; + } + break; + } std::unordered_set siblings; int allowed = 0; - for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { - if (!CPU_ISSET(cpu, &mask)) { + for (int cpu = 0; cpu < ncpus; ++cpu) { + if (!CPU_ISSET_S(cpu, size, mask)) { continue; } allowed++; @@ -363,6 +377,7 @@ int32_t sd_get_num_physical_cores() { siblings.insert(line); } } + CPU_FREE(mask); if (!siblings.empty()) { return static_cast(siblings.size()); } @@ -370,6 +385,7 @@ int32_t sd_get_num_physical_cores() { // No topology in sysfs (some containers): assume 2-way SMT above 4 CPUs, as below. return allowed <= 4 ? allowed : allowed / 2; } + break; } #elif defined(__APPLE__) && defined(__MACH__) int32_t num_physical_cores;