diff --git a/docs/backend.md b/docs/backend.md index 057d3f20e..45175135e 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -241,9 +241,13 @@ sd-cli -m model.safetensors -p "a cat" --conv-direct off,diffusion=on CUDA 2D requires a compiled Turing-or-newer MMA architecture; HIP 2D requires the supported CDNA MFMA or RDNA WMMA architectures described below for flash attention. MUSA 2D has a SIMT implementation. CUDA, HIP, and MUSA 3D also have - a SIMT implementation, so they do not require MMA. Other backends default to - `off`. Results and decision logs are cached per compute backend and dimension - in the context, as with flash attention. Each dimension is selected independently, + a SIMT implementation, so they do not require MMA. The CPU backend selects + `on` for 2D and 3D when ggml carries the row-run direct kernels + (`scripts/unsloth/ggml-patches/0014`, applied by the prebuilt workflow) and + `off` otherwise: the direct kernels use a fixed patch buffer instead of the + full im2col matrix and match im2col bit for bit with F16 weights. Other + backends default to `off`. Results and decision logs are cached per compute + backend and dimension in the context, as with flash attention. Each dimension is selected independently, and all compute devices in a module must qualify. Parameter-storage devices do not affect this choice. diff --git a/docs/performance.md b/docs/performance.md index 18491039e..2d8440241 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -32,8 +32,10 @@ segment count change. path. Performance depends on GPU architecture, backend, driver, model, and tensor shapes, so enabling direct convolution can either improve or reduce speed. -Automatic selection uses small benchmarks on Vulkan and implicit GEMM checks -on other backends. It does not guarantee the fastest end-to-end generation. +Automatic selection uses small benchmarks on Vulkan, implicit GEMM checks on +CUDA/HIP/MUSA, and the direct kernels on CPU when ggml carries the row-run +patch (see [backend selection](backend.md)). It does not guarantee the fastest +end-to-end generation. Compare `on` and `off` with the same model and generation settings over repeated runs, keeping flash attention unchanged and separating startup/probe time from steady-state generation time. See [backend selection](backend.md) diff --git a/scripts/unsloth/ggml-patches/0008-ggml-cpu-avx512-variants-need-avx-vnni.patch b/scripts/unsloth/ggml-patches/0008-ggml-cpu-avx512-variants-need-avx-vnni.patch index b3e8b9a81..6304b90ed 100644 --- a/scripts/unsloth/ggml-patches/0008-ggml-cpu-avx512-variants-need-avx-vnni.patch +++ b/scripts/unsloth/ggml-patches/0008-ggml-cpu-avx512-variants-need-avx-vnni.patch @@ -2,14 +2,15 @@ diff --git a/src/ggml-cpu/arch/x86/cpu-feats.cpp b/src/ggml-cpu/arch/x86/cpu-fea index d775a036..a29335b9 100644 --- a/src/ggml-cpu/arch/x86/cpu-feats.cpp +++ b/src/ggml-cpu/arch/x86/cpu-feats.cpp -@@ -300,6 +300,10 @@ static int ggml_backend_cpu_x86_score() { +@@ -300,6 +300,11 @@ static int ggml_backend_cpu_x86_score() { if (!is.AVX512VL()) { return 0; } if (!is.AVX512DQ()) { return 0; } if (!is.AVX512BW()) { return 0; } -+ // Unsloth: the AVX512 variants are only offered on CPUs that also have AVX-VNNI (Zen 5, -+ // Sapphire Rapids and newer). On Zen 4, which has AVX512 but not AVX-VNNI, they ran -+ // stable-diffusion.cpp 15 to 50% slower than the AVX2 variant; the AVX2 one is picked there. -+ if (!is.AVX_VNNI()) { return 0; } ++ // Unsloth: on AMD the AVX512 variants are only offered with AVX-VNNI (Zen 5). On Zen 4, which ++ // has AVX512 but not AVX-VNNI, they ran stable-diffusion.cpp 15 to 50% slower than the AVX2 ++ // variant, so the AVX2 one is picked there. Intel AVX512 CPUs without AVX-VNNI (Skylake-SP, ++ // Cascade Lake, Ice Lake) keep them: cascadelake ran Z-Image 1.7x faster than haswell. ++ if (is.is_amd && !is.AVX_VNNI()) { return 0; } score += 1<<7; #endif #ifdef GGML_AVX512_VBMI diff --git a/scripts/unsloth/ggml-patches/0014-ggml-cpu-conv-direct-row-runs.patch b/scripts/unsloth/ggml-patches/0014-ggml-cpu-conv-direct-row-runs.patch new file mode 100644 index 000000000..310e1feec --- /dev/null +++ b/scripts/unsloth/ggml-patches/0014-ggml-cpu-conv-direct-row-runs.patch @@ -0,0 +1,231 @@ +diff --git a/include/ggml-cpu.h b/include/ggml-cpu.h +--- a/include/ggml-cpu.h ++++ b/include/ggml-cpu.h +@@ -3,6 +3,10 @@ + #include "ggml.h" + #include "ggml-backend.h" + ++// The direct conv_2d / conv_3d kernels fill and scatter their patches in row runs (bit-identical to the ++// per-element loops); stable-diffusion.cpp defaults CPU convolution to direct only when this is defined. ++#define GGML_CPU_CONV_DIRECT_ROW_RUNS 1 ++ + #ifdef __cplusplus + extern "C" { + #endif +diff --git a/src/ggml-cpu/ops.cpp b/src/ggml-cpu/ops.cpp +--- a/src/ggml-cpu/ops.cpp ++++ b/src/ggml-cpu/ops.cpp +@@ -7464,6 +7464,8 @@ + // ggml_compute_forward_conv_2d + + ++#define CONV_2D_RUN 64 ++ + static void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params, + const ggml_tensor * kernel, // [KW, KH, IC, OC] + const ggml_tensor * src, // [W, H, C, N] +@@ -7521,40 +7523,44 @@ + const int64_t patch_start = patch_start_batch + params->ith * patch_per_thread; + const int64_t patch_end = std::min(patch_start + patch_per_thread, patch_end_batch); + +- //im2col for a patch +- for (int64_t p = patch_start; p < patch_end; ++p) { +- const int64_t batch_n = p / (dst_w * dst_h); +- const int64_t src_x = (p / dst_w) % dst_h; +- const int64_t src_y = p % dst_w; ++ // im2col for this thread's patches, a run of up to CONV_2D_RUN consecutive patches of one output row at a ++ // time: each (ic, ky) source row is then read once, contiguously, for the whole run instead of once per patch ++ // (one patch touches c_in planes; one plane per patch was a cache and TLB miss per element). Same values, ++ // same conversions as the per-patch loop, so the result is bit-identical. ++ const size_t ts = traits->type_size; ++ for (int64_t p = patch_start; p < patch_end;) { ++ const int64_t n_idx = p / (dst_w * dst_h); ++ const int64_t oy = (p / dst_w) % dst_h; ++ const int64_t ox0 = p % dst_w; ++ const int64_t run = std::min({ patch_end - p, dst_w - ox0, (int64_t) CONV_2D_RUN }); + +- const float * src_base = (const float *)((const char *)src_data + batch_n * src->nb[3]); +- char * dst_row = (char *) tmp + (p % patches_per_batch) * knl_n * traits->type_size; ++ const char * src_base = (const char *) src_data + n_idx * src->nb[3]; ++ char * dst_base = (char *) tmp + (p % patches_per_batch) * knl_n * ts; + + for (int64_t ic = 0; ic < c_in; ++ic) { + for (int64_t ky = 0; ky < knl_h; ++ky) { ++ const int64_t sy = oy * stride_y + ky * dilation_y - pad_y; ++ const bool row = sy >= 0 && sy < src_h; ++ const char * src_row = row ? src_base + sy * src->nb[1] + ic * src->nb[2] : nullptr; + for (int64_t kx = 0; kx < knl_w; ++kx) { +- const int64_t sy = src_x * stride_y + ky * dilation_y - pad_y; +- const int64_t sx = src_y * stride_x + kx * dilation_x - pad_x; +- +- int64_t dst_idx = ic * (knl_h * knl_w) + ky * knl_w + kx; +- +- float src_val; +- if (sy < 0 || sy >= src_h || sx < 0 || sx >= src_w) { +- src_val = 0.0f; +- } else { +- const float * src_ptr = (const float *)((const char *)src_base + sx * src->nb[0] + sy * src->nb[1] + ic * src->nb[2]); +- src_val = *src_ptr; +- } +- +- char * element_ptr = dst_row + dst_idx * traits->type_size; +- if (kernel_type == GGML_TYPE_F32) { +- *(float *) element_ptr = src_val; +- } else if (kernel_type == GGML_TYPE_F16) { +- *(ggml_fp16_t *) element_ptr = GGML_CPU_FP32_TO_FP16(src_val); ++ char * dst_col = dst_base + (ic * (knl_h * knl_w) + ky * knl_w + kx) * ts; ++ for (int64_t j = 0; j < run; ++j) { ++ const int64_t sx = (ox0 + j) * stride_x + kx * dilation_x - pad_x; ++ float src_val = 0.0f; ++ if (row && sx >= 0 && sx < src_w) { ++ src_val = *(const float *) (src_row + sx * src->nb[0]); ++ } ++ char * element_ptr = dst_col + j * knl_n * ts; ++ if (kernel_type == GGML_TYPE_F32) { ++ *(float *) element_ptr = src_val; ++ } else { ++ *(ggml_fp16_t *) element_ptr = GGML_CPU_FP32_TO_FP16(src_val); ++ } + } + } + } + } ++ p += run; + } // patches handled by this thread + + ggml_barrier(params->threadpool); +@@ -7574,17 +7580,23 @@ + const int64_t permute_start = params->ith * permute_per_thread; + const int64_t permute_end = std::min(permute_start + permute_per_thread, patch_n); + +- for (int64_t i = permute_start; i < permute_end; ++i) { +- const int64_t p = patch_start_batch + i; +- const int64_t batch_n = p / (dst_w * dst_h); +- const int64_t dst_y = (p / dst_w) % dst_h; +- const int64_t dst_x = p % dst_w; ++ // Same run blocking for the transpose back: one output plane at a time per run, so each write is a contiguous ++ // row segment rather than one element in each of c_out planes. ++ for (int64_t i = permute_start; i < permute_end;) { ++ const int64_t p = patch_start_batch + i; ++ const int64_t n_idx = p / (dst_w * dst_h); ++ const int64_t oy = (p / dst_w) % dst_h; ++ const int64_t ox0 = p % dst_w; ++ const int64_t run = std::min({ permute_end - i, dst_w - ox0, (int64_t) CONV_2D_RUN }); + + for (int64_t oc = 0; oc < c_out; ++oc) { +- const float value = gemm_output[i * c_out + oc]; +- float * dst_ptr = (float *)((char *)dst_data + dst_x * dst->nb[0] + dst_y * dst->nb[1] + oc * dst->nb[2] + batch_n * dst->nb[3]); +- *dst_ptr = value; ++ char * dst_row = (char *) dst_data + ox0 * dst->nb[0] + oy * dst->nb[1] + oc * dst->nb[2] + n_idx * dst->nb[3]; ++ const float * col = gemm_output + i * c_out + oc; ++ for (int64_t j = 0; j < run; ++j) { ++ *(float *) (dst_row + j * dst->nb[0]) = col[j * c_out]; ++ } + } ++ i += run; + } + } + } +@@ -7662,45 +7674,48 @@ + const int64_t patch_start = patch_start_batch + params->ith * patch_per_thread; + const int64_t patch_end = std::min(patch_start + patch_per_thread, patch_end_batch); + +- for (int64_t p = patch_start; p < patch_end; ++p) { ++ // Runs of consecutive patches along one output row, as in conv_2d: each (ic, kz, ky) source row is read once ++ // per run. Same values and conversions as the per-patch loop (bit-identical). ++ const size_t ts = traits->type_size; ++ for (int64_t p = patch_start; p < patch_end;) { + const int64_t p_in_batch = p % (dst_w * dst_h * dst_d); + const int64_t p_in_depth = p_in_batch % (dst_w * dst_h); + const int64_t batch_idx = p / (dst_w * dst_h * dst_d); + const int64_t dst_z = p_in_batch / (dst_w * dst_h); + const int64_t dst_y = p_in_depth / dst_w; +- const int64_t dst_x = p_in_depth % dst_w; ++ const int64_t dst_x0 = p_in_depth % dst_w; ++ const int64_t run = std::min({ patch_end - p, dst_w - dst_x0, (int64_t) CONV_2D_RUN }); + +- char * dst_row = (char *) tmp + (p % patches_per_batch) * knl_n_total * traits->type_size; ++ char * dst_base = (char *) tmp + (p % patches_per_batch) * knl_n_total * ts; + + for (int64_t ic = 0; ic < c; ++ic) { ++ const int64_t cn_idx = batch_idx * c + ic; + for (int64_t kz = 0; kz < knl_d; ++kz) { ++ const int64_t sz = dst_z * s2 + kz * d2 - p2; + for (int64_t ky = 0; ky < knl_h; ++ky) { ++ const int64_t sy = dst_y * s1 + ky * d1 - p1; ++ const bool row = sz >= 0 && sz < src_d && sy >= 0 && sy < src_h; ++ const char * src_row = row ? (const char *) src_data + sy * src->nb[1] + sz * src->nb[2] + cn_idx * src->nb[3] : nullptr; + for (int64_t kx = 0; kx < knl_w; ++kx) { +- const int64_t sz = dst_z * s2 + kz * d2 - p2; +- const int64_t sy = dst_y * s1 + ky * d1 - p1; +- const int64_t sx = dst_x * s0 + kx * d0 - p0; +- +- int64_t dst_idx = ic * knl_n_per_channel + kz * (knl_h * knl_w) + ky * knl_w + kx; +- +- float src_val; +- if (sz < 0 || sz >= src_d || sy < 0 || sy >= src_h || sx < 0 || sx >= src_w) { +- src_val = 0.0f; +- } else { +- const int64_t cn_idx = batch_idx * c + ic; +- const float * src_ptr = (const float *)((const char *)src_data + sx*src->nb[0] + sy*src->nb[1] + sz*src->nb[2] + cn_idx*src->nb[3]); +- src_val = *src_ptr; +- } +- +- char * element_ptr = dst_row + dst_idx * traits->type_size; +- if (kernel_type == GGML_TYPE_F32) { +- *(float *)element_ptr = src_val; +- } else if (kernel_type == GGML_TYPE_F16) { +- *(ggml_fp16_t *)element_ptr = GGML_CPU_FP32_TO_FP16(src_val); ++ char * dst_col = dst_base + (ic * knl_n_per_channel + kz * (knl_h * knl_w) + ky * knl_w + kx) * ts; ++ for (int64_t j = 0; j < run; ++j) { ++ const int64_t sx = (dst_x0 + j) * s0 + kx * d0 - p0; ++ float src_val = 0.0f; ++ if (row && sx >= 0 && sx < src_w) { ++ src_val = *(const float *) (src_row + sx * src->nb[0]); ++ } ++ char * element_ptr = dst_col + j * knl_n_total * ts; ++ if (kernel_type == GGML_TYPE_F32) { ++ *(float *) element_ptr = src_val; ++ } else { ++ *(ggml_fp16_t *) element_ptr = GGML_CPU_FP32_TO_FP16(src_val); ++ } + } + } + } + } + } ++ p += run; + } + + ggml_barrier(params->threadpool); +@@ -7714,21 +7729,25 @@ + const int64_t permute_start = params->ith * permute_per_thread; + const int64_t permute_end = std::min(permute_start + permute_per_thread, patch_n_in_batch); + +- for (int64_t i = permute_start; i < permute_end; ++i) { ++ for (int64_t i = permute_start; i < permute_end;) { + const int64_t p = patch_start_batch + i; + const int64_t p_in_batch = p % (dst_w * dst_h * dst_d); + const int64_t p_in_depth = p_in_batch % (dst_w * dst_h); + const int64_t batch_idx = p / (dst_w * dst_h * dst_d); + const int64_t dst_z = p_in_batch / (dst_w * dst_h); + const int64_t dst_y = p_in_depth / dst_w; +- const int64_t dst_x = p_in_depth % dst_w; ++ const int64_t dst_x0 = p_in_depth % dst_w; ++ const int64_t run = std::min({ permute_end - i, dst_w - dst_x0, (int64_t) CONV_2D_RUN }); + + for (int64_t ioc = 0; ioc < oc; ++ioc) { +- const float value = gemm_output[i * oc + ioc]; + const int64_t ocn_idx = batch_idx * oc + ioc; +- float * dst_ptr = (float *)((char *)dst_data + dst_x*dst->nb[0] + dst_y*dst->nb[1] + dst_z*dst->nb[2] + ocn_idx*dst->nb[3]); +- *dst_ptr = value; ++ char * dst_row = (char *) dst_data + dst_x0 * dst->nb[0] + dst_y * dst->nb[1] + dst_z * dst->nb[2] + ocn_idx * dst->nb[3]; ++ const float * col = gemm_output + i * oc + ioc; ++ for (int64_t j = 0; j < run; ++j) { ++ *(float *) (dst_row + j * dst->nb[0]) = col[j * oc]; ++ } + } ++ i += run; + } + } + } diff --git a/src/core/ggml_extend.cpp b/src/core/ggml_extend.cpp index 80e2121c6..46b8e7c54 100644 --- a/src/core/ggml_extend.cpp +++ b/src/core/ggml_extend.cpp @@ -460,7 +460,8 @@ ggml_tensor* ggml_ext_conv_2d(ggml_context* ctx, p1 = 0; } - if (direct) { + // The direct kernels take F16 / F32 weights only (ggml-cpu asserts on anything else); other types keep im2col. + if (direct && (w->type == GGML_TYPE_F16 || w->type == GGML_TYPE_F32)) { x = ggml_conv_2d_direct(ctx, w, x, s0, s1, p0, p1, d0, d1); } else { // ggml_conv_2d, but skips the final [OC, N] -> [N, OC] relayout when N == 1 (no data moves). @@ -503,7 +504,7 @@ ggml_tensor* ggml_ext_conv_3d(ggml_context* ctx, if (scale != 1.f) { x = ggml_ext_scale(ctx, x, scale); } - if (direct) { + if (direct && (w->type == GGML_TYPE_F16 || w->type == GGML_TYPE_F32)) { int64_t OC = w->ne[3] / IC; int64_t N = x->ne[3] / IC; x = ggml_conv_3d_direct(ctx, w, x, s0, s1, s2, p0, p1, p2, d0, d1, d2, (int)IC, (int)N, (int)OC); diff --git a/src/core/probe/conv_direct.cpp b/src/core/probe/conv_direct.cpp index 7f4666a41..d6060856d 100644 --- a/src/core/probe/conv_direct.cpp +++ b/src/core/probe/conv_direct.cpp @@ -8,6 +8,7 @@ #include "core/ggml_extend_backend.h" #include "core/probe/backend_probe.h" #include "core/util.h" +#include "ggml-cpu.h" static void build_conv_direct_probe(BackendProbeGraph& graph, bool conv3d, bool direct, int size = 32, int channels = 64) { if (conv3d) { @@ -83,6 +84,22 @@ bool sd_probe_conv_direct(ggml_backend_t backend, bool conv3d) { dims, name, result.candidate_faster ? "on" : "off", result.reference_ms, result.candidate_ms); return result.candidate_faster; } + if (sd_backend_is_cpu(backend)) { +#ifdef GGML_CPU_CONV_DIRECT_ROW_RUNS + // ggml-cpu's direct kernels work through a fixed 16 MB patch buffer instead of materialising the whole + // im2col matrix, which in a VAE decoder at 1024x1024 is 2.8 GB for one 128-channel conv and 5.6 GB for a + // 256-channel one. With F16 weights (every Conv2d / Conv3d block) they produce the same values as im2col + + // mul_mat, bit for bit; F32 weights (runtime LoRA tensors) and depthwise convs keep F32 activations where + // im2col rounds them to F16, so those come out closer to the exact result rather than identical. + LOG_INFO("conv%dd direct auto: %s -> on (CPU)", dims, name); + return true; +#else + // Without the row-run kernels (scripts/unsloth/ggml-patches/0014, applied by the prebuilt workflow) the + // per-element direct loops are slower than im2col on large convolutions. + LOG_INFO("conv%dd direct auto: %s -> off (CPU, ggml without the row-run direct kernels)", dims, name); + return false; +#endif + } if (sd_backend_supports_conv_implicit_gemm(backend, conv3d)) { LOG_INFO("conv%dd direct auto: %s -> on (implicit GEMM)", dims, name); return true; diff --git a/src/core/util.cpp b/src/core/util.cpp index a691b423c..080ca5341 100644 --- a/src/core/util.cpp +++ b/src/core/util.cpp @@ -29,6 +29,11 @@ #include #endif +#ifdef __linux__ +#include +#include +#endif + #include "ggml-backend.h" #include "ggml.h" #include "stable-diffusion.h" @@ -340,20 +345,47 @@ bool MmapWrapper::copy_data(void* buf, size_t n, size_t offset) const { // LICENSE: https://github.com/ggerganov/llama.cpp/blob/master/LICENSE int32_t sd_get_num_physical_cores() { #ifdef __linux__ - // enumerate the set of thread siblings, num entries is num cores - std::unordered_set siblings; - for (uint32_t cpu = 0; cpu < UINT32_MAX; ++cpu) { - std::ifstream thread_siblings("/sys/devices/system/cpu/cpu" + std::to_string(cpu) + "/topology/thread_siblings"); - if (!thread_siblings.is_open()) { - break; // no more cpus + // One thread per physical core this process may run on: the distinct thread-sibling sets among the CPUs in + // the affinity mask (a taskset / container cpuset leaves fewer CPUs than the machine has). Upstream's loop + // counts every CPU of the machine, so a 4-CPU cpuset on a 192-CPU host still gets 96 threads. + // The mask is sized dynamically: a kernel built for more than CPU_SETSIZE (1024) CPUs rejects a smaller one + // with EINVAL, so grow it until the call succeeds. + for (int ncpus = CPU_SETSIZE; ncpus <= (1 << 20); ncpus *= 2) { + cpu_set_t* mask = CPU_ALLOC(ncpus); + const size_t size = CPU_ALLOC_SIZE(ncpus); + if (mask == nullptr) { + break; + } + CPU_ZERO_S(size, mask); + if (sched_getaffinity(0, size, mask) != 0) { + CPU_FREE(mask); + if (errno == EINVAL) { + continue; + } + break; } - std::string line; - if (std::getline(thread_siblings, line)) { - siblings.insert(line); + std::unordered_set siblings; + int allowed = 0; + for (int cpu = 0; cpu < ncpus; ++cpu) { + if (!CPU_ISSET_S(cpu, size, mask)) { + continue; + } + allowed++; + std::ifstream thread_siblings("/sys/devices/system/cpu/cpu" + std::to_string(cpu) + "/topology/thread_siblings"); + std::string line; + if (thread_siblings.is_open() && std::getline(thread_siblings, line) && !line.empty()) { + siblings.insert(line); + } } - } - if (siblings.size() > 0) { - return static_cast(siblings.size()); + CPU_FREE(mask); + if (!siblings.empty()) { + return static_cast(siblings.size()); + } + if (allowed > 0) { + // No topology in sysfs (some containers): assume 2-way SMT above 4 CPUs, as below. + return allowed <= 4 ? allowed : allowed / 2; + } + break; } #elif defined(__APPLE__) && defined(__MACH__) int32_t num_physical_cores;