From ce95c09ce14ffbde1fcffa00a05688c63048a023 Mon Sep 17 00:00:00 2001 From: losewayy <1134285612@qq.com> Date: Sat, 10 Oct 2026 22:33:37 +0800 Subject: [PATCH 01/18] fix: halve VAE decode tile size on OOM retry instead of clamping to 256px (#2120) --- src/core/backend_fit.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/core/backend_fit.cpp b/src/core/backend_fit.cpp index 6a7ab4337..84667ef36 100644 --- a/src/core/backend_fit.cpp +++ b/src/core/backend_fit.cpp @@ -497,11 +497,7 @@ namespace sd::backend_fit { return false; } auto smaller_tile = [&](int size) { - int next_size = size / 2; - if (!tiling_params.enabled) { - next_size = std::min(next_size, 256 / scale_factor); - } - return std::min(size, std::max(4, next_size)); + return std::min(size, std::max(4, size / 2)); }; const int tile_size_w = smaller_tile(latent_tile_size_w); const int tile_size_h = smaller_tile(latent_tile_size_h); From 0e1d733b7ad9268b80171597ae84cae26353f3fa Mon Sep 17 00:00:00 2001 From: leejet Date: Sat, 10 Oct 2026 22:40:06 +0800 Subject: [PATCH 02/18] docs: restrict README UI listings to open-source projects --- CONTRIBUTING.md | 4 ++++ README.md | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 776626d7e..1f4c06792 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,6 +29,10 @@ PRs should include: - Linked issue/PR context when applicable. - Screenshots or sample outputs for UI/visual behavior changes. +## README UI Listings + +The [UIs list in the README](README.md#uis) only includes open-source projects that use `stable-diffusion.cpp` as a backend. Do not submit PRs to add closed-source UIs. + ## Code Style Format code according to the repository style before submitting changes. diff --git a/README.md b/README.md index 832ee339f..f6d1bceed 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,6 @@ These projects wrap `stable-diffusion.cpp` for easier use in other languages/fra These projects use `stable-diffusion.cpp` as a backend for their image generation. - [GIMP Plugins](https://github.com/themanyone/gimp-plugins) -- [Jellybox](https://jellybox.com) - [Stable Diffusion GUI](https://github.com/fszontagh/sd.cpp.gui.wx) - [Stable Diffusion CLI-GUI](https://github.com/piallai/stable-diffusion.cpp) - [Local Diffusion](https://github.com/rmatif/Local-Diffusion) From 05845fde4152598c0d4c3cc763d23d80c27db214 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Sat, 10 Oct 2026 09:49:28 -0500 Subject: [PATCH 03/18] ci: fix ROCm ccache build caching and update to ROCm 10.1.0 (#1890) --- .github/workflows/build.yml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5586e734f..b717e56d9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -451,8 +451,8 @@ jobs: runs-on: windows-2022 env: - ROCM_VERSION: "7.14.0" - GPU_TARGETS: "gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1153;gfx1200;gfx1201" + ROCM_VERSION: "10.1.0" + gpu_targets: "gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1153;gfx1200;gfx1201" steps: - uses: actions/checkout@v3 @@ -498,7 +498,7 @@ jobs: write-host "Installing ROCm wheels for multi-arch support" # Install ROCm wheels for multi-arch support (this may take several minutes) - python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{env.ROCM_VERSION}}" + python -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel]==${{env.ROCM_VERSION}}" # Pre-expand the devel tree so it is included in the cache write-host "Initializing ROCm devel tree" @@ -594,9 +594,9 @@ jobs: strategy: matrix: include: - - ROCM_VERSION: "7.14.0" + - ROCM_VERSION: "10.1.0" gpu_targets: "gfx900;gfx906;gfx908;gfx90a;gfx90c;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1153;gfx1200;gfx1201" - build: x64 + build: 'x64' steps: - name: Clone @@ -610,6 +610,16 @@ jobs: with: key: ubuntu-rocm-cmake-${{ matrix.ROCM_VERSION }}-${{ matrix.build }} evict-old-files: 1d + max-size: "2G" + + - name: Tune ccache for reinstalled ROCm toolchain + run: | + # ROCm is pip-installed fresh each run, so the clang binary's mtime + # changes every time. With the default compiler_check=mtime that + # invalidates the cache; hash compiler contents instead so warm + # builds hit. + ccache --set-config=compiler_check=content + ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime - name: Dependencies id: depends @@ -646,7 +656,7 @@ jobs: # libraries = HIP runtime and CMake configs needed for linking # devel = compilers, headers, static libs python -m pip install --upgrade pip - python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{matrix.ROCM_VERSION}}" + python -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel]==${{matrix.ROCM_VERSION}}" # Get ROCm installation paths using the rocm-sdk CLI tool ROCM_PATH=$(rocm-sdk path --root) From b7a74e68d7409924f57fb52d3d721772d1084853 Mon Sep 17 00:00:00 2001 From: losewayy <1134285612@qq.com> Date: Sat, 10 Oct 2026 23:15:16 +0800 Subject: [PATCH 04/18] fix: demote resident params to disk residency when memory reclamation fails (#2119) --- src/model_manager.cpp | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/model_manager.cpp b/src/model_manager.cpp index e719825f7..2815e168b 100644 --- a/src/model_manager.cpp +++ b/src/model_manager.cpp @@ -1904,6 +1904,49 @@ bool ModelManager::ensure_compute_backend_capacity( } } + // The loader retains mmap buffers after storage blocks are released, so + // demoting mapped states would only force duplicate allocations on reload. + std::unordered_set mapped_states; + for (const auto& block : params_storage_blocks_) { + if (!block->mmap_tensor_stores.empty()) { + mapped_states.insert(block->states.begin(), block->states.end()); + } + } + std::unordered_set required_set(required_states.begin(), required_states.end()); + std::vector demoted_states; + auto revert_demotions = [&]() { + for (TensorState* state : demoted_states) { + if (state->loaded_to_params_backend) { + state->residency_mode = ResidencyMode::ParamBackend; + } + } + }; + auto can_demote_to_disk = [&](TensorState* state) { + return state != nullptr && state->tensor != nullptr && state->tensor->view_src == nullptr && + state->residency_mode == ResidencyMode::ParamBackend && + state->params_backend != nullptr && state->params_backend == state->compute_backend && + state->compute_backend == compute_backend && !sd_backend_is_cpu(state->params_backend) && + state->pin_count == 0 && !state->staged_to_compute_backend && state->loaded_to_params_backend && + protected_states.find(state) == protected_states.end() && + required_set.find(state) == required_set.end() && + mapped_states.find(state) == mapped_states.end() && + !state->sources.empty() && state->applied_lora_epoch == UINT64_MAX && + state->split_buffer_type == nullptr; + }; + for (TensorState* state : global_candidates) { + if (!can_demote_to_disk(state)) { + continue; + } + state->residency_mode = ResidencyMode::Disk; + demoted_states.push_back(state); + add_evictable_state(state); + if (release_eviction_states()) { + revert_demotions(); + return true; + } + } + revert_demotions(); + const auto capacity = check_capacity(request, required_states, true); const std::string available_device = capacity.available_device_bytes == SIZE_MAX ? "unknown" From 1b0ba10893f4e2e0656103011fc1c4645a02a734 Mon Sep 17 00:00:00 2001 From: Daniele <57776841+daniandtheweb@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:35:58 +0200 Subject: [PATCH 05/18] feat: run conditional and unconditional CFG in one batched UNet forward (#2085) --- docs/performance.md | 10 +++ examples/common/common.cpp | 46 ++++++++----- examples/common/common.h | 1 + include/stable-diffusion.h | 1 + src/model/diffusion/unet.hpp | 4 +- src/pipeline/diffusion_engine.cpp | 106 +++++++++++++++++++++++++++++- src/stable-diffusion.cpp | 1 + 7 files changed, 147 insertions(+), 22 deletions(-) diff --git a/docs/performance.md b/docs/performance.md index 6797ad4be..acd071cf6 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -21,6 +21,16 @@ CPU fallback. It excludes weights and cache buffers. Within a runner lifecycle, the summary is printed only on the first graph or when backend capacities or the segment count change. +## Run conditional and unconditional CFG in one batched UNet forward. + +For UNet models, the conditional and unconditional guidance branches are +concatenated into a single batch of two and run through one UNet forward per +step instead of two separate forwards. This is enabled by default whenever the +run qualifies for it. + +Use `--batched-cfg off` to force separate conditional and unconditional +forwards. + ## Use VAE tiling to reduce encode and decode memory usage. `--vae-tiling` enables spatial tiling for both VAE encoding and decoding. The diff --git a/examples/common/common.cpp b/examples/common/common.cpp index 89b226f3c..faa637266 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -378,6 +378,23 @@ static int parse_scale_override(int argc, const char** argv, int index, float& s return 1; } +static int parse_on_off_arg(int argc, const char** argv, int index, const char* option, bool& value) { + if (++index >= argc) { + LOG_ERROR("%s requires 'on' or 'off'", option); + return -1; + } + const std::string arg = argv[index]; + if (arg == "on") { + value = true; + } else if (arg == "off") { + value = false; + } else { + LOG_ERROR("invalid %s value '%s'; expected 'on' or 'off'", option, argv[index]); + return -1; + } + return 1; +} + ArgOptions SDContextParams::get_options() { ArgOptions options; options.string_options = { @@ -637,23 +654,6 @@ ArgOptions SDContextParams::get_options() { true, &vae_conv_direct}, }; - auto on_auto_fit_arg = [&](int argc, const char** argv, int index) { - if (++index >= argc) { - LOG_ERROR("--auto-fit requires 'on' or 'off'"); - return -1; - } - const std::string arg = argv[index]; - if (arg == "on") { - auto_fit = true; - } else if (arg == "off") { - auto_fit = false; - } else { - LOG_ERROR("invalid --auto-fit value '%s'; expected 'on' or 'off'", argv[index]); - return -1; - } - return 1; - }; - auto on_type_arg = [&](int argc, const char** argv, int index) { if (++index >= argc) { return -1; @@ -742,7 +742,15 @@ ArgOptions SDContextParams::get_options() { "on|off (default: on). Preserve --backend (otherwise select one GPU) and place weights on the compute GPU, " "RAM, another GPU, or disk in that order, according to available memory (--max-vram limits GPU budgets). " "Disabled by explicit --params-backend; uses automatic graph segmentation when needed", - on_auto_fit_arg}, + [this](int argc, const char** argv, int index) { + return parse_on_off_arg(argc, argv, index, "--auto-fit", auto_fit); + }}, + {"", + "--batched-cfg", + "on|off (default: on). Run the conditional and unconditional CFG branches in one batched UNet forward when supported", + [this](int argc, const char** argv, int index) { + return parse_on_off_arg(argc, argv, index, "--batched-cfg", batched_cfg); + }}, {"", "--type", "weight type (examples: f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0, q2_K, q3_K, q4_K). " @@ -940,6 +948,7 @@ std::string SDContextParams::to_string() const { << " max_vram: \"" << max_vram << "\",\n" << " disable_prefetch: " << (disable_prefetch ? "true" : "false") << ",\n" << " disable_segmented_compute: " << (disable_segmented_compute ? "true" : "false") << ",\n" + << " batched_cfg: " << (batched_cfg ? "true" : "false") << ",\n" << " eager_load: " << (eager_load ? "true" : "false") << ",\n" << " backend: \"" << backend << "\",\n" << " params_backend: \"" << params_backend << "\",\n" @@ -1022,6 +1031,7 @@ sd_ctx_params_t SDContextParams::to_sd_ctx_params_t(bool taesd_preview) { sd_ctx_params.max_vram = max_vram.c_str(); sd_ctx_params.disable_prefetch = disable_prefetch; sd_ctx_params.disable_segmented_compute = disable_segmented_compute; + sd_ctx_params.batched_cfg = batched_cfg; sd_ctx_params.eager_load = eager_load; sd_ctx_params.backend = effective_backend.c_str(); sd_ctx_params.params_backend = effective_params_backend.c_str(); diff --git a/examples/common/common.h b/examples/common/common.h index 8b60c7601..bd9075c36 100644 --- a/examples/common/common.h +++ b/examples/common/common.h @@ -157,6 +157,7 @@ struct SDContextParams { bool disable_prefetch = false; bool disable_segmented_compute = false; bool eager_load = false; + bool batched_cfg = true; std::string backend; std::string params_backend; std::string split_mode; diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index a2e157556..c21f7657d 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -245,6 +245,7 @@ typedef struct { const char* rpc_servers; const char* model_args; bool disable_segmented_compute; // Force monolithic graph execution even when automatic graph cutting would fit memory better + bool batched_cfg; // Run the conditional and unconditional CFG branches in one batched UNet forward when supported float linear_scale; // Override linear input scaling; 0 keeps the model default float attn_scale; // Override flash-attention K/V scaling; 0 keeps the model default const char* tokenizer; // tokenizer.json path or main=FILE,clip-l=FILE,clip-g=FILE assignments; required for PiD and Lens diff --git a/src/model/diffusion/unet.hpp b/src/model/diffusion/unet.hpp index 1c8ddf4e6..7c0ff0950 100644 --- a/src/model/diffusion/unet.hpp +++ b/src/model/diffusion/unet.hpp @@ -587,7 +587,9 @@ class UnetModelBlock : public GGMLBlock { label_emb = ggml_silu_inplace(ctx->ggml_ctx, label_emb); label_emb = label_embed_2->forward(ctx, label_emb); // [N, time_embed_dim] - emb = ggml_add(ctx->ggml_ctx, emb, label_emb); // [N, time_embed_dim] + emb = label_emb->ne[1] > emb->ne[1] + ? ggml_add(ctx->ggml_ctx, label_emb, emb) + : ggml_add(ctx->ggml_ctx, emb, label_emb); // [N, time_embed_dim] } // sd::ggml_graph_cut::mark_graph_cut(emb, "unet.prelude", "emb"); diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index 68129bb64..6400b243e 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -2227,6 +2227,27 @@ void StableDiffusionGGML::report_sample_progress(int step, } } +static sd::Tensor batch_two_condition_tensors(const sd::Tensor& a, const sd::Tensor& b) { + if (a.empty() || b.empty() || a.dim() != b.dim()) { + return {}; + } + if (a.dim() == 1) { + if (a.shape() != b.shape()) { + return {}; + } + auto batched = sd::ops::concat(a, b, 0); + batched.reshape_({a.shape()[0], 2}); + return batched; + } + const int64_t batch_dim = a.dim() - 1; + for (int64_t d = 0; d < batch_dim; d++) { + if (a.shape()[d] != b.shape()[d]) { + return {}; + } + } + return sd::ops::concat(a, b, static_cast(batch_dim)); +} + void StableDiffusionGGML::compute_sample_controls(const sd::Tensor& control_image, const sd::Tensor& noised_input, const sd::Tensor& timesteps_tensor, @@ -2604,6 +2625,57 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr* c_concat_override) -> sd::Tensor { + const sd::Tensor& condition_concat = + c_concat_override != nullptr ? *c_concat_override : condition.c_concat; + + sd::Tensor batched_context = batch_two_condition_tensors(condition.c_crossattn, uncond.c_crossattn); + sd::Tensor batched_y = batch_two_condition_tensors(condition.c_vector, uncond.c_vector); + sd::Tensor batched_concat = batch_two_condition_tensors(condition_concat, uncond.c_concat); + if (!condition.c_crossattn.empty() && batched_context.empty()) { + return {}; + } + if ((!condition.c_vector.empty() || !uncond.c_vector.empty()) && batched_y.empty()) { + return {}; + } + if ((!condition_concat.empty() || !uncond.c_concat.empty()) && batched_concat.empty()) { + return {}; + } + + std::vector> uncond_controls; + compute_sample_controls(control_image, noised_input, timesteps_tensor, uncond, &uncond_controls); + if (controls.size() != uncond_controls.size()) { + return {}; + } + std::vector> batched_controls; + batched_controls.reserve(controls.size()); + for (size_t i = 0; i < controls.size(); i++) { + sd::Tensor batched_control = batch_two_condition_tensors(controls[i], uncond_controls[i]); + if (batched_control.empty()) { + return {}; + } + batched_controls.push_back(std::move(batched_control)); + } + + sd::Tensor batched_x = + sd::ops::concat(noised_input, noised_input, static_cast(noised_input.dim() - 1)); + + DiffusionParams batched_params = diffusion_params; + batched_params.x = &batched_x; + batched_params.context = batched_context.empty() ? nullptr : &batched_context; + batched_params.c_concat = batched_concat.empty() ? nullptr : &batched_concat; + batched_params.y = batched_y.empty() ? nullptr : &batched_y; + batched_params.ref_latents = nullptr; + batched_params.extra = UNetDiffusionExtra{1, &batched_controls, control_strength}; + + sd::Tensor output = work_diffusion_model->compute(n_threads, batched_params); + if (output.empty()) { + LOG_ERROR("batched diffusion model compute failed"); + } + return output; + }; + const SDCondition* positive_condition = &cond; const sd::Tensor* c_concat_override = nullptr; for (const auto& extension : generation_extensions) { @@ -2643,12 +2715,40 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptrparams.batched_cfg && + sd_version_is_unet(version) && + !uncond.empty() && + img_uncond.empty() && + !skip_uncond && + !cache_runtime.ucache_enabled() && + !(is_skiplayer_step && slg_uncond) && + ip_adapter_tokens.empty() && + ip_adapter_uncond_tokens.empty() && + !config_->animatediff_loaded && + (noised_input.dim() < 4 || noised_input.shape()[3] <= 1) && + std::none_of(generation_extensions.begin(), + generation_extensions.end(), + [](const std::shared_ptr& extension) { + return extension->is_enabled(); + }); + + if (batch_cfg_ok) { + sd::Tensor batched_out = run_batched_condition(*positive_condition, c_concat_override); + if (!batched_out.empty() && batched_out.dim() >= 4 && batched_out.shape()[3] == 2) { + auto parts = sd::ops::chunk(batched_out, 2, 3); + cond_out = std::move(parts[0]); + uncond_out = std::move(parts[1]); + } + } + if (cond_out.empty()) { - return {}; + cond_out = run_condition(*positive_condition, c_concat_override); + if (cond_out.empty()) { + return {}; + } } - if (!uncond.empty()) { + if (uncond_out.empty() && !uncond.empty()) { if (!skip_uncond) { const std::vector* uncond_skip_layers = nullptr; if (is_skiplayer_step && slg_uncond) { diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index b13c055f3..44a660bd9 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -335,6 +335,7 @@ void sd_ctx_params_init(sd_ctx_params_t* sd_ctx_params) { sd_ctx_params->max_vram = nullptr; sd_ctx_params->disable_prefetch = false; sd_ctx_params->disable_segmented_compute = false; + sd_ctx_params->batched_cfg = true; sd_ctx_params->eager_load = false; sd_ctx_params->enable_mmap = false; sd_ctx_params->diffusion_flash_attn = false; From 2988060a4338a5dcd94721c964a8e87003c1ac8b Mon Sep 17 00:00:00 2001 From: leejet Date: Sun, 11 Oct 2026 00:01:18 +0800 Subject: [PATCH 06/18] feat: add TAEQI2.1 support for Qwen Image 2.1 (#2123) --- docs/qwen_image_2.1.md | 2 ++ docs/taesd.md | 12 ++++++++ src/model/vae/tae.hpp | 51 ++++++++++++++++++++++++--------- src/pipeline/model_builders.cpp | 2 +- 4 files changed, 53 insertions(+), 14 deletions(-) diff --git a/docs/qwen_image_2.1.md b/docs/qwen_image_2.1.md index 56999bcc8..5da89c534 100644 --- a/docs/qwen_image_2.1.md +++ b/docs/qwen_image_2.1.md @@ -16,6 +16,8 @@ Qwen Image 2.1 supports text-to-image generation and image editing, using Qwen3- Use `qwen_image_2.1_vae_bf16.safetensors` with this model. The earlier Qwen Image and Wan 2.2 VAE weights are not interchangeable with the Qwen Image 2.1 VAE weights. +For faster encoding, decoding, or previews, see [TAEQI2.1](taesd.md#qwen-image-21-taeqi21). + ## Examples Run the following commands from the build directory. Use image dimensions divisible by 32. The resolution-dependent flow schedule is selected automatically. diff --git a/docs/taesd.md b/docs/taesd.md index a41c64d43..954086d18 100644 --- a/docs/taesd.md +++ b/docs/taesd.md @@ -37,3 +37,15 @@ sd.cpp also supports [TAEHV](https://github.com/madebyollin/taehv) (#937), which ``` Then simply replace the `--vae xxx.safetensors` with `--tae xxx.safetensors` in the commands. If it still out of VRAM, add `--vae-conv-direct` to your command though might be slower. + +### Qwen Image 2.1 (TAEQI2.1) + +For Qwen Image 2.1, use [taeqi2_1](https://github.com/madebyollin/taesd), which supports 64-channel latents, 16x spatial scaling, and RGBA images. + +Download the official [safetensors weights](https://huggingface.co/madebyollin/taeqi2_1/blob/main/taeqi2_1.safetensors) directly; no conversion is needed: + +```bash +curl -L -o taeqi2_1.safetensors https://huggingface.co/madebyollin/taeqi2_1/resolve/main/taeqi2_1.safetensors +``` + +Replace `--vae PATH` with `--taesd taeqi2_1.safetensors` in the [Qwen Image 2.1 examples](qwen_image_2.1.md) to use it for encoding and decoding. For previews only, keep `--vae PATH` and add `--taesd taeqi2_1.safetensors --taesd-preview-only --preview tae`. diff --git a/src/model/vae/tae.hpp b/src/model/vae/tae.hpp index 173b40c5c..a0700cab3 100644 --- a/src/model/vae/tae.hpp +++ b/src/model/vae/tae.hpp @@ -84,12 +84,17 @@ class TinyEncoder : public UnaryBlock { int channels = 64; int z_channels = 4; int num_blocks = 3; + bool f16; public: - TinyEncoder(int z_channels = 4, bool use_midblock_gn = false) - : z_channels(z_channels) { - int index = 0; + TinyEncoder(int z_channels = 4, bool use_midblock_gn = false, bool f16 = false) + : z_channels(z_channels), f16(f16) { + in_channels = f16 ? 16 : 3; + int index = f16 ? 1 : 0; blocks[std::to_string(index++)] = std::shared_ptr(new Conv2d(in_channels, channels, {3, 3}, {1, 1}, {1, 1})); + if (f16) { + index++; // nn.ReLU() + } blocks[std::to_string(index++)] = std::shared_ptr(new TAEBlock(channels, channels)); blocks[std::to_string(index++)] = std::shared_ptr(new Conv2d(channels, channels, {3, 3}, {2, 2}, {1, 1}, {1, 1}, false)); @@ -97,12 +102,14 @@ class TinyEncoder : public UnaryBlock { blocks[std::to_string(index++)] = std::shared_ptr(new TAEBlock(channels, channels)); } - blocks[std::to_string(index++)] = std::shared_ptr(new Conv2d(channels, channels, {3, 3}, {2, 2}, {1, 1}, {1, 1}, false)); + blocks[std::to_string(index++)] = std::shared_ptr(new Conv2d(channels, f16 ? channels * 2 : channels, {3, 3}, {2, 2}, {1, 1}, {1, 1}, false)); + channels *= f16 ? 2 : 1; for (int i = 0; i < num_blocks; i++) { blocks[std::to_string(index++)] = std::shared_ptr(new TAEBlock(channels, channels)); } - blocks[std::to_string(index++)] = std::shared_ptr(new Conv2d(channels, channels, {3, 3}, {2, 2}, {1, 1}, {1, 1}, false)); + blocks[std::to_string(index++)] = std::shared_ptr(new Conv2d(channels, f16 ? channels * 2 : channels, {3, 3}, {2, 2}, {1, 1}, {1, 1}, false)); + channels *= f16 ? 2 : 1; for (int i = 0; i < num_blocks; i++) { blocks[std::to_string(index++)] = std::shared_ptr(new TAEBlock(channels, channels, use_midblock_gn)); } @@ -114,7 +121,11 @@ class TinyEncoder : public UnaryBlock { // x: [n, in_channels, h, w] // return: [n, z_channels, h/8, w/8] - for (int i = 0; i < num_blocks * 3 + 6; i++) { + for (int i = f16 ? 1 : 0; i < num_blocks * 3 + 6 + (f16 ? 2 : 0); i++) { + if (f16 && i == 2) { + x = ggml_relu_inplace(ctx->ggml_ctx, x); + continue; + } auto block = std::dynamic_pointer_cast(blocks[std::to_string(i)]); x = block->forward(ctx, x); @@ -131,9 +142,11 @@ class TinyDecoder : public UnaryBlock { int num_blocks = 3; public: - TinyDecoder(int z_channels = 4, bool use_midblock_gn = false) + TinyDecoder(int z_channels = 4, bool use_midblock_gn = false, bool f16 = false) : z_channels(z_channels) { - int index = 0; + channels = f16 ? 256 : 64; + out_channels = f16 ? 16 : 3; + int index = 0; blocks[std::to_string(index++)] = std::shared_ptr(new Conv2d(z_channels, channels, {3, 3}, {1, 1}, {1, 1})); index++; // nn.ReLU() @@ -142,13 +155,15 @@ class TinyDecoder : public UnaryBlock { blocks[std::to_string(index++)] = std::shared_ptr(new TAEBlock(channels, channels, use_midblock_gn)); } index++; // nn.Upsample() - blocks[std::to_string(index++)] = std::shared_ptr(new Conv2d(channels, channels, {3, 3}, {1, 1}, {1, 1}, {1, 1}, false)); + blocks[std::to_string(index++)] = std::shared_ptr(new Conv2d(channels, f16 ? channels / 2 : channels, {3, 3}, {1, 1}, {1, 1}, {1, 1}, false)); + channels /= f16 ? 2 : 1; for (int i = 0; i < num_blocks; i++) { blocks[std::to_string(index++)] = std::shared_ptr(new TAEBlock(channels, channels)); } index++; // nn.Upsample() - blocks[std::to_string(index++)] = std::shared_ptr(new Conv2d(channels, channels, {3, 3}, {1, 1}, {1, 1}, {1, 1}, false)); + blocks[std::to_string(index++)] = std::shared_ptr(new Conv2d(channels, f16 ? channels / 2 : channels, {3, 3}, {1, 1}, {1, 1}, {1, 1}, false)); + channels /= f16 ? 2 : 1; for (int i = 0; i < num_blocks; i++) { blocks[std::to_string(index++)] = std::shared_ptr(new TAEBlock(channels, channels)); @@ -691,6 +706,7 @@ class TAESD : public GGMLBlock { protected: bool decode_only; bool taef2 = false; + bool f16 = false; public: int z_channels = 4; @@ -708,10 +724,14 @@ class TAESD : public GGMLBlock { z_channels = 32; use_midblock_gn = true; } - blocks["decoder.layers"] = std::shared_ptr(new TinyDecoder(z_channels, use_midblock_gn)); + f16 = version == VERSION_QWEN_IMAGE_2_1; + if (f16) { + z_channels = 64; + } + blocks["decoder.layers"] = std::shared_ptr(new TinyDecoder(z_channels, use_midblock_gn, f16)); if (!decode_only) { - blocks["encoder.layers"] = std::shared_ptr(new TinyEncoder(z_channels, use_midblock_gn)); + blocks["encoder.layers"] = std::shared_ptr(new TinyEncoder(z_channels, use_midblock_gn, f16)); } } @@ -720,10 +740,15 @@ class TAESD : public GGMLBlock { if (taef2) { z = unpatchify(ctx->ggml_ctx, z, 2); } - return decoder->forward(ctx, z); + auto x = decoder->forward(ctx, z); + return f16 ? unpatchify(ctx->ggml_ctx, x, 2) : x; } ggml_tensor* encode(GGMLRunnerContext* ctx, ggml_tensor* x) { + if (f16) { + GGML_ASSERT(x->ne[0] % 2 == 0 && x->ne[1] % 2 == 0); + x = patchify(ctx->ggml_ctx, x, 2); + } auto encoder = std::dynamic_pointer_cast(blocks["encoder.layers"]); auto z = encoder->forward(ctx, x); if (taef2) { diff --git a/src/pipeline/model_builders.cpp b/src/pipeline/model_builders.cpp index 96b5a974c..87cdf4036 100644 --- a/src/pipeline/model_builders.cpp +++ b/src/pipeline/model_builders.cpp @@ -533,7 +533,7 @@ namespace sd::model_builders { } auto create_tae = [&](bool decode_only) -> std::shared_ptr { - if (sd_version_uses_wan_vae(version) || sd_version_is_hunyuan_video(version) || sd_version_is_ltxav(version) || sd_version_is_minimax_h3(version)) { + if ((sd_version_uses_wan_vae(version) && version != VERSION_QWEN_IMAGE_2_1) || sd_version_is_hunyuan_video(version) || sd_version_is_ltxav(version) || sd_version_is_minimax_h3(version)) { return std::make_shared(ctx.backends.runtime_backend(SDBackendModule::VAE), tensor_storage_map, "decoder", From 792bcd344c3fa4d14143b240d5f6cbb324583b18 Mon Sep 17 00:00:00 2001 From: Gabriele Lozupone <54721912+GabrieleLozupone@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:38:38 +0200 Subject: [PATCH 07/18] feat: add MoSR upscaler support (#2128) --- docs/esrgan.md | 4 +- src/model/upscaler/mosr.hpp | 337 ++++++++++++++++++++++++++++++++++++ src/upscaler.cpp | 88 +++++++--- src/upscaler.h | 5 + 4 files changed, 410 insertions(+), 24 deletions(-) create mode 100644 src/model/upscaler/mosr.hpp diff --git a/docs/esrgan.md b/docs/esrgan.md index 708f49bad..4068ef11c 100644 --- a/docs/esrgan.md +++ b/docs/esrgan.md @@ -2,7 +2,9 @@ You can use ESRGAN—such as the model [RealESRGAN_x4plus_anime_6B.pth](https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.2.4/RealESRGAN_x4plus_anime_6B.pth)—to upscale the generated images and improve their overall resolution and clarity. -RGBA images, including Qwen Image 2.1 output, keep their alpha channel during model upscaling and hires fix. ESRGAN processes the RGB channels; the alpha channel is resized with bilinear interpolation and recombined with the upscaled image. +MoSR models with the PixelShuffle upsampler, such as [4xNomos2_hq_mosr.pth](https://github.com/Phhofm/models/releases/download/4xNomos2_hq_mosr/4xNomos2_hq_mosr.pth), are also supported. The architecture is detected from the weights, so they are used the same way as ESRGAN models. + +RGBA images, including Qwen Image 2.1 output, keep their alpha channel during model upscaling and hires fix. The upscaler processes the RGB channels; the alpha channel is resized with bilinear interpolation and recombined with the upscaled image. - Specify the model path using the `--upscale-model PATH` parameter. example: diff --git a/src/model/upscaler/mosr.hpp b/src/model/upscaler/mosr.hpp new file mode 100644 index 000000000..0aa158c81 --- /dev/null +++ b/src/model/upscaler/mosr.hpp @@ -0,0 +1,337 @@ +#ifndef __SD_MODEL_UPSCALER_MOSR_HPP__ +#define __SD_MODEL_UPSCALER_MOSR_HPP__ + +#include +#include +#include +#include +#include +#include + +#include "core/ggml_extend.h" +#include "core/ggml_runner.h" +#include "core/util.h" +#include "model/common/ggml_block.hpp" + +/* + =================================== MoSR =================================== + References: + https://github.com/umzi2/MoSR + https://github.com/chaiNNer-org/spandrel/blob/main/libs/spandrel/spandrel/architectures/MoSR/__arch/mosr_arch.py + + Only the PixelShuffle ("ps") upsampler is supported. +*/ + +struct MoSRConfig { + int scale = 4; + int n_block = 24; + int in_ch = 3; + int dim = 64; + int hidden = 96; + int conv_ch = 64; + int kernel_size = 7; + + bool is_valid() const { + // The upscaler feeds RGB tiles, so only 3-channel models are usable. + return in_ch == 3 && scale > 0 && n_block > 0 && conv_ch <= hidden; + } + + static MoSRConfig detect_from_weights(const String2TensorStorage& tensor_storage_map, + const std::string& prefix = "") { + MoSRConfig config; + auto find_weight = [&](const std::string& suffix) -> const TensorStorage* { + std::string name = prefix.empty() ? suffix : prefix + "." + suffix; + auto iter = tensor_storage_map.find(name); + if (iter == tensor_storage_map.end() || iter->second.n_dims != 4) { + return nullptr; + } + return &iter->second; + }; + + const TensorStorage* first = find_weight("gblocks.0.weight"); + const TensorStorage* fc1 = find_weight("gblocks.1.fc1.weight"); + const TensorStorage* conv = find_weight("gblocks.1.conv.weight"); + const TensorStorage* up = find_weight("upsampler.0.weight"); + if (first == nullptr || fc1 == nullptr || conv == nullptr || up == nullptr) { + // Missing tensors, or the unsupported "gps"/"dys" upsamplers. + config.n_block = 0; + return config; + } + + int seq_len = 0; + const std::string gblocks_prefix = prefix.empty() ? "gblocks." : prefix + ".gblocks."; + for (const auto& [name, _] : tensor_storage_map) { + if (!starts_with(name, gblocks_prefix)) { + continue; + } + size_t pos = name.find('.', gblocks_prefix.size()); + if (pos == std::string::npos) { + continue; + } + try { + int idx = std::stoi(name.substr(gblocks_prefix.size(), pos - gblocks_prefix.size())); + seq_len = std::max(seq_len, idx + 1); + } catch (...) { + } + } + + // gblocks = [conv_in, n_block x GatedCNNBlock, conv, Mish, conv, Mish, conv_1x1] + config.n_block = seq_len - 6; + config.in_ch = (int)first->ne[2]; + config.dim = (int)first->ne[3]; + config.hidden = (int)fc1->ne[3] / 2; + config.conv_ch = (int)conv->ne[3]; + config.kernel_size = (int)conv->ne[0]; + config.scale = (int)std::lround(std::sqrt((double)up->ne[3] / config.in_ch)); + if (config.scale * config.scale * config.in_ch != up->ne[3]) { + config.scale = 0; + } + + LOG_VERBOSE("mosr: scale = %d, n_block = %d, in_ch = %d, dim = %d, hidden = %d, conv_ch = %d, kernel_size = %d", + config.scale, + config.n_block, + config.in_ch, + config.dim, + config.hidden, + config.conv_ch, + config.kernel_size); + return config; + } +}; + +static inline ggml_tensor* mosr_mish(ggml_context* ctx, ggml_tensor* x) { + return ggml_mul(ctx, x, ggml_tanh(ctx, ggml_softplus(ctx, x))); +} + +class MoSRLayerNorm2d : public UnaryBlock { +protected: + int64_t channels; + + void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override { + params["weight"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, channels); + params["bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, channels); + } + +public: + explicit MoSRLayerNorm2d(int64_t channels) + : channels(channels) {} + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override { + // [N, C, H, W] -> [N, H, W, C] so the norm reduces over channels. + x = ggml_ext_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 2, 0, 3)); + x = ggml_ext_layer_norm(ctx->ggml_ctx, x, params["weight"], params["bias"], 1e-6f); + return ggml_ext_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 2, 0, 1, 3)); + } +}; + +class MoSRDepthwiseConv2d : public UnaryBlock { +protected: + int64_t channels; + int kernel_size; + + void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override { + // The direct depthwise kernels of the CUDA and Vulkan backends need an F32 kernel. + params["weight"] = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, kernel_size, kernel_size, 1, channels); + params["bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, channels); + } + +public: + MoSRDepthwiseConv2d(int64_t channels, int kernel_size) + : channels(channels), kernel_size(kernel_size) {} + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override { + const int pad = kernel_size / 2; + x = ggml_conv_2d_dw_direct(ctx->ggml_ctx, params["weight"], ggml_ext_cont(ctx->ggml_ctx, x), 1, 1, pad, pad, 1, 1); + auto b = ggml_reshape_4d(ctx->ggml_ctx, params["bias"], 1, 1, channels, 1); + return ggml_add_inplace(ctx->ggml_ctx, x, b); + } +}; + +class MoSRGatedCNNBlock : public GGMLBlock { +protected: + int hidden; + int conv_ch; + +public: + MoSRGatedCNNBlock(int dim, int hidden, int conv_ch, int kernel_size) + : hidden(hidden), conv_ch(conv_ch) { + blocks["norm"] = std::shared_ptr(new MoSRLayerNorm2d(dim)); + blocks["fc1"] = std::shared_ptr(new Conv2d(dim, hidden * 2, {3, 3}, {1, 1}, {1, 1})); + blocks["conv"] = std::shared_ptr(new MoSRDepthwiseConv2d(conv_ch, kernel_size)); + blocks["fc2"] = std::shared_ptr(new Conv2d(hidden, dim, {3, 3}, {1, 1}, {1, 1})); + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { + // x: [n, dim, h, w] + // return: [n, dim, h, w] + auto norm = std::dynamic_pointer_cast(blocks["norm"]); + auto fc1 = std::dynamic_pointer_cast(blocks["fc1"]); + auto conv = std::dynamic_pointer_cast(blocks["conv"]); + auto fc2 = std::dynamic_pointer_cast(blocks["fc2"]); + + auto h = fc1->forward(ctx, norm->forward(ctx, x)); + + auto channel_view = [&](int64_t offset, int64_t count) { + return ggml_view_4d(ctx->ggml_ctx, h, h->ne[0], h->ne[1], count, h->ne[3], + h->nb[1], h->nb[2], h->nb[3], offset * h->nb[2]); + }; + auto g = channel_view(0, hidden); + auto i = channel_view(hidden, hidden - conv_ch); + auto c = conv->forward(ctx, channel_view(2 * hidden - conv_ch, conv_ch)); + if (hidden > conv_ch) { + c = ggml_concat(ctx->ggml_ctx, ggml_ext_cont(ctx->ggml_ctx, i), c, 2); + } + + h = ggml_mul(ctx->ggml_ctx, c, mosr_mish(ctx->ggml_ctx, ggml_ext_cont(ctx->ggml_ctx, g))); + h = mosr_mish(ctx->ggml_ctx, fc2->forward(ctx, h)); + return ggml_add(ctx->ggml_ctx, h, ggml_scale_bias(ctx->ggml_ctx, x, 1.f, -0.5f)); + } +}; + +class MoSRConvBlock : public GGMLBlock { +public: + MoSRConvBlock(int in_ch, int out_ch) { + blocks["block.0"] = std::shared_ptr(new Conv2d(in_ch, out_ch, {3, 3}, {1, 1}, {1, 1})); + blocks["block.2"] = std::shared_ptr(new Conv2d(out_ch, out_ch, {3, 3}, {1, 1}, {1, 1})); + blocks["conv11"] = std::shared_ptr(new Conv2d(in_ch, out_ch, {1, 1})); + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { + auto conv0 = std::dynamic_pointer_cast(blocks["block.0"]); + auto conv2 = std::dynamic_pointer_cast(blocks["block.2"]); + auto conv11 = std::dynamic_pointer_cast(blocks["conv11"]); + + auto out = mosr_mish(ctx->ggml_ctx, conv0->forward(ctx, x)); + out = mosr_mish(ctx->ggml_ctx, conv2->forward(ctx, out)); + return ggml_add(ctx->ggml_ctx, out, conv11->forward(ctx, x)); + } +}; + +class MoSRNet : public GGMLBlock { +protected: + MoSRConfig config; + + static std::string gblock_name(int idx) { + return "gblocks." + std::to_string(idx); + } + + // torch.nn.PixelShuffle: [n, c*r*r, h, w] -> [n, c, h*r, w*r] + static ggml_tensor* pixel_shuffle(ggml_context* ctx, ggml_tensor* x, int r) { + const int64_t w = x->ne[0]; + const int64_t h = x->ne[1]; + const int64_t c = x->ne[2] / (r * r); + const int64_t n = x->ne[3]; + x = ggml_reshape_4d(ctx, x, w, h, r, r * c * n); // [n*c*r, r, h, w] + x = ggml_cont(ctx, ggml_permute(ctx, x, 1, 2, 0, 3)); // [n*c*r, h, w, r] + x = ggml_reshape_4d(ctx, x, r * w, h, r, c * n); // [n*c, r, h, w*r] + x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3)); // [n*c, h, r, w*r] + return ggml_reshape_4d(ctx, x, r * w, r * h, c, n); + } + +public: + explicit MoSRNet(MoSRConfig config) + : config(std::move(config)) { + const int dim = this->config.dim; + const int nb = this->config.n_block; + + blocks[gblock_name(0)] = std::shared_ptr(new Conv2d(this->config.in_ch, dim, {3, 3}, {1, 1}, {1, 1})); + for (int i = 1; i <= nb; i++) { + blocks[gblock_name(i)] = std::shared_ptr(new MoSRGatedCNNBlock(dim, + this->config.hidden, + this->config.conv_ch, + this->config.kernel_size)); + } + blocks[gblock_name(nb + 1)] = std::shared_ptr(new Conv2d(dim, dim * 2, {3, 3}, {1, 1}, {1, 1})); + blocks[gblock_name(nb + 3)] = std::shared_ptr(new Conv2d(dim * 2, dim, {3, 3}, {1, 1}, {1, 1})); + blocks[gblock_name(nb + 5)] = std::shared_ptr(new Conv2d(dim, dim, {1, 1})); + blocks["shortcut"] = std::shared_ptr(new MoSRConvBlock(this->config.in_ch, dim)); + blocks["upsampler.0"] = std::shared_ptr(new Conv2d(dim, + this->config.in_ch * this->config.scale * this->config.scale, + {3, 3}, + {1, 1}, + {1, 1})); + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { + // x: [n, in_ch, h, w] + // return: [n, in_ch, h*scale, w*scale] + const int nb = config.n_block; + auto conv_in = std::dynamic_pointer_cast(blocks[gblock_name(0)]); + auto conv_mid1 = std::dynamic_pointer_cast(blocks[gblock_name(nb + 1)]); + auto conv_mid2 = std::dynamic_pointer_cast(blocks[gblock_name(nb + 3)]); + auto conv_out = std::dynamic_pointer_cast(blocks[gblock_name(nb + 5)]); + auto shortcut = std::dynamic_pointer_cast(blocks["shortcut"]); + auto upsampler = std::dynamic_pointer_cast(blocks["upsampler.0"]); + + auto skip = shortcut->forward(ctx, x); + sd::ggml_graph_cut::mark_graph_cut(skip, "mosr.shortcut", "skip"); + + auto feat = conv_in->forward(ctx, x); + sd::ggml_graph_cut::mark_graph_cut(feat, "mosr.prelude", "feat"); + for (int i = 1; i <= nb; i++) { + auto block = std::dynamic_pointer_cast(blocks[gblock_name(i)]); + feat = block->forward(ctx, feat); + sd::ggml_graph_cut::mark_graph_cut(feat, "mosr.gblocks." + std::to_string(i), "feat"); + } + feat = mosr_mish(ctx->ggml_ctx, conv_mid1->forward(ctx, feat)); + feat = mosr_mish(ctx->ggml_ctx, conv_mid2->forward(ctx, feat)); + feat = conv_out->forward(ctx, feat); + feat = ggml_add(ctx->ggml_ctx, feat, ggml_scale_bias(ctx->ggml_ctx, skip, 1.f, -0.5f)); + sd::ggml_graph_cut::mark_graph_cut(feat, "mosr.body.out", "feat"); + + auto out = pixel_shuffle(ctx->ggml_ctx, upsampler->forward(ctx, feat), config.scale); + sd::ggml_graph_cut::mark_graph_cut(out, "mosr.final", "out"); + return out; + } +}; + +struct MoSR : public GGMLRunner { + MoSRConfig config; + std::unique_ptr mosr_net; + + MoSR(ggml_backend_t backend, + const String2TensorStorage& tensor_storage_map = {}, + std::shared_ptr weight_manager = nullptr) + : GGMLRunner(backend, weight_manager), + config(MoSRConfig::detect_from_weights(tensor_storage_map)) { + if (!config.is_valid()) { + return; + } + mosr_net = std::make_unique(config); + mosr_net->init(params_ctx, tensor_storage_map, ""); + } + + std::string get_desc() override { + return "mosr"; + } + + void get_param_tensors(std::map& tensors) { + if (!mosr_net) { + return; + } + + mosr_net->get_param_tensors(tensors); + } + + ggml_cgraph* build_graph(const sd::Tensor& x_tensor) { + if (!mosr_net) + return nullptr; + constexpr int kGraphNodes = 1 << 16; // 65k + ggml_cgraph* gf = new_graph_custom(kGraphNodes); + ggml_tensor* x = make_input(x_tensor); + + auto runner_ctx = get_context(); + ggml_tensor* out = mosr_net->forward(&runner_ctx, x); + ggml_build_forward_expand(gf, out); + return gf; + } + + sd::Tensor compute(const int n_threads, + const sd::Tensor& x) { + auto get_graph = [&]() -> ggml_cgraph* { return build_graph(x); }; + auto result = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim()); + return result; + } +}; + +#endif // __SD_MODEL_UPSCALER_MOSR_HPP__ diff --git a/src/upscaler.cpp b/src/upscaler.cpp index dc3498e64..dfe5e4b22 100644 --- a/src/upscaler.cpp +++ b/src/upscaler.cpp @@ -25,13 +25,38 @@ UpscalerGGML::~UpscalerGGML() { // ModelManager holds raw ggml tensor pointers owned by the runner context. model_manager.reset(); esrgan_upscaler.reset(); + mosr_upscaler.reset(); } void UpscalerGGML::set_max_graph_vram_bytes(size_t max_vram_bytes) { max_graph_vram_bytes = max_vram_bytes; + if (runner() != nullptr) { + runner()->set_max_graph_vram_bytes(max_vram_bytes); + } +} + +GGMLRunner* UpscalerGGML::runner() { + if (mosr_upscaler) { + return mosr_upscaler.get(); + } + return esrgan_upscaler.get(); +} + +int UpscalerGGML::get_scale() { + if (mosr_upscaler) { + return mosr_upscaler->config.scale; + } if (esrgan_upscaler) { - esrgan_upscaler->set_max_graph_vram_bytes(max_vram_bytes); + return esrgan_upscaler->config.scale; + } + return 1; +} + +sd::Tensor UpscalerGGML::compute(const sd::Tensor& input_tensor) { + if (mosr_upscaler) { + return mosr_upscaler->compute(n_threads, input_tensor); } + return esrgan_upscaler->compute(n_threads, input_tensor); } bool UpscalerGGML::load_from_file(const std::string& esrgan_path, @@ -82,20 +107,33 @@ bool UpscalerGGML::load_from_file(const std::string& esrgan_path, } model_loader.set_wtype_override(model_data_type); LOG_INFO("Upscaler weight type: %s", ggml_type_name(model_data_type)); - esrgan_upscaler = std::make_shared(backend_for(SDBackendModule::UPSCALER), - model_loader.get_tensor_storage_map(), + const auto& tensor_storage_map = model_loader.get_tensor_storage_map(); + std::map tensors; + if (tensor_storage_map.count("gblocks.0.weight") > 0) { + mosr_upscaler = std::make_shared(backend_for(SDBackendModule::UPSCALER), + tensor_storage_map, model_manager); - if (esrgan_upscaler == nullptr || esrgan_upscaler->rrdb_net == nullptr) { - LOG_ERROR("init esrgan model from metadata failed: '%s'", esrgan_path.c_str()); - return false; + if (mosr_upscaler->mosr_net == nullptr) { + LOG_ERROR("init mosr model from metadata failed: '%s'", esrgan_path.c_str()); + return false; + } + mosr_upscaler->get_param_tensors(tensors); + } else { + esrgan_upscaler = std::make_shared(backend_for(SDBackendModule::UPSCALER), + tensor_storage_map, + model_manager); + if (esrgan_upscaler->rrdb_net == nullptr) { + LOG_ERROR("init esrgan model from metadata failed: '%s'", esrgan_path.c_str()); + return false; + } + esrgan_upscaler->get_param_tensors(tensors); } - esrgan_upscaler->set_max_graph_vram_bytes(max_graph_vram_bytes); + LOG_INFO("Upscaler architecture: %s", runner()->get_desc().c_str()); + runner()->set_max_graph_vram_bytes(max_graph_vram_bytes); if (direct) { - esrgan_upscaler->set_conv2d_direct_enabled(true); + runner()->set_conv2d_direct_enabled(true); } - std::map tensors; - esrgan_upscaler->get_param_tensors(tensors); if (!model_manager->set_loader(model_loader) || !model_manager->register_param_tensors(ModelComponent::Upscaler, std::move(tensors), @@ -103,7 +141,7 @@ bool UpscalerGGML::load_from_file(const std::string& esrgan_path, backend_for(SDBackendModule::UPSCALER), params_backend_for(SDBackendModule::UPSCALER)) || !model_manager->validate_registered_tensors()) { - LOG_ERROR("register esrgan tensors with model manager failed"); + LOG_ERROR("register upscaler tensors with model manager failed"); return false; } return true; @@ -113,7 +151,7 @@ sd::Tensor UpscalerGGML::upscale_tensor(const sd::Tensor& input_te sd::ParallelScope tensor_scope(&tensor_executor); if (input_tensor.empty() || input_tensor.dim() != 4 || (input_tensor.shape()[2] != 3 && input_tensor.shape()[2] != 4)) { - LOG_ERROR("esrgan expects a 4D RGB or RGBA image tensor"); + LOG_ERROR("upscaler expects a 4D RGB or RGBA image tensor"); return {}; } @@ -124,14 +162,14 @@ sd::Tensor UpscalerGGML::upscale_tensor(const sd::Tensor& input_te } const sd::Tensor& model_input = has_alpha ? rgb : input_tensor; sd::Tensor upscaled; - const int scale = esrgan_upscaler->config.scale; + const int scale = get_scale(); if (tile_size <= 0 || (input_tensor.shape()[0] <= tile_size && input_tensor.shape()[1] <= tile_size)) { - upscaled = esrgan_upscaler->compute(n_threads, model_input); + upscaled = compute(model_input); } else { auto on_processing = [&](const sd::Tensor& input_tile) -> sd::Tensor { - auto output_tile = esrgan_upscaler->compute(n_threads, input_tile); + auto output_tile = compute(input_tile); if (output_tile.empty()) { - LOG_ERROR("esrgan compute failed while processing a tile"); + LOG_ERROR("upscaler compute failed while processing a tile"); return {}; } return output_tile; @@ -148,9 +186,9 @@ sd::Tensor UpscalerGGML::upscale_tensor(const sd::Tensor& input_te false, on_processing); } - esrgan_upscaler->runner_end(); + runner()->runner_end(); if (upscaled.empty()) { - LOG_ERROR("esrgan compute failed"); + LOG_ERROR("upscaler compute failed"); return {}; } if (has_alpha) { @@ -168,7 +206,7 @@ sd_image_t UpscalerGGML::upscale(sd_image_t input_image, uint32_t upscale_factor sd::ParallelScope tensor_scope(&tensor_executor); // upscale_factor, unused for RealESRGAN_x4plus_anime_6B.pth sd_image_t upscaled_image = {0, 0, 0, nullptr}; - const int scale = esrgan_upscaler->config.scale; + const int scale = get_scale(); int output_width = (int)input_image.width * scale; int output_height = (int)input_image.height * scale; LOG_INFO("upscaling from (%i x %i) to (%i x %i)", @@ -256,10 +294,10 @@ bool upscale(upscaler_ctx_t* upscaler_ctx, } int get_upscale_factor(upscaler_ctx_t* upscaler_ctx) { - if (upscaler_ctx == nullptr || upscaler_ctx->upscaler == nullptr || upscaler_ctx->upscaler->esrgan_upscaler == nullptr) { + if (upscaler_ctx == nullptr || upscaler_ctx->upscaler == nullptr) { return 1; } - return upscaler_ctx->upscaler->esrgan_upscaler->config.scale; + return upscaler_ctx->upscaler->get_scale(); } int get_upscaler_model_scale(const char* model_path) { @@ -272,8 +310,12 @@ int get_upscaler_model_scale(const char* model_path) { return 0; } const auto& tensors = loader.get_tensor_storage_map(); - auto first = tensors.find("conv_first.weight"); - auto last = tensors.find("conv_last.weight"); + if (tensors.count("gblocks.0.weight") > 0) { + MoSRConfig config = MoSRConfig::detect_from_weights(tensors); + return config.is_valid() ? config.scale : 0; + } + auto first = tensors.find("conv_first.weight"); + auto last = tensors.find("conv_last.weight"); if (first == tensors.end() || last == tensors.end() || tensors.count("body.0.rdb1.conv1.weight") == 0 || first->second.n_dims != 4 || last->second.n_dims != 4 || diff --git a/src/upscaler.h b/src/upscaler.h index c23e9d0f2..b93065fcc 100644 --- a/src/upscaler.h +++ b/src/upscaler.h @@ -4,6 +4,7 @@ #include "core/ggml_extend_backend.h" #include "core/tensor.hpp" #include "model/upscaler/esrgan.hpp" +#include "model/upscaler/mosr.hpp" #include "model_manager.h" #include "stable-diffusion.h" @@ -15,6 +16,7 @@ struct UpscalerGGML { std::shared_ptr model_manager; ggml_type model_data_type = GGML_TYPE_F16; std::shared_ptr esrgan_upscaler; + std::shared_ptr mosr_upscaler; std::string esrgan_path; int n_threads; sd::ParallelExecutor tensor_executor; @@ -34,6 +36,9 @@ struct UpscalerGGML { bool load_from_file(const std::string& esrgan_path, int n_threads); void set_max_graph_vram_bytes(size_t max_vram_bytes); + GGMLRunner* runner(); + int get_scale(); + sd::Tensor compute(const sd::Tensor& input_tensor); sd::Tensor upscale_tensor(const sd::Tensor& input_tensor); sd_image_t upscale(sd_image_t input_image, uint32_t upscale_factor); }; From d637cfb935feb2254898a8f1524046c958f3df80 Mon Sep 17 00:00:00 2001 From: leejet Date: Mon, 12 Oct 2026 23:09:42 +0800 Subject: [PATCH 08/18] feat: add Qwen Image 2.1 Turbo sigma preset (#2130) --- docs/qwen_image_2.1.md | 10 ++++++++++ examples/common/common.cpp | 7 +++++-- src/pipeline/diffusion_engine.cpp | 6 +++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/qwen_image_2.1.md b/docs/qwen_image_2.1.md index 5da89c534..475404c7b 100644 --- a/docs/qwen_image_2.1.md +++ b/docs/qwen_image_2.1.md @@ -32,6 +32,16 @@ Run the following commands from the build directory. Use image dimensions divisi To use GGUF diffusion weights, set `--diffusion-model` to the path of a file such as `qwen_image_2.1-Q4_K.gguf`. +### Turbo + +For [Qwen Image 2.1 Turbo](https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo), use the Turbo diffusion weights and select its fixed 8-step sigma schedule with `--sigmas qwen-image-2.1-turbo`. For example, add these sampling options to the command above: + +```text +--sigmas qwen-image-2.1-turbo --steps 8 --cfg-scale 1.0 --sampling-method euler +``` + +The preset is equivalent to `--sigmas 1.0,0.978453,0.95418,0.926626,0.89508,0.845148,0.704534,0.414568,0.0`. It only supplies the sigma values; CFG scale and sampling method remain independently configurable. As with an explicit sigma list, the sigma count determines the actual number of sampling steps. The same preset name is also accepted by `--hires-sigmas` for the highres fix second pass. + ### Image editing Pass the reference image with `-r` and describe the edit in `-p`. Vision weights are required; the example below loads them separately with `--llm_vision`. diff --git a/examples/common/common.cpp b/examples/common/common.cpp index faa637266..46d33ce1e 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -1475,6 +1475,9 @@ ArgOptions SDGenerationParams::get_options() { return -1; } std::string sigmas_str = value; + if (sigmas_str == "qwen-image-2.1-turbo") { + sigmas_str = "1.0,0.978453,0.95418,0.926626,0.89508,0.845148,0.704534,0.414568,0.0"; + } if (!sigmas_str.empty() && sigmas_str.front() == '[') { sigmas_str.erase(0, 1); } @@ -1721,11 +1724,11 @@ ArgOptions SDGenerationParams::get_options() { on_scheduler_arg}, {"", "--sigmas", - "custom sigma values for the sampler, comma-separated (e.g., \"14.61,7.8,3.5,0.0\").", + "custom sigma values for the sampler, comma-separated (e.g., \"14.61,7.8,3.5,0.0\") or preset: qwen-image-2.1-turbo.", on_sigmas_arg}, {"", "--hires-sigmas", - "custom sigma values for the highres fix second pass, comma-separated (e.g., \"0.85,0.725,0.421875,0.0\").", + "custom sigma values for the highres fix second pass, comma-separated (e.g., \"0.85,0.725,0.421875,0.0\") or preset: qwen-image-2.1-turbo.", on_hires_sigmas_arg}, {"", "--skip-layers", diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index 6400b243e..ddc4eb15b 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -2735,9 +2735,9 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr batched_out = run_batched_condition(*positive_condition, c_concat_override); if (!batched_out.empty() && batched_out.dim() >= 4 && batched_out.shape()[3] == 2) { - auto parts = sd::ops::chunk(batched_out, 2, 3); - cond_out = std::move(parts[0]); - uncond_out = std::move(parts[1]); + auto parts = sd::ops::chunk(batched_out, 2, 3); + cond_out = std::move(parts[0]); + uncond_out = std::move(parts[1]); } } From 7e3e20a50b6801cfdd402a2ff9b8f74737db7ff8 Mon Sep 17 00:00:00 2001 From: Robert Oswald <47303535+robertosw@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:09:13 +0200 Subject: [PATCH 09/18] ci: add ROCm Dockerfile (#2094) --- docker/Dockerfile.rocm | 50 ++++++++++++++++++++++++++++++++++++++++++ docs/docker.md | 33 +++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 docker/Dockerfile.rocm diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm new file mode 100644 index 000000000..fe8a9b15e --- /dev/null +++ b/docker/Dockerfile.rocm @@ -0,0 +1,50 @@ +FROM rocm/dev-ubuntu-24.04 AS build + +# sd-server embeds the web UI at build time, so the build image needs Node/pnpm. +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential git cmake ninja-build ccache ca-certificates curl gnupg && \ + mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \ + gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \ + rm /tmp/nodesource-repo.gpg.key && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \ + apt-get update && apt-get install -y --no-install-recommends nodejs && \ + npm install -g pnpm@10.15.1 && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +WORKDIR /sd.cpp + +# ROCm's clang is not on PATH in Docker RUN shells (profile scripts are not loaded). +ENV PATH="/opt/rocm/llvm/bin:$PATH" + +COPY . . + +ARG AMDGPU_TARGETS="" + +RUN cmake . -B ./build -G Ninja \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DSD_HIPBLAS=ON \ + -DGGML_NATIVE=OFF \ + -DAMDGPU_TARGETS="${AMDGPU_TARGETS}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON +RUN cmake --build ./build --config Release --parallel + +FROM rocm/dev-ubuntu-24.04 AS runtime + +# ROCm clang links against its own OpenMP runtime in /opt/rocm/llvm/lib, +# and the ROCm runtime libs (hipblas, rocblas, amdhip64) live in /opt/rocm/lib. +ENV LD_LIBRARY_PATH="/opt/rocm/llvm/lib:/opt/rocm/lib:$LD_LIBRARY_PATH" + +RUN apt-get update && \ + apt-get install --yes --no-install-recommends libgomp1 && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +COPY --from=build /sd.cpp/build/bin /sd.cpp/bin +RUN printf '#!/bin/sh\nexec /sd.cpp/bin/sd-cli "$@"\n' > /sd-cli && \ + printf '#!/bin/sh\nexec /sd.cpp/bin/sd-server "$@"\n' > /sd-server && \ + chmod +x /sd-cli /sd-server + +ENTRYPOINT [ "/sd-cli" ] diff --git a/docs/docker.md b/docs/docker.md index 5bacdd2d4..28e9ab5df 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -24,12 +24,25 @@ docker build -f docker/Dockerfile -t sd . ## Building variants using Docker -Vulkan: +### Vulkan: ```shell docker build -f docker/Dockerfile.vulkan -t sd . ``` +### ROCm (AMD GPU): + +Defining one or multiple targets with `--build-arg AMDGPU_TARGETS` is strongly recommended. +Building the image will otherwise likely fail or build for a target that does not fit your +hardware. + +```shell +# Build for one target +docker build -f docker/Dockerfile.rocm --build-arg "AMDGPU_TARGETS=gfx1100" -t sd . +# Build for multiple targets +docker build -f docker/Dockerfile.rocm --build-arg "AMDGPU_TARGETS=gfx1030;gfx1100" -t sd . +``` + ## Run locally built image's CLI ```shell @@ -37,3 +50,21 @@ docker run --rm -v /path/to/models:/models -v /path/to/output/:/output sd [args. # For example # docker run --rm -v ./models:/models -v ./build:/output sd -m /models/sd-v1-4.ckpt -p "a lovely cat" -v -o /output/output.png ``` + +### ROCm (AMD GPU) + +The user that executes this command needs to be in the `video` group and have read and write +access to `/dev/dri` and `/dev/kfd`. + +`--device` and `--group-add` allow the container access to the GPU. + +```shell +docker run --rm --device=/dev/dri --device=/dev/kfd --group-add video \ + -v /path/to/models:/models -v /path/to/output/:/output \ + sd [args...] +# For example +# docker run --rm --device=/dev/dri --device=/dev/kfd --group-add video \ +# -v ./models:/models -v ./build:/output \ +# sd -m /models/sd-v1-4.ckpt -v -o /output/output.png \ +# -p "a lovely cat" +``` \ No newline at end of file From 60423ab5f6c4ccffb77e8d45e5b08eb97ca1645d Mon Sep 17 00:00:00 2001 From: losewayy <1134285612@qq.com> Date: Tue, 13 Oct 2026 21:10:05 +0800 Subject: [PATCH 10/18] fix: reclaim before accepting wrapped Vulkan iGPU memory estimates (#2124) --- src/model_manager.cpp | 29 +++++++++++++++++++++++++---- src/model_manager.h | 1 + 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/model_manager.cpp b/src/model_manager.cpp index 2815e168b..330b86f04 100644 --- a/src/model_manager.cpp +++ b/src/model_manager.cpp @@ -1757,7 +1757,14 @@ ModelManager::CapacityCheck ModelManager::check_capacity( } // Vulkan's heap budget subtraction can underflow when usage exceeds the budget. if (total_bytes > 0 && free_bytes > total_bytes && sd_backend_is(backend, "Vulkan")) { - return size_t{0}; + if (ggml_backend_dev_type(device) != GGML_BACKEND_DEVICE_TYPE_IGPU) { + return size_t{0}; + } + // A wrapped report may reflect real pressure; reclaim before accepting the tracked estimate. + LOG_VERBOSE("ignoring impossible free memory report on %s (%.2f MB free / %.2f MB total), using tracked residency", + ggml_backend_name(backend), free_bytes / (1024.0 * 1024.0), total_bytes / (1024.0 * 1024.0)); + free_bytes = total_bytes; + result.requires_reclamation = true; } if (total_bytes > 0) { free_bytes = std::min(free_bytes, resident < total_bytes ? total_bytes - resident : 0); @@ -1803,8 +1810,11 @@ bool ModelManager::fits_compute_backend_capacity( const DeviceMemoryRequest& request, const std::vector& required_params) const { std::vector states; - return resolve_required_tensor_states(required_params, states, request.compute_backend) && - check_capacity(request, states).fits(); + if (!resolve_required_tensor_states(required_params, states, request.compute_backend)) { + return false; + } + const auto capacity = check_capacity(request, states); + return !capacity.requires_reclamation && capacity.fits(); } bool ModelManager::ensure_compute_backend_capacity( @@ -1826,7 +1836,10 @@ bool ModelManager::ensure_compute_backend_capacity( return true; } - auto fits = [&]() { return check_capacity(request, required_states).fits(); }; + auto fits = [&]() { + const auto capacity = check_capacity(request, required_states); + return !capacity.requires_reclamation && capacity.fits(); + }; if (fits()) { return true; } @@ -1866,7 +1879,12 @@ bool ModelManager::ensure_compute_backend_capacity( eviction_states.insert(state); } }; + size_t last_eviction_count = 0; auto release_eviction_states = [&]() { + if (eviction_states.size() == last_eviction_count) { + return false; + } + last_eviction_count = eviction_states.size(); release_compute_staging_blocks(false, &eviction_states); release_params_storage_blocks(false, &eviction_states); return fits(); @@ -1947,6 +1965,9 @@ bool ModelManager::ensure_compute_backend_capacity( } revert_demotions(); + if (check_capacity(request, required_states).fits()) { + return true; + } const auto capacity = check_capacity(request, required_states, true); const std::string available_device = capacity.available_device_bytes == SIZE_MAX ? "unknown" diff --git a/src/model_manager.h b/src/model_manager.h index 643fc0724..fdb110825 100644 --- a/src/model_manager.h +++ b/src/model_manager.h @@ -152,6 +152,7 @@ class ModelManager : public DeviceResidencyManager { size_t required_budget_bytes = 0; size_t available_device_bytes = SIZE_MAX; size_t available_budget_bytes = SIZE_MAX; + bool requires_reclamation = false; bool fits() const { return required_device_bytes <= available_device_bytes && required_budget_bytes <= available_budget_bytes; From d763dae3b83f976487d1904a0038e880532efc10 Mon Sep 17 00:00:00 2001 From: leejet Date: Tue, 13 Oct 2026 22:58:45 +0800 Subject: [PATCH 11/18] feat: add per-module automatic flash attention selection (#2132) --- CMakeLists.txt | 10 ++ docs/anima.md | 2 +- docs/animatediff.md | 6 +- docs/backend.md | 73 +++++++++++ docs/backend_probe.md | 46 +++++++ docs/boogu_image.md | 4 +- docs/ernie_image.md | 4 +- docs/flux2.md | 14 +-- docs/hunyuan_video.md | 2 +- docs/ideogram4.md | 2 +- docs/int8_convrot.md | 2 +- docs/ip_adapter.md | 4 +- docs/iris.md | 2 +- docs/krea2.md | 2 +- docs/lens.md | 4 +- docs/lingbot_video.md | 2 +- docs/llada_image.md | 3 +- docs/longcat_image.md | 2 +- docs/ltx2.md | 14 +-- docs/mage_flow.md | 4 +- docs/ming_image.md | 2 +- docs/minimax_h3.md | 6 +- docs/minit2i.md | 1 - docs/ovis_image.md | 2 +- docs/performance.md | 2 +- docs/pid.md | 2 +- docs/qwen_image.md | 2 +- docs/qwen_image_2.1.md | 4 +- docs/qwen_image_edit.md | 6 +- docs/sage_attention.md | 8 +- docs/sefi_image.md | 6 +- docs/sensenova_u1.md | 1 - docs/troubleshooting.md | 3 +- docs/wan.md | 38 +++--- docs/z_image.md | 4 +- docs/z_image_l2p.md | 2 +- examples/cli/README.md | 6 + examples/common/common.cpp | 48 ++++++-- examples/common/common.h | 17 ++- examples/server/README.md | 3 +- include/stable-diffusion.h | 1 + scripts/format-code.ps1 | 2 + scripts/format-code.sh | 1 + src/core/ggml_extend.cpp | 2 +- src/core/ggml_extend_backend.cpp | 101 ++++++++++++++++ src/core/ggml_extend_backend.h | 2 + src/core/probe/backend_probe.cpp | 193 ++++++++++++++++++++++++++++++ src/core/probe/backend_probe.h | 64 ++++++++++ src/core/probe/flash_attn.cpp | 101 ++++++++++++++++ src/core/probe/flash_attn.h | 8 ++ src/pipeline/diffusion_engine.cpp | 81 ++++++++++--- src/pipeline/diffusion_engine.h | 6 +- src/stable-diffusion.cpp | 3 + 53 files changed, 805 insertions(+), 125 deletions(-) create mode 100644 docs/backend_probe.md create mode 100644 src/core/probe/backend_probe.cpp create mode 100644 src/core/probe/backend_probe.h create mode 100644 src/core/probe/flash_attn.cpp create mode 100644 src/core/probe/flash_attn.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 134719ca4..bd7c6d9e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -225,6 +225,8 @@ file(GLOB SD_LIB_SOURCES CONFIGURE_DEPENDS "src/core/*.h" "src/core/*.cpp" "src/core/*.hpp" + "src/core/probe/*.h" + "src/core/probe/*.cpp" "src/extensions/*.h" "src/extensions/*.cpp" "src/extensions/*.hpp" @@ -342,6 +344,14 @@ if (SD_CUDA) target_link_libraries(${SD_LIB} PUBLIC CUDA::cuda_driver) set_property(SOURCE src/core/ggml_extend_backend.cpp APPEND PROPERTY COMPILE_DEFINITIONS SD_USE_CUDA) endif() +if (SD_HIPBLAS) + find_package(hip REQUIRED CONFIG HINTS "$ENV{ROCM_PATH}" /opt/rocm) + target_link_libraries(${SD_LIB} PRIVATE hip::host) + set_property(SOURCE src/core/ggml_extend_backend.cpp APPEND PROPERTY COMPILE_DEFINITIONS SD_USE_HIP) + if (GGML_HIP_MMQ_MFMA) + set_property(SOURCE src/core/ggml_extend_backend.cpp APPEND PROPERTY COMPILE_DEFINITIONS SD_HIP_MFMA) + endif() +endif() target_include_directories(${SD_LIB} PUBLIC . src include) target_include_directories(${SD_LIB} PRIVATE src/core) target_include_directories(${SD_LIB} PUBLIC . thirdparty) diff --git a/docs/anima.md b/docs/anima.md index b852358fd..18b94952b 100644 --- a/docs/anima.md +++ b/docs/anima.md @@ -15,7 +15,7 @@ ## Examples ```sh -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\anima-preview.safetensors --vae ..\models\vae\qwen_image_vae.safetensors --llm ..\models\text_encoders\qwen_3_06b_base.safetensors -p "a lovely cat holding a sign says 'anima.cpp'" --cfg-scale 6.0 --sampling-method euler -v --offload-to-cpu --diffusion-fa +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\anima-preview.safetensors --vae ..\models\vae\qwen_image_vae.safetensors --llm ..\models\text_encoders\qwen_3_06b_base.safetensors -p "a lovely cat holding a sign says 'anima.cpp'" --cfg-scale 6.0 --sampling-method euler -v --offload-to-cpu ``` anima image example diff --git a/docs/animatediff.md b/docs/animatediff.md index 2e923b0bf..80c46ae64 100644 --- a/docs/animatediff.md +++ b/docs/animatediff.md @@ -51,7 +51,7 @@ SD 1.5 defaults (euler_a, low CFG) produces noise-like output. .\bin\Release\sd-cli.exe -M vid_gen \ --model ..\models\checkpoints\realisticVisionV60B1.safetensors \ --motion-module ..\models\animatediff\mm_sd15_v3.safetensors \ - --offload-to-cpu --diffusion-fa \ + --offload-to-cpu \ -p "a red apple on a wooden table" \ --cfg-scale 8.0 --sampling-method euler --scheduler discrete \ -H 512 -W 512 --video-frames 8 --fps 8 --steps 20 -s 42 \ @@ -66,7 +66,7 @@ use layer streaming: .\bin\Release\sd-cli.exe -M vid_gen \ --model ..\models\checkpoints\realisticVisionV60B1.safetensors \ --motion-module ..\models\animatediff\mm_sd15_v3.safetensors \ - --offload-to-cpu --diffusion-fa \ + --offload-to-cpu \ -p "photo of coastline, rocks, storm weather, wind, waves, lightning" \ --cfg-scale 8.0 --sampling-method euler --scheduler discrete \ -H 384 -W 384 --video-frames 16 --fps 8 --steps 20 -s 42 \ @@ -79,7 +79,7 @@ Low-VRAM streaming (verified with a 2 GiB cap on RTX 3060): .\bin\Release\sd-cli.exe -M vid_gen \ --model ..\models\checkpoints\realisticVisionV60B1.safetensors \ --motion-module ..\models\animatediff\mm_sd15_v3.safetensors \ - --max-vram 2.0 --diffusion-fa \ + --max-vram 2.0 \ -p "photo of coastline, rocks, storm weather, wind, waves, lightning" \ --cfg-scale 8.0 --sampling-method euler --scheduler discrete \ -H 384 -W 384 --video-frames 8 --fps 8 --steps 20 -s 42 \ diff --git a/docs/backend.md b/docs/backend.md index 70eef34f8..fbf79279f 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -215,6 +215,79 @@ If a VAE decode fails, decoding retries with spatial tiling even when `--auto-fi is off; supported video decoders try temporal tiling first and can then add spatial tiling. Spatial retries use half-size tiles along each latent dimension. +## Flash attention (`--flash-attn on|off|auto`) + +`--flash-attn` defaults to `auto`. Selection happens while creating the model +context, before inference, and is cached once per compute backend in that context. +Parameter-storage backends are not benchmarked. + +The option also accepts module assignments, using the same module aliases as +`--backend`: + +```shell +sd-cli -m model.safetensors -p "a cat" --flash-attn auto,te=off,vae=on +sd-cli -m model.safetensors -p "a cat" --flash-attn off,diffusion=auto +``` + +Supported modules are `diffusion`, `te`, `clip_vision`, `vae`, `controlnet`, and +`audio_encoder`. Modules without an assignment use the default (`auto` if no +default is supplied). A bare mode or `all=`, `default=`, or `*=` sets the default; +module-specific settings take precedence regardless of their position. Within +one assignment string, the last setting for the same module or default wins. +Repeating `--flash-attn` replaces the entire previous string. + +Only modules set to `auto` need backend probing. An explicit `on` or `off` for +one module does not change another module's decision, even when they share a +backend. Modules without configurable attention are rejected instead of silently +ignoring their settings. + +- `on` requests flash attention for all models. Unsupported attention operations + still fall back to ordinary attention. +- `off` disables flash attention, including the flash-attention fallback from + SageAttention. SageAttention itself remains controlled by `--sage-attn`. +- `auto` enables CPU and supported CUDA MMA configurations without timing a + benchmark. CUDA uses both the device's capability and the compiled GGML + architectures. HIP also skips timing on known CDNA MFMA architectures + (`gfx908`, `gfx90a`, `gfx940`/`gfx941`/`gfx942`, `gfx950`) and RDNA 3/3.5/4 + WMMA architectures (`gfx11xx`/`gfx12xx`). The CDNA shortcut requires + `GGML_HIP_MMQ_MFMA` to be enabled at build time. All shortcuts still check + support for the FA probe graph. Vulkan devices whose description identifies + NVIDIA RTX hardware (GeForce RTX, TITAN RTX, Quadro RTX and professional RTX) + also skip timing. This is a device-name heuristic and needs no CUDA runtime. + Unsupported graphs are disabled; other configurations, including older or + unknown HIP architectures and other Vulkan devices, are benchmarked. + +The probe compares ordinary and flash attention through the same attention +builder used by models, including layout conversion and F16 conversion. It uses +two small self-attention graphs (512 tokens with head size 64, and 1024 tokens +with head size 128, both with four heads), checks their outputs, warms up each +path, then alternates five synchronized measurements. Flash attention is enabled +when the sum of its median times is at least 5% lower. These representative +shapes are a startup heuristic, not a guarantee for every model or resolution. +Allocation/execution failures and inconclusive probes select `off`. Timing stops +starting new measurement pairs after one second of measured computation; +backend initialization, warmup and an in-flight pair can take longer. + +Different modules can select different results. A multi-device model enables +flash attention only if all of its compute devices select `on`, preserving the +model's existing single boolean setting. This conservative rule also covers +row-split configurations, which can fall back to layer split during initialization. + +`--fa` and `--diffusion-fa` are deprecated and emit a warning when used. +Use `--flash-attn on` and `--flash-attn off,diffusion=on`, respectively. +Both aliases write the same mode string as `--flash-attn`; the last occurrence +of any of these options wins. Library callers +can set `sd_ctx_params_t::flash_attn_mode` to the same string, including +`"on"`, `"off"`, `"auto"`, or `"auto,te=off,vae=on"`. +A null or empty string uses automatic selection unless a +legacy `flash_attn` or `diffusion_flash_attn` boolean is set. The context copies +the string; no new enum is required. + +```shell +sd-cli -m model.safetensors -p "a cat" --flash-attn auto +sd-cli -m model.safetensors -p "a cat" --flash-attn off +``` + ## Modules | Module | Purpose | Accepted names | diff --git a/docs/backend_probe.md b/docs/backend_probe.md new file mode 100644 index 000000000..db9ee238c --- /dev/null +++ b/docs/backend_probe.md @@ -0,0 +1,46 @@ +# Adding backend performance probes + +`src/core/probe/backend_probe.h` provides the shared measurement machinery. Operator +policy stays separate: `src/core/probe/flash_attn.cpp`, for example, defines attention +shapes and its CPU/CUDA/HIP/Vulkan shortcuts, then calls `sd_run_backend_probe` for unknown +configurations. A future convolution probe can use the same runner without +changing the attention implementation. + +For each representative shape, create a `BackendProbeCase`: + +- `build(graph, candidate)` creates inputs and either the reference or candidate + graph in `graph.ctx`, registers input tensors in `graph.inputs`, and assigns + `graph.output`. Leave the output null if the requested implementation is + unavailable; do not silently benchmark its fallback as the candidate. +- By default, inputs receive deterministic F32 data, with the same seed for both + variants. Supply `initialize` for F16 convolution weights, integer inputs, or + operator-specific values. It runs after allocation, outside timed execution. + Both variants must receive equivalent logical inputs. +- Default output validation requires matching shapes and contiguous F32 outputs, + checks finiteness, and applies absolute and relative tolerances. Supply + `compare` for other output types or validation rules. + +Graphs must be safe to execute repeatedly without modifying their inputs. +`BackendProbeGraph` owns its GGML context and buffer, so all return paths release +probe resources. Constructors allocate metadata only; `supported()` can also be +used before a known-architecture shortcut, without allocating device tensors. + +`BackendProbeOptions` controls graph metadata capacity, warmup count, measurement +count, measured-time budget, minimum free device memory, comparison tolerances, +and minimum improvement. Warmup and allocation are excluded from timing. The +runner alternates the two variants, synchronizes the backend around execution, +and compares sums of per-case medians. The time budget is checked before each +measurement pair; it cannot interrupt an in-flight backend operation. + +Only use `candidate_faster` when `completed` is true. On incomplete probes, +`error` reports why comparison failed and the timing totals may be partial. +The caller handles logging, known architecture rules, fallback policy, and +context-lifetime caching. Keep caches separate by operation (or key a shared +cache by both operation and backend); an attention result must not select a +convolution implementation. If shapes or settings vary within one context, +include them in the corresponding cache key as well. + +No convolution policy or CLI option is added by this infrastructure. + +Module assignment parsing remains in `src/core/ggml_extend_backend.cpp`, next +to backend assignment parsing. The probe directory does not parse CLI/API options. diff --git a/docs/boogu_image.md b/docs/boogu_image.md index 2744aae57..c5ad97dd8 100644 --- a/docs/boogu_image.md +++ b/docs/boogu_image.md @@ -17,7 +17,7 @@ Boogu Image uses a Boogu diffusion transformer, the FLUX VAE, and Qwen3-VL as th ### Boogu Image Base ``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\boogu_image_base_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --vae ..\models\vae\ae.sft -p "a lovely cat" --diffusion-fa -v --offload-to-cpu +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\boogu_image_base_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --vae ..\models\vae\ae.sft -p "a lovely cat" -v --offload-to-cpu ``` Boogu Image Base example @@ -25,7 +25,7 @@ Boogu Image uses a Boogu diffusion transformer, the FLUX VAE, and Qwen3-VL as th ### Boogu Image Edit ``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\boogu_image_edit_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --llm_vision ..\models\text_encoders\mmproj-Qwen3VL-8B-Instruct-F16.gguf --vae ..\models\vae\ae.sft --diffusion-fa -v --offload-to-cpu -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'boogu.cpp'" +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\boogu_image_edit_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --llm_vision ..\models\text_encoders\mmproj-Qwen3VL-8B-Instruct-F16.gguf --vae ..\models\vae\ae.sft -v --offload-to-cpu -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'boogu.cpp'" ``` Boogu Image Edit example \ No newline at end of file diff --git a/docs/ernie_image.md b/docs/ernie_image.md index 99f942211..3b1fc124c 100644 --- a/docs/ernie_image.md +++ b/docs/ernie_image.md @@ -21,7 +21,7 @@ You can run ERNIE-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — o ### ERNIE-Image-Turbo ``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\ernie-image-turbo.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 8 -v --offload-to-cpu --diffusion-fa +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\ernie-image-turbo.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 8 -v --offload-to-cpu ``` ERNIE-Image Turbo example @@ -29,7 +29,7 @@ You can run ERNIE-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — o ### ERNIE-Image ``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\ernie-image-UD-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 5.0 -v --offload-to-cpu --diffusion-fa +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\ernie-image-UD-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 5.0 -v --offload-to-cpu ``` ERNIE-Image example diff --git a/docs/flux2.md b/docs/flux2.md index d3b2eff1f..53fc8be5c 100644 --- a/docs/flux2.md +++ b/docs/flux2.md @@ -16,7 +16,7 @@ ### Examples ``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux2-dev-Q4_K_S.gguf --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf -r .\kontext_input.png -p "change 'flux.cpp' to 'flux2-dev.cpp'" --cfg-scale 1.0 --sampling-method euler -v --diffusion-fa --offload-to-cpu +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux2-dev-Q4_K_S.gguf --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf -r .\kontext_input.png -p "change 'flux.cpp' to 'flux2-dev.cpp'" --cfg-scale 1.0 --sampling-method euler -v --offload-to-cpu ``` flux2 example @@ -42,19 +42,19 @@ ### Examples ``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-2-klein-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu --diffusion-fa +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-2-klein-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu ``` flux2-klein-4b ``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-2-klein-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\qwen_3_4b.safetensors -r .\kontext_input.png -p "change 'flux.cpp' to 'klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --diffusion-fa --offload-to-cpu --steps 4 +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-2-klein-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\qwen_3_4b.safetensors -r .\kontext_input.png -p "change 'flux.cpp' to 'klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --offload-to-cpu --steps 4 ``` flux2-klein-4b-edit ``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-2-klein-base-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "a lovely cat" --cfg-scale 4.0 --steps 20 -v --offload-to-cpu --diffusion-fa +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-2-klein-base-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "a lovely cat" --cfg-scale 4.0 --steps 20 -v --offload-to-cpu ``` flux2-klein-base-4b @@ -78,19 +78,19 @@ ### Examples ``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-2-klein-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\qwen_3_8b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu --diffusion-fa +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-2-klein-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\qwen_3_8b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu ``` flux2-klein-9b ``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-2-klein-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\qwen_3_8b.safetensors -r .\kontext_input.png -p "change 'flux.cpp' to 'klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --diffusion-fa --offload-to-cpu --steps 4 +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-2-klein-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\qwen_3_8b.safetensors -r .\kontext_input.png -p "change 'flux.cpp' to 'klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --offload-to-cpu --steps 4 ``` flux2-klein-9b-edit ``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-2-klein-base-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\qwen_3_8b.safetensors -p "a lovely cat" --cfg-scale 4.0 --steps 20 -v --offload-to-cpu --diffusion-fa +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-2-klein-base-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\qwen_3_8b.safetensors -p "a lovely cat" --cfg-scale 4.0 --steps 20 -v --offload-to-cpu ``` flux2-klein-base-9b \ No newline at end of file diff --git a/docs/hunyuan_video.md b/docs/hunyuan_video.md index 2b05e0f71..fadb884ef 100644 --- a/docs/hunyuan_video.md +++ b/docs/hunyuan_video.md @@ -18,7 +18,7 @@ and ByT5 Small GlyphXL for glyph-aware text conditioning. ## Text-to-video example ```shell -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\hunyuanvideo1.5_720p_t2v_fp16.safetensors --vae ..\models\vae\hunyuanvideo15_vae_fp16.safetensors --llm ..\models\text_encoders\qwen_2.5_vl_7b.safetensors --t5xxl ..\models\text_encoders\byt5_small_glyphxl_fp16.safetensors -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -W 1280 -H 720 --offload-to-cpu --diffusion-fa --video-frames 33 --vae-tiling +.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\hunyuanvideo1.5_720p_t2v_fp16.safetensors --vae ..\models\vae\hunyuanvideo15_vae_fp16.safetensors --llm ..\models\text_encoders\qwen_2.5_vl_7b.safetensors --t5xxl ..\models\text_encoders\byt5_small_glyphxl_fp16.safetensors -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -W 1280 -H 720 --offload-to-cpu --video-frames 33 --vae-tiling ``` \ No newline at end of file diff --git a/docs/ideogram4.md b/docs/ideogram4.md index 003a02d59..f0f1dd250 100644 --- a/docs/ideogram4.md +++ b/docs/ideogram4.md @@ -49,7 +49,7 @@ If you want lower VRAM usage, you can change the quantization from q8_0 to a low ## Examples ```sh -.\bin\Release\sd-cli.exe --diffusion-model ideogram4-Q8_0.gguf --uncond-diffusion-model ideogram4_uncond-Q8_0.gguf --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors -p '{"high_level_description":"A square 1024 x 1024 luxury fashion magazine cover featuring exactly one short chubby fluffy cat as the main model. The cat sits on a soft ivory studio floor, facing the viewer with a stylish calm expression, wearing tiny black sunglasses, a red silk scarf, and a small gold collar charm. In front of the cat on the floor is a wide horizontal luxury nameplate that clearly reads ideogram4.cpp. The whole design feels premium, fashionable, clean, and editorial.","style_description":{"aesthetics":"luxury fashion magazine cover, high-end pet couture campaign, minimalist editorial design, elegant studio photography, soft paper texture, refined typography, fashionable and polished","lighting":"Soft diffused studio lighting, gentle spotlight on the cat, subtle floor shadow, warm ivory highlights, clean separation between subject and background","photo":"high-resolution fashion editorial photography look, front-facing cat portrait, crisp fur details, glossy sunglasses, clear readable nameplate text, shallow depth of field","medium":"mixed media fashion photography and premium editorial graphic design","color_palette":["#F4EFE7","#111111","#D8B56D","#B73A3A","#FFFFFF","#8A7A6A"]},"compositional_deconstruction":{"canvas":"Square 1024 x 1024 canvas with a normal upright orientation. Do not rotate the poster or any text. Use a clean fashion magazine cover layout.","background":"Warm ivory studio backdrop with subtle paper grain, a soft spotlight gradient, faint floor shadow, and a few minimal gold editorial lines. The background is spacious, premium, and uncluttered.","layout":"Top center has a small elegant headline. Center area features one cat as the main fashion model. Lower foreground has a wide horizontal luxury nameplate placed on the floor in front of the cat. Bottom center has a small footer. All text is horizontal, upright, and readable left to right.","elements":[{"type":"text","desc":"Top center headline reading LOOK WHAT I FOUND in a refined high-fashion serif font. The headline is horizontal, centered, elegant, and secondary to the nameplate text."},{"type":"obj","desc":"Exactly one short chubby fluffy cat sitting in the center like a luxury fashion model. The cat has a large round head, compact body, short legs, soft detailed fur, expressive eyes, and a calm confident pose. The cat is cute and rounded, not tall, not stretched, not duplicated."},{"type":"obj","desc":"Tiny glossy black sunglasses worn naturally by the cat, slightly oversized but still showing the cat face clearly. The sunglasses add a chic fashion-editorial attitude."},{"type":"obj","desc":"A red silk scarf tied neatly around the cat neck, with soft folds and a couture feeling. The scarf must not cover the cat face or the nameplate."},{"type":"obj","desc":"A small gold collar charm or fashion accessory under the scarf, subtle and premium, adding a luxury campaign detail."},{"type":"obj","desc":"In the lower foreground, place a wide horizontal luxury nameplate on the floor in front of the cat. The nameplate is low, flat, landscape-oriented, much wider than tall, like a fashion show seat card or premium display plaque. It is centered, front-facing, level, and fully visible. It must not become vertical, tall, standing, rotated, or side-facing."},{"type":"text","desc":"Print the exact text ideogram4.cpp only on the wide horizontal nameplate. Use clean bold black lettering, perfectly spelled, lowercase, with the number 4 and .cpp extension. The text must fit completely inside the nameplate, stay horizontal, and be readable from left to right."},{"type":"obj","desc":"Add sparse premium editorial accents around the edges: thin gold lines, small code brackets, tiny cursor marks, subtle dots, and minimal geometric details. No extra cats, no stickers, no animal faces, no busy decorations."},{"type":"text","desc":"Bottom center footer reading tiny paws, big compile energy in a small refined monospace or editorial font. The footer is horizontal, centered, understated, and much smaller than the nameplate text."}]}}' --diffusion-fa -v --offload-to-cpu -H 1024 -W 1024 +.\bin\Release\sd-cli.exe --diffusion-model ideogram4-Q8_0.gguf --uncond-diffusion-model ideogram4_uncond-Q8_0.gguf --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors -p '{"high_level_description":"A square 1024 x 1024 luxury fashion magazine cover featuring exactly one short chubby fluffy cat as the main model. The cat sits on a soft ivory studio floor, facing the viewer with a stylish calm expression, wearing tiny black sunglasses, a red silk scarf, and a small gold collar charm. In front of the cat on the floor is a wide horizontal luxury nameplate that clearly reads ideogram4.cpp. The whole design feels premium, fashionable, clean, and editorial.","style_description":{"aesthetics":"luxury fashion magazine cover, high-end pet couture campaign, minimalist editorial design, elegant studio photography, soft paper texture, refined typography, fashionable and polished","lighting":"Soft diffused studio lighting, gentle spotlight on the cat, subtle floor shadow, warm ivory highlights, clean separation between subject and background","photo":"high-resolution fashion editorial photography look, front-facing cat portrait, crisp fur details, glossy sunglasses, clear readable nameplate text, shallow depth of field","medium":"mixed media fashion photography and premium editorial graphic design","color_palette":["#F4EFE7","#111111","#D8B56D","#B73A3A","#FFFFFF","#8A7A6A"]},"compositional_deconstruction":{"canvas":"Square 1024 x 1024 canvas with a normal upright orientation. Do not rotate the poster or any text. Use a clean fashion magazine cover layout.","background":"Warm ivory studio backdrop with subtle paper grain, a soft spotlight gradient, faint floor shadow, and a few minimal gold editorial lines. The background is spacious, premium, and uncluttered.","layout":"Top center has a small elegant headline. Center area features one cat as the main fashion model. Lower foreground has a wide horizontal luxury nameplate placed on the floor in front of the cat. Bottom center has a small footer. All text is horizontal, upright, and readable left to right.","elements":[{"type":"text","desc":"Top center headline reading LOOK WHAT I FOUND in a refined high-fashion serif font. The headline is horizontal, centered, elegant, and secondary to the nameplate text."},{"type":"obj","desc":"Exactly one short chubby fluffy cat sitting in the center like a luxury fashion model. The cat has a large round head, compact body, short legs, soft detailed fur, expressive eyes, and a calm confident pose. The cat is cute and rounded, not tall, not stretched, not duplicated."},{"type":"obj","desc":"Tiny glossy black sunglasses worn naturally by the cat, slightly oversized but still showing the cat face clearly. The sunglasses add a chic fashion-editorial attitude."},{"type":"obj","desc":"A red silk scarf tied neatly around the cat neck, with soft folds and a couture feeling. The scarf must not cover the cat face or the nameplate."},{"type":"obj","desc":"A small gold collar charm or fashion accessory under the scarf, subtle and premium, adding a luxury campaign detail."},{"type":"obj","desc":"In the lower foreground, place a wide horizontal luxury nameplate on the floor in front of the cat. The nameplate is low, flat, landscape-oriented, much wider than tall, like a fashion show seat card or premium display plaque. It is centered, front-facing, level, and fully visible. It must not become vertical, tall, standing, rotated, or side-facing."},{"type":"text","desc":"Print the exact text ideogram4.cpp only on the wide horizontal nameplate. Use clean bold black lettering, perfectly spelled, lowercase, with the number 4 and .cpp extension. The text must fit completely inside the nameplate, stay horizontal, and be readable from left to right."},{"type":"obj","desc":"Add sparse premium editorial accents around the edges: thin gold lines, small code brackets, tiny cursor marks, subtle dots, and minimal geometric details. No extra cats, no stickers, no animal faces, no busy decorations."},{"type":"text","desc":"Bottom center footer reading tiny paws, big compile energy in a small refined monospace or editorial font. The footer is horizontal, centered, understated, and much smaller than the nameplate text."}]}}' -v --offload-to-cpu -H 1024 -W 1024 ``` ideogram4 image example diff --git a/docs/int8_convrot.md b/docs/int8_convrot.md index dcbda8376..04d9c4f73 100644 --- a/docs/int8_convrot.md +++ b/docs/int8_convrot.md @@ -102,5 +102,5 @@ The dedicated CUDA and HIP convrot activation paths currently require a group si ComfyUI INT8 convrot safetensors can be passed to `--diffusion-model` without conversion: ```powershell -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\krea2_turbo_int8_convrot.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\wan_2.1_vae.safetensors -p "a lovely cat holding a sign says 'krea2.cpp'" --steps 8 --cfg-scale 1 --diffusion-fa -v --offload-to-cpu +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\krea2_turbo_int8_convrot.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\wan_2.1_vae.safetensors -p "a lovely cat holding a sign says 'krea2.cpp'" --steps 8 --cfg-scale 1 -v --offload-to-cpu ``` diff --git a/docs/ip_adapter.md b/docs/ip_adapter.md index 76a651011..27f9ae689 100644 --- a/docs/ip_adapter.md +++ b/docs/ip_adapter.md @@ -48,7 +48,7 @@ sd-cli -m ..\models\sd_v1.5.safetensors --clip_vision ..\models\clip_vision_h.sa ## Example (SDXL) ``` -sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter_sdxl_vit-h.safetensors --ip-adapter-image ..\assets\reference.png --ip-adapter-strength 0.8 -p "a woman, best quality" -n "lowres, bad anatomy" --cfg-scale 6 --steps 25 --sampling-method dpm++2m --scheduler karras -W 1024 -H 1024 --diffusion-fa --vae-tiling +sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter_sdxl_vit-h.safetensors --ip-adapter-image ..\assets\reference.png --ip-adapter-strength 0.8 -p "a woman, best quality" -n "lowres, bad anatomy" --cfg-scale 6 --steps 25 --sampling-method dpm++2m --scheduler karras -W 1024 -H 1024 --vae-tiling ``` The SDXL VAE decode at 1024x1024 is memory heavy; add `--vae-tiling` (and @@ -83,5 +83,5 @@ Add the usual Control Net options to keep the reference appearance while controlling the pose: ``` -sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter_sdxl_vit-h.safetensors --ip-adapter-image ..\assets\character.png --ip-adapter-strength 0.9 --control-net ..\models\OpenPoseXL2.safetensors --control-image ..\assets\pose.png --control-strength 0.8 -p "a character, side view" --cfg-scale 6 --steps 25 -W 1024 -H 1024 --diffusion-fa --vae-tiling +sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter_sdxl_vit-h.safetensors --ip-adapter-image ..\assets\character.png --ip-adapter-strength 0.9 --control-net ..\models\OpenPoseXL2.safetensors --control-image ..\assets\pose.png --control-strength 0.8 -p "a character, side view" --cfg-scale 6 --steps 25 -W 1024 -H 1024 --vae-tiling ``` diff --git a/docs/iris.md b/docs/iris.md index 940fd1eaa..322a3da1a 100644 --- a/docs/iris.md +++ b/docs/iris.md @@ -13,7 +13,7 @@ Iris-3B generates images directly in pixel space and uses Qwen3-VL-4B-Instruct a ## Examples ``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\iris-3b.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf -p "a lovely cat" --cfg-scale 3 -H 1024 -W 1024 --diffusion-fa +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\iris-3b.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf -p "a lovely cat" --cfg-scale 3 -H 1024 -W 1024 ``` iris-3b example diff --git a/docs/krea2.md b/docs/krea2.md index 10f8eab80..67fad023e 100644 --- a/docs/krea2.md +++ b/docs/krea2.md @@ -21,7 +21,7 @@ Krea2 uses a Krea2 diffusion transformer, the Wan2.1 VAE, and Qwen3-VL 4B as the ### Krea2 ``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\Krea-2-Raw-Q8_0.gguf --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\wan_2.1_vae.safetensors -p "a lovely cat holding a sign says 'krea2.cpp'" --diffusion-fa -v --offload-to-cpu +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\Krea-2-Raw-Q8_0.gguf --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\wan_2.1_vae.safetensors -p "a lovely cat holding a sign says 'krea2.cpp'" -v --offload-to-cpu ``` Krea2 Raw example diff --git a/docs/lens.md b/docs/lens.md index 64dc594c4..09e9575ac 100644 --- a/docs/lens.md +++ b/docs/lens.md @@ -22,7 +22,7 @@ Lens and Lens Turbo require an external GPT-OSS `tokenizer.json` matching the te ### Lens ``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\lens_bf16.safetensors --llm "..\models\text_encoders\gpt-oss-20b-UD-Q8_K_XL.gguf" --tokenizer ..\models\tokenizers\tokenizer_gpt_oss.json --vae ..\models\vae\flux2_ae.safetensors --cfg-scale 5.0 -p "A crystal dragon soaring through an aurora borealis sky, its entire body made of transparent faceted crystal refracting the green and purple aurora light into rainbow spectra, ice particles trailing from its wings, high fantasy digital art" --diffusion-fa -v +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\lens_bf16.safetensors --llm "..\models\text_encoders\gpt-oss-20b-UD-Q8_K_XL.gguf" --tokenizer ..\models\tokenizers\tokenizer_gpt_oss.json --vae ..\models\vae\flux2_ae.safetensors --cfg-scale 5.0 -p "A crystal dragon soaring through an aurora borealis sky, its entire body made of transparent faceted crystal refracting the green and purple aurora light into rainbow spectra, ice particles trailing from its wings, high fantasy digital art" -v ``` Lens example @@ -30,7 +30,7 @@ Lens and Lens Turbo require an external GPT-OSS `tokenizer.json` matching the te ### Lens Turbo ``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\lens_turbo_bf16.safetensors --llm "..\models\text_encoders\gpt-oss-20b-UD-Q8_K_XL.gguf" --tokenizer ..\models\tokenizers\tokenizer_gpt_oss.json --vae ..\models\vae\flux2_ae.safetensors --cfg-scale 1.0 -p "A crystal dragon soaring through an aurora borealis sky, its entire body made of transparent faceted crystal refracting the green and purple aurora light into rainbow spectra, ice particles trailing from its wings, high fantasy digital art" --diffusion-fa -v --steps 4 +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\lens_turbo_bf16.safetensors --llm "..\models\text_encoders\gpt-oss-20b-UD-Q8_K_XL.gguf" --tokenizer ..\models\tokenizers\tokenizer_gpt_oss.json --vae ..\models\vae\flux2_ae.safetensors --cfg-scale 1.0 -p "A crystal dragon soaring through an aurora borealis sky, its entire body made of transparent faceted crystal refracting the green and purple aurora light into rainbow spectra, ice particles trailing from its wings, high fantasy digital art" -v --steps 4 ``` Lens Turbo example diff --git a/docs/lingbot_video.md b/docs/lingbot_video.md index d1248bd5b..b92165ab5 100644 --- a/docs/lingbot_video.md +++ b/docs/lingbot_video.md @@ -28,5 +28,5 @@ Lingbot Video uses a Lingbot diffusion transformer, the Wan2.1 VAE, and Qwen3-VL ### LingBot-Video T2V ```sh -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\lingbot-video-dens-1.3b.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\wan_2.1_vae.safetensors -p '{"caption":{"comprehensive_description":"A lovely cat sits comfortably on a soft cushion near a sunlit window, looking calm, gentle, and adorable. The cat has soft fluffy fur, bright expressive eyes, small rounded ears, delicate whiskers, and a relaxed posture. Warm daylight falls across the cat from one side, creating soft highlights on the fur and a cozy glow around the scene. The background is softly blurred, showing hints of a peaceful indoor home environment with warm tones and gentle natural light. The overall atmosphere is cute, tender, serene, and photorealistic, emphasizing the cat''s charming appearance, soft texture, and affectionate presence.","camera_info":{"color":"Warm","frame_size":"Close Up","shot_type_angle":"Eye level","lens_size":"Medium Lens","composition":"Centered balanced","lighting":"Soft light","lighting_type":"Daylight"},"world_knowledge":[],"prominent_elements":[{"name":"lovely cat","description":"A cute and gentle domestic cat with soft fluffy fur, expressive eyes, and a calm relaxed presence.","location":"center of the frame","relative_size":"large","shape_and_color":"Small animal body with rounded face, triangular ears, bright eyes, and soft fur in warm natural tones","texture":"soft, fluffy, silky","appearance_details":"The cat has clean well-groomed fur, delicate whiskers, small ears, a cute nose, and bright attentive eyes. Its expression appears calm, affectionate, and slightly curious.","relationship":"Acts as the main subject and emotional focal point of the scene.","orientation":"facing the camera","pose":"sitting comfortably with a relaxed posture","expression":"gentle, adorable, calm, slightly curious","clothing":"","gender":"","skin_tone_and_texture":""},{"name":"cat eyes","description":"Bright expressive eyes that give the cat a sweet and affectionate appearance.","location":"upper center of the cat face","relative_size":"small","shape_and_color":"Round almond-like eyes with glossy reflections","texture":"clear, glossy, reflective","appearance_details":"The eyes catch the soft daylight, creating small natural highlights that make the cat look vivid and alive.","relationship":"Enhance the emotional charm and cuteness of the cat.","orientation":"looking toward the camera","pose":"","expression":"soft and attentive","clothing":"","gender":"","skin_tone_and_texture":""},{"name":"soft cushion","description":"A comfortable cushion or blanket where the cat is resting.","location":"bottom portion of the frame","relative_size":"medium","shape_and_color":"Soft rounded fabric surface in light warm neutral tones","texture":"plush, fabric, cozy","appearance_details":"The cushion gently supports the cat and adds a comfortable home-like feeling to the scene.","relationship":"Provides a cozy resting place for the cat.","orientation":"horizontal","pose":"","expression":"","clothing":"","gender":"","skin_tone_and_texture":""},{"name":"sunlit indoor background","description":"A softly blurred indoor background with warm daylight and peaceful home atmosphere.","location":"behind the cat, filling the upper and side areas of the frame","relative_size":"large","shape_and_color":"Soft abstract shapes in warm beige, cream, and pale golden tones","texture":"soft, blurry, bokeh-like","appearance_details":"The background is intentionally out of focus, keeping attention on the cat while creating a cozy and serene mood.","relationship":"Provides a warm and gentle environment that supports the cute domestic scene.","orientation":"upright","pose":"","expression":"","clothing":"","gender":"","skin_tone_and_texture":"","is_cluster":true,"number_of_objects":"numerous"}]}}' -n '{"universal_negative":{"visual_quality":["low quality","worst quality","blurry","pixelated","jpeg artifacts","low resolution","unstable color","color flicker","underexposed","overexposed","invisible subject","subject hidden in darkness"],"artistic_style":["painting","illustration","drawing","cartoon","3d render","cgi","sketch","digital art"],"composition_and_content":["text","watermark","signature","logo","subtitles","pillarboxed","side bars","portrait image in landscape frame"],"temporal_and_motion_stability":["flickering","jittery","motion blur","temporal inconsistency","warping","morphing","incoherent motion","unnatural movement","static object with sudden jump","frame-to-frame inconsistency"],"material_and_structure":["plastic-like glass","unrealistic texture","deformed bottle","liquid freezing improperly","distorted reflections"]}}' --diffusion-fa --offload-to-cpu --cfg-scale 3 --video-frames 33 -v +.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\lingbot-video-dens-1.3b.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\wan_2.1_vae.safetensors -p '{"caption":{"comprehensive_description":"A lovely cat sits comfortably on a soft cushion near a sunlit window, looking calm, gentle, and adorable. The cat has soft fluffy fur, bright expressive eyes, small rounded ears, delicate whiskers, and a relaxed posture. Warm daylight falls across the cat from one side, creating soft highlights on the fur and a cozy glow around the scene. The background is softly blurred, showing hints of a peaceful indoor home environment with warm tones and gentle natural light. The overall atmosphere is cute, tender, serene, and photorealistic, emphasizing the cat''s charming appearance, soft texture, and affectionate presence.","camera_info":{"color":"Warm","frame_size":"Close Up","shot_type_angle":"Eye level","lens_size":"Medium Lens","composition":"Centered balanced","lighting":"Soft light","lighting_type":"Daylight"},"world_knowledge":[],"prominent_elements":[{"name":"lovely cat","description":"A cute and gentle domestic cat with soft fluffy fur, expressive eyes, and a calm relaxed presence.","location":"center of the frame","relative_size":"large","shape_and_color":"Small animal body with rounded face, triangular ears, bright eyes, and soft fur in warm natural tones","texture":"soft, fluffy, silky","appearance_details":"The cat has clean well-groomed fur, delicate whiskers, small ears, a cute nose, and bright attentive eyes. Its expression appears calm, affectionate, and slightly curious.","relationship":"Acts as the main subject and emotional focal point of the scene.","orientation":"facing the camera","pose":"sitting comfortably with a relaxed posture","expression":"gentle, adorable, calm, slightly curious","clothing":"","gender":"","skin_tone_and_texture":""},{"name":"cat eyes","description":"Bright expressive eyes that give the cat a sweet and affectionate appearance.","location":"upper center of the cat face","relative_size":"small","shape_and_color":"Round almond-like eyes with glossy reflections","texture":"clear, glossy, reflective","appearance_details":"The eyes catch the soft daylight, creating small natural highlights that make the cat look vivid and alive.","relationship":"Enhance the emotional charm and cuteness of the cat.","orientation":"looking toward the camera","pose":"","expression":"soft and attentive","clothing":"","gender":"","skin_tone_and_texture":""},{"name":"soft cushion","description":"A comfortable cushion or blanket where the cat is resting.","location":"bottom portion of the frame","relative_size":"medium","shape_and_color":"Soft rounded fabric surface in light warm neutral tones","texture":"plush, fabric, cozy","appearance_details":"The cushion gently supports the cat and adds a comfortable home-like feeling to the scene.","relationship":"Provides a cozy resting place for the cat.","orientation":"horizontal","pose":"","expression":"","clothing":"","gender":"","skin_tone_and_texture":""},{"name":"sunlit indoor background","description":"A softly blurred indoor background with warm daylight and peaceful home atmosphere.","location":"behind the cat, filling the upper and side areas of the frame","relative_size":"large","shape_and_color":"Soft abstract shapes in warm beige, cream, and pale golden tones","texture":"soft, blurry, bokeh-like","appearance_details":"The background is intentionally out of focus, keeping attention on the cat while creating a cozy and serene mood.","relationship":"Provides a warm and gentle environment that supports the cute domestic scene.","orientation":"upright","pose":"","expression":"","clothing":"","gender":"","skin_tone_and_texture":"","is_cluster":true,"number_of_objects":"numerous"}]}}' -n '{"universal_negative":{"visual_quality":["low quality","worst quality","blurry","pixelated","jpeg artifacts","low resolution","unstable color","color flicker","underexposed","overexposed","invisible subject","subject hidden in darkness"],"artistic_style":["painting","illustration","drawing","cartoon","3d render","cgi","sketch","digital art"],"composition_and_content":["text","watermark","signature","logo","subtitles","pillarboxed","side bars","portrait image in landscape frame"],"temporal_and_motion_stability":["flickering","jittery","motion blur","temporal inconsistency","warping","morphing","incoherent motion","unnatural movement","static object with sudden jump","frame-to-frame inconsistency"],"material_and_structure":["plastic-like glass","unrealistic texture","deformed bottle","liquid freezing improperly","distorted reflections"]}}' --offload-to-cpu --cfg-scale 3 --video-frames 33 -v ``` diff --git a/docs/llada_image.md b/docs/llada_image.md index 9b24158be..d939e78ce 100644 --- a/docs/llada_image.md +++ b/docs/llada_image.md @@ -122,7 +122,6 @@ save_file(merged, "llada_connectors.safetensors") --height 1024 \ --steps 4 \ --cfg-scale 1.0 \ - --diffusion-fa \ --output output.png ``` @@ -145,7 +144,7 @@ only in the positive branch. Editing requires connectors that include the SigVQ unchanged instead of applying the instruction; LLaDA-Image-Turbo edits correctly at both. - Editing runs the reference and the target in one sequence, so it needs roughly twice the tokens of text to image at the same size. On 12 GB, editing at 1024x1024 needs - `--diffusion-fa`; without it the diffusion graph does not fit. + flash attention; without it the diffusion graph does not fit. - The weights total about 16 GB, but segmented execution streams them, so a much smaller budget works. At 512x512, `--max-vram 6` costs almost nothing over unconstrained execution, and `--max-vram 3` still produces byte-identical output at roughly 2.5x the time. diff --git a/docs/longcat_image.md b/docs/longcat_image.md index c62849227..ae730b472 100644 --- a/docs/longcat_image.md +++ b/docs/longcat_image.md @@ -24,7 +24,7 @@ LongCat uses quoted text for character-level text rendering. Put target text ins ### LongCat Image ``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\LongCat-Image-Q4_K_M.gguf --vae ..\models\vae\ae.sft --llm ..\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf -p "a lovely cat holding a sign says 'longcat.cpp'" --cfg-scale 5.0 --sampling-method euler --flow-shift 3 -v --offload-to-cpu --diffusion-fa +.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\LongCat-Image-Q4_K_M.gguf --vae ..\models\vae\ae.sft --llm ..\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf -p "a lovely cat holding a sign says 'longcat.cpp'" --cfg-scale 5.0 --sampling-method euler --flow-shift 3 -v --offload-to-cpu ``` longcat example diff --git a/docs/ltx2.md b/docs/ltx2.md index 9caa97411..e42698ffb 100644 --- a/docs/ltx2.md +++ b/docs/ltx2.md @@ -55,7 +55,7 @@ To run the text encoder quantized, convert it once with sd-cli: ### LTX-2.3 dev T2V ``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "worst quality, low quality, blurry, distorted, artifacts" -W 1280 -H 720 --diffusion-fa --offload-to-cpu --video-frames 33 --fps 24 -o t2v.webm +.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "worst quality, low quality, blurry, distorted, artifacts" -W 1280 -H 720 --offload-to-cpu --video-frames 33 --fps 24 -o t2v.webm ```